potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 15:23:05 +04:00
Code Packages Releases Wiki Activity

Add Loongson Advanced SIMD Extension support: -DCPU_BASELINE=LASX

Browse Source 
* Add Loongson Advanced SIMD Extension support: -DCPU_BASELINE=LASX
* Add resize.lasx.cpp for Loongson SIMD acceleration
* Add imgwarp.lasx.cpp for Loongson SIMD acceleration
* Add LASX acceleration support for dnn/conv
* Add CV_PAUSE(v) for Loongarch
* Set LASX by default on Loongarch64
* LoongArch: tune test threshold for Core/HAL.mat_decomp/15

Co-authored-by: shengwenxue <shengwenxue@loongson.cn>
This commit is contained in:
wxsheng
2022-09-10 14:39:43 +08:00
committed by GitHub
parent 866191478f
commit 4154bd0667
24 changed files with 5071 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/src/int8layers/convolution_layer.cpp
+17 -1
View File
@@ -579,13 +579,14 @@ public:
bool is1x1_;
bool useAVX2;
bool useAVX512;
bool useLASX;
int blk_size_cn;
int inpZp, outZp;
const std::vector<float>* multiplier;
ParallelConv()
: input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0),
biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false)
biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false), useLASX(false)
, blk_size_cn(0), inpZp(0), outZp(0), multiplier(0)
{}
@@ -641,6 +642,8 @@ public:
p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D;
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D;
p.useLASX = checkHardwareSupport(CPU_LASX) && isConv2D;
int kernel_d = isConv3D? kernel_size[0] : 1;
int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2];
int kernel_w = kernel_size.back();
@@ -837,6 +840,13 @@ public:
stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l,
biasptr, multptr, inptr_, height, width, outptr_, out_d, outH, outW, inpZp, outZp);
else
#endif
#if CV_TRY_LASX
if(useLASX)
opt_LASX::fastDepthwiseConv(wptr, kernel_h, kernel_w,
stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l,
biasptr, multptr, inptr_, height, width, outptr_, out_d, outH, outW, inpZp, outZp);
else
#endif
{
const int8_t w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2],
@@ -1210,6 +1220,12 @@ public:
opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn);
else
#endif
#if CV_TRY_LASX
if(useLASX)
opt_LASX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn);
else
#endif
for( int i = 0; i < outCn; i += 2 )
{
modules/dnn/src/int8layers/fully_connected_layer.cpp
+8 -1
View File
@@ -226,7 +226,7 @@ public:
{
public:
FullyConnected() : srcMat(0), weights(0), biasMat(0), outputMultiplier(0), activationLUT(0), activ(0),
dstMat(0), nstripes(0), outZp(0), useAVX2(false), useAVX512(false) {}
dstMat(0), nstripes(0), outZp(0), useAVX2(false), useAVX512(false), useLASX(false) {}
static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat, const Mat& outputMultiplier,
const Mat& activationLUT, Mat& dstMat, const ActivationLayerInt8* activ, int nstripes, int outZp)
@@ -250,6 +250,7 @@ public:
p.activ = !activationLUT.empty() ? activ : 0;
p.useAVX2 = checkHardwareSupport(CPU_AVX2);
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
p.useLASX = checkHardwareSupport(CPU_LASX);
parallel_for_(Range(0, nstripes), p, nstripes);
}
@@ -294,6 +295,11 @@ public:
if( useAVX2 )
opt_AVX2::fastGEMM1T( sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp );
else
#endif
#if CV_TRY_LASX
if( useLASX )
opt_LASX::fastGEMM1T( sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp );
else
#endif
{
int i = 0;
@@ -349,6 +355,7 @@ public:
int nstripes, outZp;
bool useAVX2;
bool useAVX512;
bool useLASX;
};
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
modules/dnn/src/int8layers/layers_common.simd.hpp
+624
View File
@@ -633,5 +633,629 @@ void fastGEMM1T( const int8_t* vec, const int8_t* weights,
}
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_LASX
inline __m256i _v256_fmadds8_s32(const __m256i& a, const __m256i& b, const __m256i& c)
{
__m256i vzero = __lasx_xvreplgr2vr_d(0);
__m256i even_ab = __lasx_xvmaddwev_h_b(vzero, a, b);
__m256i madd_ab = __lasx_xvmaddwod_h_b(even_ab, a, b);
__m256i even_madd_ab = __lasx_xvsrai_w(__lasx_xvslli_w(madd_ab, 16), 16);
__m256i odd_madd_ab = __lasx_xvsrai_w(madd_ab, 16);
return __lasx_xvadd_w(__lasx_xvadd_w(even_madd_ab, odd_madd_ab), c);
}
enum { FASCONV_BASE_VECSZ = 4 };
void fastConv( const int8_t* weights, size_t wstep, const int* bias,
const int8_t* rowbuf, int* output, const int* outShape,
int blockSize, int vecsize, int vecsize_aligned, int outZp,
const float* multiplier, bool initOutput, bool finalOutput )
{
int outCn = outShape[1];
size_t outPlaneSize = outShape[2]*outShape[3];
int CV_DECL_ALIGNED(16) maskbuf[FASCONV_BASE_VECSZ] = {0};
int rsz = blockSize % FASCONV_BASE_VECSZ;
for( int i = 0; i < rsz; i++ )
maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1;
__m128i mask = __lsx_vld((const float*)maskbuf, 0);
// now compute dot product of the weights
// and im2row-transformed part of the tensor
for( int i = 0; i < outCn; i += 3 )
{
const int8_t* wptr0 = weights + i*wstep;
const int8_t* wptr1 = wptr0 + wstep;
const int8_t* wptr2 = wptr1 + wstep;
int* outptr0 = output + i*outPlaneSize;
int* outptr1 = outptr0 + outPlaneSize;
int* outptr2 = outptr1 + outPlaneSize;
int bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2];
float mult0 = multiplier[i], mult1 = multiplier[i+1], mult2 = multiplier[i+2];
if( i+2 >= outCn )
{
wptr2 = wptr1;
outptr2 = outptr1;
bias2 = bias1;
mult2 = mult1;
if( i+1 >= outCn )
{
wptr2 = wptr1 = wptr0;
outptr2 = outptr1 = outptr0;
bias2 = bias1 = bias0;
mult2 = mult1 = mult0;
}
}
int j = 0;
for( ; j < blockSize; j += FASCONV_BASE_VECSZ )
{
bool tail = false;
if (j + FASCONV_BASE_VECSZ > blockSize)
{
if (j == 0)
break;
j = blockSize - FASCONV_BASE_VECSZ;
tail = true;
}
int k = 0;
const int8_t* rptr = rowbuf + j*vecsize_aligned;
__m256i vs00 = __lasx_xvreplgr2vr_d(0), vs01 = __lasx_xvreplgr2vr_d(0),
vs02 = __lasx_xvreplgr2vr_d(0), vs03 = __lasx_xvreplgr2vr_d(0),
vs10 = __lasx_xvreplgr2vr_d(0), vs11 = __lasx_xvreplgr2vr_d(0),
vs12 = __lasx_xvreplgr2vr_d(0), vs13 = __lasx_xvreplgr2vr_d(0),
vs20 = __lasx_xvreplgr2vr_d(0), vs21 = __lasx_xvreplgr2vr_d(0),
vs22 = __lasx_xvreplgr2vr_d(0), vs23 = __lasx_xvreplgr2vr_d(0);
for (; k < vecsize; k += 32, rptr += 32 )
{
__m256i w0 = __lasx_xvld((const __m256i*)(wptr0 + k), 0);
__m256i w1 = __lasx_xvld((const __m256i*)(wptr1 + k), 0);
__m256i w2 = __lasx_xvld((const __m256i*)(wptr2 + k), 0);
__m256i r0 = __lasx_xvld((const __m256i*)(rptr), 0);
vs00 = _v256_fmadds8_s32(w0, r0, vs00);
vs10 = _v256_fmadds8_s32(w1, r0, vs10);
vs20 = _v256_fmadds8_s32(w2, r0, vs20);
r0 = __lasx_xvld((const __m256i*)(rptr + vecsize_aligned), 0);
vs01 = _v256_fmadds8_s32(w0, r0, vs01);
vs11 = _v256_fmadds8_s32(w1, r0, vs11);
vs21 = _v256_fmadds8_s32(w2, r0, vs21);
r0 = __lasx_xvld((const __m256i*)(rptr + vecsize_aligned*2), 0);
vs02 = _v256_fmadds8_s32(w0, r0, vs02);
vs12 = _v256_fmadds8_s32(w1, r0, vs12);
vs22 = _v256_fmadds8_s32(w2, r0, vs22);
r0 = __lasx_xvld((const __m256i*)(rptr + vecsize_aligned*3), 0);
vs03 = _v256_fmadds8_s32(w0, r0, vs03);
vs13 = _v256_fmadds8_s32(w1, r0, vs13);
vs23 = _v256_fmadds8_s32(w2, r0, vs23);
}
/*t0*/
__m256i vs00_hadd_w = __lasx_xvhaddw_d_w(vs00, vs00);
__m256i vs00_hadd_d = __lasx_xvhaddw_q_d(vs00_hadd_w, vs00_hadd_w);
__m256i vs01_hadd_w = __lasx_xvhaddw_d_w(vs01, vs01);
__m256i vs01_hadd_d = __lasx_xvhaddw_q_d(vs01_hadd_w, vs01_hadd_w);
__m256i vs02_hadd_w = __lasx_xvhaddw_d_w(vs02, vs02);
__m256i vs02_hadd_d = __lasx_xvhaddw_q_d(vs02_hadd_w, vs02_hadd_w);
__m256i vs03_hadd_w = __lasx_xvhaddw_d_w(vs03, vs03);
__m256i vs03_hadd_d = __lasx_xvhaddw_q_d(vs03_hadd_w, vs03_hadd_w);
__m256i vs01_vs00 = __lasx_xvpackev_w(vs01_hadd_d, vs00_hadd_d);
__m256i vs03_vs02 = __lasx_xvpackev_w(vs03_hadd_d, vs02_hadd_d);
__m256i t0 = __lasx_xvpackev_d(vs03_vs02, vs01_vs00);
/*t1*/
__m256i vs10_hadd_w = __lasx_xvhaddw_d_w(vs10, vs10);
__m256i vs10_hadd_d = __lasx_xvhaddw_q_d(vs10_hadd_w, vs10_hadd_w);
__m256i vs11_hadd_w = __lasx_xvhaddw_d_w(vs11, vs11);
__m256i vs11_hadd_d = __lasx_xvhaddw_q_d(vs11_hadd_w, vs11_hadd_w);
__m256i vs12_hadd_w = __lasx_xvhaddw_d_w(vs12, vs12);
__m256i vs12_hadd_d = __lasx_xvhaddw_q_d(vs12_hadd_w, vs12_hadd_w);
__m256i vs13_hadd_w = __lasx_xvhaddw_d_w(vs13, vs13);
__m256i vs13_hadd_d = __lasx_xvhaddw_q_d(vs13_hadd_w, vs13_hadd_w);
__m256i vs11_vs10 = __lasx_xvpackev_w(vs11_hadd_d, vs10_hadd_d);
__m256i vs13_vs12 = __lasx_xvpackev_w(vs13_hadd_d, vs12_hadd_d);
__m256i t1 = __lasx_xvpackev_d(vs13_vs12, vs11_vs10);
/*t2*/
__m256i vs20_hadd_w = __lasx_xvhaddw_d_w(vs20, vs20);
__m256i vs20_hadd_d = __lasx_xvhaddw_q_d(vs20_hadd_w, vs20_hadd_w);
__m256i vs21_hadd_w = __lasx_xvhaddw_d_w(vs21, vs21);
__m256i vs21_hadd_d = __lasx_xvhaddw_q_d(vs21_hadd_w, vs21_hadd_w);
__m256i vs22_hadd_w = __lasx_xvhaddw_d_w(vs22, vs22);
__m256i vs22_hadd_d = __lasx_xvhaddw_q_d(vs22_hadd_w, vs22_hadd_w);
__m256i vs23_hadd_w = __lasx_xvhaddw_d_w(vs23, vs23);
__m256i vs23_hadd_d = __lasx_xvhaddw_q_d(vs23_hadd_w, vs23_hadd_w);
__m256i vs21_vs20 = __lasx_xvpackev_w(vs21_hadd_d, vs20_hadd_d);
__m256i vs23_vs22 = __lasx_xvpackev_w(vs23_hadd_d, vs22_hadd_d);
__m256i t2 = __lasx_xvpackev_d(vs23_vs22, vs21_vs20);
t0 = __lasx_xvadd_w(t0, __lasx_xvpermi_q(t0, t0, 1));
t1 = __lasx_xvadd_w(t1, __lasx_xvpermi_q(t1, t1, 1));
t2 = __lasx_xvadd_w(t2, __lasx_xvpermi_q(t2, t2, 1));
__m128i s0, s1, s2;
if( initOutput )
{
s0 = __lsx_vreplgr2vr_w(bias0);
s1 = __lsx_vreplgr2vr_w(bias1);
s2 = __lsx_vreplgr2vr_w(bias2);
}
else
{
s0 = __lsx_vld((__m128i*)(outptr0 + j), 0);
s1 = __lsx_vld((__m128i*)(outptr1 + j), 0);
s2 = __lsx_vld((__m128i*)(outptr2 + j), 0);
}
s0 = __lsx_vadd_w(s0, *(__m128i*)&t0);
s1 = __lsx_vadd_w(s1, *(__m128i*)&t1);
s2 = __lsx_vadd_w(s2, *(__m128i*)&t2);
if( finalOutput )
{
__m128i voutzp = __lsx_vreplgr2vr_w(outZp);
__m128i outmin = __lsx_vreplgr2vr_w(-128), outmax = __lsx_vreplgr2vr_w(127);
__m256 v_mult0 = _v256_setall_ps(mult0);
__m256 v_mult1 = _v256_setall_ps(mult1);
__m256 v_mult2 = _v256_setall_ps(mult2);
s0 = __lsx_vadd_w(voutzp, __lsx_vftint_w_s(__lsx_vfmul_s(__lsx_vffint_s_w(s0), *(__m128*)&v_mult0)));
s1 = __lsx_vadd_w(voutzp, __lsx_vftint_w_s(__lsx_vfmul_s(__lsx_vffint_s_w(s1), *(__m128*)&v_mult1)));
s2 = __lsx_vadd_w(voutzp, __lsx_vftint_w_s(__lsx_vfmul_s(__lsx_vffint_s_w(s2), *(__m128*)&v_mult2)));
s0 = __lsx_vmin_w(__lsx_vmax_w(s0, outmin), outmax);
s1 = __lsx_vmin_w(__lsx_vmax_w(s1, outmin), outmax);
s2 = __lsx_vmin_w(__lsx_vmax_w(s2, outmin), outmax);
}
if( tail )
{
s0 = __lsx_vbitsel_v(__lsx_vld((const float*)outptr0 + j, 0), s0, mask);
s1 = __lsx_vbitsel_v(__lsx_vld((const float*)outptr1 + j, 0), s1, mask);
s2 = __lsx_vbitsel_v(__lsx_vld((const float*)outptr2 + j, 0), s2, mask);
}
__lsx_vst(s0, (__m128i*)(outptr0 + j), 0);
__lsx_vst(s1, (__m128i*)(outptr1 + j), 0);
__lsx_vst(s2, (__m128i*)(outptr2 + j), 0);
}
for( ; j <= blockSize - 2; j += 2 )
{
const int8_t* rptr0 = rowbuf + j*vecsize_aligned;
const int8_t* rptr1 = rowbuf + (j+1)*vecsize_aligned;
int s00, s01, s10, s11, s20, s21;
if( initOutput )
{
s00 = s01 = bias0;
s10 = s11 = bias1;
s20 = s21 = bias2;
}
else
{
s00 = outptr0[j]; s01 = outptr0[j+1];
s10 = outptr1[j]; s11 = outptr1[j+1];
s20 = outptr2[j]; s21 = outptr2[j+1];
}
for( int k = 0; k < vecsize; k++ )
{
int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k];
int8_t r = rptr0[k];
s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r;
r = rptr1[k];
s01 += (int)w0*r; s11 += (int)w1*r; s21 += (int)w2*r;
}
if( finalOutput )
{
s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127);
s01 = std::min(std::max(outZp + (int)std::round(s01*mult0), -128), 127);
s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127);
s11 = std::min(std::max(outZp + (int)std::round(s11*mult1), -128), 127);
s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127);
s21 = std::min(std::max(outZp + (int)std::round(s21*mult2), -128), 127);
}
outptr0[j] = s00;
outptr0[j+1] = s01;
outptr1[j] = s10;
outptr1[j+1] = s11;
outptr2[j] = s20;
outptr2[j+1] = s21;
}
for( ; j < blockSize; j++ )
{
const int8_t* rptr0 = rowbuf + j*vecsize_aligned;
int s00, s10, s20;
if( initOutput )
{
s00 = bias0;
s10 = bias1;
s20 = bias2;
}
else
{
s00 = outptr0[j];
s10 = outptr1[j];
s20 = outptr2[j];
}
for( int k = 0; k < vecsize; k++ )
{
int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k];
int8_t r = rptr0[k];
s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r;
}
if( finalOutput )
{
s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127);
s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127);
s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127);
}
outptr0[j] = s00;
outptr1[j] = s10;
outptr2[j] = s20;
}
}
}
static inline void _v256_expand_mul_add(const __m256i& a, const __m256i& b,
__m256i& out0, __m256i& out1, __m256i& out2, __m256i& out3)
{
__m256i a0 = __lasx_xvsllwil_h_b(__lasx_xvpermi_d(a, 0x10), 0);
__m256i a1 = __lasx_xvsllwil_h_b(__lasx_xvpermi_d(a, 0x32), 0);
__m256i b0 = __lasx_xvsllwil_h_b(__lasx_xvpermi_d(b, 0x10), 0);
__m256i b1 = __lasx_xvsllwil_h_b(__lasx_xvpermi_d(b, 0x32), 0);
__m256i a0b0 = __lasx_xvmul_h(a0, b0);
__m256i a1b1 = __lasx_xvmul_h(a1, b1);
out0 = __lasx_xvadd_w(out0, __lasx_xvsllwil_w_h(__lasx_xvpermi_d(a0b0, 0x10), 0));
out1 = __lasx_xvadd_w(out1, __lasx_xvsllwil_w_h(__lasx_xvpermi_d(a0b0, 0x32), 0));
out2 = __lasx_xvadd_w(out2, __lasx_xvsllwil_w_h(__lasx_xvpermi_d(a1b1, 0x10), 0));
out3 = __lasx_xvadd_w(out3, __lasx_xvsllwil_w_h(__lasx_xvpermi_d(a1b1, 0x32), 0));
}
static inline void _v256_load_deinterleave(const int8_t* ptr, __m256i& a, __m256i& b)
{
__m256i t0 = __lasx_xvld((const __m256i*)ptr, 0);
__m256i t1 = __lasx_xvld((const __m256i*)ptr, 32*1);
const __m256i sh = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
__m256i p0 = __lasx_xvshuf_b(t0, t0, sh);
__m256i p1 = __lasx_xvshuf_b(t1, t1, sh);
__m256i lo = __lasx_xvpermi_q(p0, p1, 0x02);
__m256i hi = __lasx_xvpermi_q(p0, p1, 0x13);
a = __lasx_xvilvl_d(hi, lo);
b = __lasx_xvilvh_d(hi, lo);
}
void fastDepthwiseConv( const int8_t* wptr,
int kernel_h, int kernel_w,
int stride_h, int stride_w,
int dilation_h, int dilation_w,
int pad_t, int pad_l,
const int* biasptr, const float* multptr,
const int8_t* inptr_,
int height, int width,
int* outptr_,
int out_d, int outH, int outW,
int inpZp, int outZp)
{
const int8_t w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2],
w10 = wptr[3], w11 = wptr[4], w12 = wptr[5],
w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8];
int outW1 = min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w);
float mult = multptr[out_d];
int bias = biasptr[out_d];
int biasCopy;
for (int out_i = 0; out_i < outH; out_i++)
{
int in_i = out_i * stride_h - pad_t, out_j = 0;
const int8_t* imgptr0 = inptr_ + in_i*width;
const int8_t* imgptr1 = imgptr0 + dilation_h*width;
const int8_t* imgptr2 = imgptr0 + (dilation_h*2)*width;
int8_t w00 = w00_, w01 = w01_, w02 = w02_;
int8_t w20 = w20_, w21 = w21_, w22 = w22_;
int out;
biasCopy = bias;
if (in_i < 0)
{
biasCopy += inpZp * (w00 + w01 + w02);
w00 = w01 = w02 = 0;
imgptr0 = imgptr1;
}
else if (in_i + dilation_h*(kernel_h-1) >= height)
{
biasCopy += inpZp * (w20 + w21 + w22);
w20 = w21 = w22 = 0;
imgptr2 = imgptr1;
}
int* outptr = outptr_ + out_i*outW;
if (pad_l > 0)
{
out = (int)imgptr0[0]*w01 + (int)imgptr0[dilation_w]*w02 +
(int)imgptr1[0]*w11 + (int)imgptr1[dilation_w]*w12 +
(int)imgptr2[0]*w21 + (int)imgptr2[dilation_w]*w22 +
biasCopy + inpZp*(w00 + w10 + w20);
outptr[0] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127);
out_j = 1;
}
if (stride_w == 1 || (stride_w == 2 && dilation_w == 1))
{
const int VECSZ = 32;
__m256i vw00 = __lasx_xvreplgr2vr_b(w00), vw01 = __lasx_xvreplgr2vr_b(w01), vw02 = __lasx_xvreplgr2vr_b(w02),
vw10 = __lasx_xvreplgr2vr_b(w10), vw11 = __lasx_xvreplgr2vr_b(w11), vw12 = __lasx_xvreplgr2vr_b(w12),
vw20 = __lasx_xvreplgr2vr_b(w20), vw21 = __lasx_xvreplgr2vr_b(w21), vw22 = __lasx_xvreplgr2vr_b(w22);
__m256i vbias = __lasx_xvreplgr2vr_w(biasCopy), voutzp = __lasx_xvreplgr2vr_w(outZp),
outmin = __lasx_xvreplgr2vr_w(-128), outmax = __lasx_xvreplgr2vr_w(127);
__m256 vmult = _v256_setall_ps(mult);
__m256i vout0, vout1, vout2, vout3;
if( stride_w == 1 )
{
for( ; out_j < outW1; out_j += VECSZ )
{
if (out_j + VECSZ > outW1)
{
if (out_j <= pad_l)
break;
out_j = outW1 - VECSZ;
}
int in_j = out_j * stride_w - pad_l;
__m256i v00 = __lasx_xvld((const __m256i*)(imgptr0 + in_j), 0),
v01 = __lasx_xvld((const __m256i*)(imgptr0 + in_j + dilation_w), 0),
v02 = __lasx_xvld((const __m256i*)(imgptr0 + in_j + dilation_w*2), 0),
v10 = __lasx_xvld((const __m256i*)(imgptr1 + in_j), 0),
v11 = __lasx_xvld((const __m256i*)(imgptr1 + in_j + dilation_w), 0),
v12 = __lasx_xvld((const __m256i*)(imgptr1 + in_j + dilation_w*2), 0),
v20 = __lasx_xvld((const __m256i*)(imgptr2 + in_j), 0),
v21 = __lasx_xvld((const __m256i*)(imgptr2 + in_j + dilation_w), 0),
v22 = __lasx_xvld((const __m256i*)(imgptr2 + in_j + dilation_w*2), 0);
vout0 = vout1 = vout2 = vout3 = vbias;
_v256_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3);
vout0 = __lasx_xvadd_w(voutzp, __lasx_xvftint_w_s(__lasx_xvfmul_s(__lasx_xvffint_s_w(vout0), vmult)));
vout1 = __lasx_xvadd_w(voutzp, __lasx_xvftint_w_s(__lasx_xvfmul_s(__lasx_xvffint_s_w(vout1), vmult)));
vout2 = __lasx_xvadd_w(voutzp, __lasx_xvftint_w_s(__lasx_xvfmul_s(__lasx_xvffint_s_w(vout2), vmult)));
vout3 = __lasx_xvadd_w(voutzp, __lasx_xvftint_w_s(__lasx_xvfmul_s(__lasx_xvffint_s_w(vout3), vmult)));
vout0 = __lasx_xvmin_w(__lasx_xvmax_w(vout0, outmin), outmax);
vout1 = __lasx_xvmin_w(__lasx_xvmax_w(vout1, outmin), outmax);
vout2 = __lasx_xvmin_w(__lasx_xvmax_w(vout2, outmin), outmax);
vout3 = __lasx_xvmin_w(__lasx_xvmax_w(vout3, outmin), outmax);
__lasx_xvst(vout0, (__m256i*)(outptr + out_j), 0);
__lasx_xvst(vout1, (__m256i*)(outptr + out_j), 8*4);
__lasx_xvst(vout2, (__m256i*)(outptr + out_j), 16*4);
__lasx_xvst(vout3, (__m256i*)(outptr + out_j), 24*4);
}
}
else
{
for( ; out_j < outW1; out_j += VECSZ )
{
if (out_j + VECSZ > outW1)
{
if (out_j <= pad_l)
break;
out_j = outW1 - VECSZ;
}
int in_j = out_j * stride_w - pad_l;
__m256i v00, v01, v02, v10, v11, v12, v20, v21, v22, unused;
_v256_load_deinterleave(imgptr0 + in_j, v00, v01);
_v256_load_deinterleave(imgptr0 + in_j + 2, v02, unused);
_v256_load_deinterleave(imgptr1 + in_j, v10, v11);
_v256_load_deinterleave(imgptr1 + in_j + 2, v12, unused);
_v256_load_deinterleave(imgptr2 + in_j, v20, v21);
_v256_load_deinterleave(imgptr2 + in_j + 2, v22, unused);
vout0 = vout1 = vout2 = vout3 = vbias;
_v256_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3);
_v256_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3);
vout0 = __lasx_xvadd_w(voutzp, __lasx_xvftint_w_s(__lasx_xvfmul_s(__lasx_xvffint_s_w(vout0), vmult)));
vout1 = __lasx_xvadd_w(voutzp, __lasx_xvftint_w_s(__lasx_xvfmul_s(__lasx_xvffint_s_w(vout1), vmult)));
vout2 = __lasx_xvadd_w(voutzp, __lasx_xvftint_w_s(__lasx_xvfmul_s(__lasx_xvffint_s_w(vout2), vmult)));
vout3 = __lasx_xvadd_w(voutzp, __lasx_xvftint_w_s(__lasx_xvfmul_s(__lasx_xvffint_s_w(vout3), vmult)));
vout0 = __lasx_xvmin_w(__lasx_xvmax_w(vout0, outmin), outmax);
vout1 = __lasx_xvmin_w(__lasx_xvmax_w(vout1, outmin), outmax);
vout2 = __lasx_xvmin_w(__lasx_xvmax_w(vout2, outmin), outmax);
vout3 = __lasx_xvmin_w(__lasx_xvmax_w(vout3, outmin), outmax);
__lasx_xvst(vout0, (__m256i*)(outptr + out_j), 0);
__lasx_xvst(vout1, (__m256i*)(outptr + out_j), 8*4);
__lasx_xvst(vout2, (__m256i*)(outptr + out_j), 16*4);
__lasx_xvst(vout3, (__m256i*)(outptr + out_j), 24*4);
}
}
}
for (; out_j < outW1; out_j++)
{
int in_j = out_j * stride_w - pad_l;
out = (int)imgptr0[in_j]*w00 + (int)imgptr0[in_j + dilation_w]*w01 + (int)imgptr0[in_j + dilation_w*2]*w02 +
(int)imgptr1[in_j]*w10 + (int)imgptr1[in_j + dilation_w]*w11 + (int)imgptr1[in_j + dilation_w*2]*w12 +
(int)imgptr2[in_j]*w20 + (int)imgptr2[in_j + dilation_w]*w21 + (int)imgptr2[in_j + dilation_w*2]*w22 + biasCopy;
outptr[out_j] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127);
}
for (; out_j < outW; out_j++ )
{
int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2;
int s0 = 1, s1 = 1, s2 = 1;
if (in_j0 >= width)
{
in_j0 = 0;
s0 = 0;
biasCopy += inpZp*(w00 + w10 + w20);
}
if (in_j1 >= width)
{
in_j1 = 0;
s1 = 0;
biasCopy += inpZp*(w01 + w11 + w21);
}
if (in_j2 >= width)
{
in_j2 = 0;
s2 = 0;
biasCopy += inpZp*(w02 + w12 + w22);
}
out = (int)imgptr0[in_j0]*w00*s0 + (int)imgptr0[in_j1]*w01*s1 + (int)imgptr0[in_j2]*w02*s2 +
(int)imgptr1[in_j0]*w10*s0 + (int)imgptr1[in_j1]*w11*s1 + (int)imgptr1[in_j2]*w12*s2 +
(int)imgptr2[in_j0]*w20*s0 + (int)imgptr2[in_j1]*w21*s1 + (int)imgptr2[in_j2]*w22*s2 + biasCopy;
outptr[out_j] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127);
}
}
}
// dst = vec * weights^t + bias
void fastGEMM1T( const int8_t* vec, const int8_t* weights,
size_t wstep, const int* bias, const float* multiplier,
int* dst, int nvecs, int vecsize, int outZp )
{
int i = 0;
for( ; i <= nvecs - 8; i += 8 )
{
const int8_t* wptr = weights + i*wstep;
__m256i vs0 = __lasx_xvreplgr2vr_d(0), vs1 = __lasx_xvreplgr2vr_d(0),
vs2 = __lasx_xvreplgr2vr_d(0), vs3 = __lasx_xvreplgr2vr_d(0),
vs4 = __lasx_xvreplgr2vr_d(0), vs5 = __lasx_xvreplgr2vr_d(0),
vs6 = __lasx_xvreplgr2vr_d(0), vs7 = __lasx_xvreplgr2vr_d(0);
__m128i voutzp = __lsx_vreplgr2vr_w(outZp);
__m128i outmin = __lsx_vreplgr2vr_w(-128), outmax = __lsx_vreplgr2vr_w(127);
for( int k = 0; k < vecsize; k += 32, wptr += 32 )
{
__m256i v = __lasx_xvld((const __m256i*)(vec + k), 0);
vs0 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)wptr, 0), v, vs0);
vs1 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)(wptr + wstep), 0), v, vs1);
vs2 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)(wptr + wstep*2), 0), v, vs2);
vs3 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)(wptr + wstep*3), 0), v, vs3);
vs4 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)(wptr + wstep*4), 0), v, vs4);
vs5 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)(wptr + wstep*5), 0), v, vs5);
vs6 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)(wptr + wstep*6), 0), v, vs6);
vs7 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)(wptr + wstep*7), 0), v, vs7);
}
/*s0*/
__m256i vs0_hadd_w = __lasx_xvhaddw_d_w(vs0, vs0);
__m256i vs0_hadd_d = __lasx_xvhaddw_q_d(vs0_hadd_w, vs0_hadd_w);
__m256i vs1_hadd_w = __lasx_xvhaddw_d_w(vs1, vs1);
__m256i vs1_hadd_d = __lasx_xvhaddw_q_d(vs1_hadd_w, vs1_hadd_w);
__m256i vs2_hadd_w = __lasx_xvhaddw_d_w(vs2, vs2);
__m256i vs2_hadd_d = __lasx_xvhaddw_q_d(vs2_hadd_w, vs2_hadd_w);
__m256i vs3_hadd_w = __lasx_xvhaddw_d_w(vs3, vs3);
__m256i vs3_hadd_d = __lasx_xvhaddw_q_d(vs3_hadd_w, vs3_hadd_w);
__m256i vs1_vs0 = __lasx_xvpackev_w(vs1_hadd_d, vs0_hadd_d);
__m256i vs3_vs2 = __lasx_xvpackev_w(vs3_hadd_d, vs2_hadd_d);
__m256i s0 = __lasx_xvpackev_d(vs3_vs2, vs1_vs0);
/*s1*/
__m256i vs4_hadd_w = __lasx_xvhaddw_d_w(vs4, vs4);
__m256i vs4_hadd_d = __lasx_xvhaddw_q_d(vs4_hadd_w, vs4_hadd_w);
__m256i vs5_hadd_w = __lasx_xvhaddw_d_w(vs5, vs5);
__m256i vs5_hadd_d = __lasx_xvhaddw_q_d(vs5_hadd_w, vs5_hadd_w);
__m256i vs6_hadd_w = __lasx_xvhaddw_d_w(vs6, vs6);
__m256i vs6_hadd_d = __lasx_xvhaddw_q_d(vs6_hadd_w, vs6_hadd_w);
__m256i vs7_hadd_w = __lasx_xvhaddw_d_w(vs7, vs7);
__m256i vs7_hadd_d = __lasx_xvhaddw_q_d(vs7_hadd_w, vs7_hadd_w);
__m256i vs5_vs4 = __lasx_xvpackev_w(vs5_hadd_d, vs4_hadd_d);
__m256i vs7_vs6 = __lasx_xvpackev_w(vs7_hadd_d, vs6_hadd_d);
__m256i s1 = __lasx_xvpackev_d(vs7_vs6, vs5_vs4);
s0 = __lasx_xvadd_w(s0, __lasx_xvpermi_q(s0, s0, 1));
s1 = __lasx_xvadd_w(s1, __lasx_xvpermi_q(s1, s1, 1));
__m128i t0 = __lsx_vadd_w(*(__m128i*)(&s0), __lsx_vld((__m128i*)(bias + i), 0));
__m128i t1 = __lsx_vadd_w(*(__m128i*)(&s1), __lsx_vld((__m128i*)(bias + i), 4*4));
t0 = __lsx_vadd_w(voutzp, __lsx_vftint_w_s(__lsx_vfmul_s(__lsx_vffint_s_w(t0), (__m128)__lsx_vld(multiplier + i, 0))));
t1 = __lsx_vadd_w(voutzp, __lsx_vftint_w_s(__lsx_vfmul_s(__lsx_vffint_s_w(t1), (__m128)__lsx_vld(multiplier + i, 4*4))));
t0 = __lsx_vmin_w(__lsx_vmax_w(t0, outmin), outmax);
t1 = __lsx_vmin_w(__lsx_vmax_w(t1, outmin), outmax);
__lsx_vst(t0, (__m128i*)(dst + i), 0);
__lsx_vst(t1, (__m128i*)(dst + i), 4*4);
}
for( ; i < nvecs; i++ )
{
const int8_t* wptr = weights + i*wstep;
__m256i vs0 = __lasx_xvreplgr2vr_d(0);
for( int k = 0; k < vecsize; k += 32, wptr += 32 )
{
__m256i v = __lasx_xvld((const __m256i*)(vec + k), 0);
vs0 = _v256_fmadds8_s32(__lasx_xvld((const __m256i*)wptr, 0), v, vs0);
}
__m256i s0_hadd_w = __lasx_xvhaddw_d_w(vs0, vs0);
int temp = ((v4i64)s0_hadd_w)[0] + ((v4i64)s0_hadd_w)[1] + ((v4i64)s0_hadd_w)[2] + ((v4i64)s0_hadd_w)[3];
dst[i] = outZp + (int)std::round((temp + bias[i]) * multiplier[i]);
}
}
#endif // CV_LASX
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // namespace
modules/dnn/src/layers/convolution_layer.cpp
+23 -1
View File
@@ -986,12 +986,13 @@ public:
bool useAVX2;
bool useAVX512;
bool useRVV;
bool useLASX;
int blk_size_cn;
ParallelConv()
: input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0),
biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false)
, blk_size_cn(0)
, useLASX(false), blk_size_cn(0)
{}
static void run( const Mat& input, Mat& output, const Mat& weights,
@@ -1049,6 +1050,7 @@ public:
p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D;
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D;
p.useRVV = checkHardwareSupport(CPU_RVV) && isConv2D;
p.useLASX = checkHardwareSupport(CPU_LASX) && isConv2D;
int kernel_d = isConv3D? kernel_size[0] : 1;
int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2];
@@ -1256,6 +1258,13 @@ public:
stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l,
biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW);
else
#endif
#if CV_TRY_LASX
if(useLASX)
opt_LASX::fastDepthwiseConv(wptr, kernel_h, kernel_w,
stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l,
biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW);
else
#endif
{
const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2],
@@ -1631,6 +1640,12 @@ public:
opt_RVV::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
else
#endif
#if CV_TRY_LASX
if(useLASX)
opt_LASX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
else
#endif
for( int i = 0; i < outCn; i += 2 )
{
@@ -2437,6 +2452,7 @@ public:
useAVX2 = checkHardwareSupport(CPU_AVX2);
useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
useRVV = checkHardwareSupport(CPU_RVV);
useLASX = checkHardwareSupport(CPU_LASX);
}
void operator()(const Range& range_) const CV_OVERRIDE
@@ -2474,6 +2490,11 @@ public:
opt_RVV::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
}
else
#endif
#if CV_TRY_LASX
if( useLASX )
opt_LASX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
else
#endif
for( m = 0; m < mmax; m += 2 )
{
@@ -2574,6 +2595,7 @@ public:
bool useAVX2;
bool useAVX512;
bool useRVV;
bool useLASX;
};
class Col2ImInvoker : public cv::ParallelLoopBody
modules/dnn/src/layers/fully_connected_layer.cpp
+8 -1
View File
@@ -173,7 +173,7 @@ public:
class FullyConnected : public ParallelLoopBody
{
public:
FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false) {}
FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false), useLASX(false) {}
static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat,
Mat& dstMat, const ActivationLayer* activ, int nstripes)
@@ -197,6 +197,7 @@ public:
p.useAVX2 = checkHardwareSupport(CPU_AVX2);
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
p.useRVV = checkHardwareSupport(CPU_RVV);
p.useLASX = checkHardwareSupport(CPU_LASX);
parallel_for_(Range(0, nstripes), p, nstripes);
}
@@ -250,6 +251,11 @@ public:
if( useRVV )
opt_RVV::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize);
else
#endif
#if CV_TRY_LASX
if( useLASX )
opt_LASX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize);
else
#endif
{
int i = 0;
@@ -305,6 +311,7 @@ public:
bool useAVX2;
bool useAVX512;
bool useRVV;
bool useLASX;
};
#ifdef HAVE_OPENCL
modules/dnn/src/layers/layers_common.simd.hpp
+679
View File
@@ -1343,5 +1343,684 @@ void fastDepthwiseConv( const float* wptr,
#endif // CV_RVV
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_LASX
enum { FASCONV_BASE_VECSZ = 4 };
void fastConv( const float* weights, size_t wstep, const float* bias,
const float* rowbuf, float* output, const int* outShape,
int blockSize, int vecsize, int vecsize_aligned,
const float* relu, bool initOutput )
{
int outCn = outShape[1];
size_t outPlaneSize = outShape[2]*outShape[3];
float r0 = 1.f, r1 = 1.f, r2 = 1.f;
__m256 t1 = _v256_setall_ps(1.f), t2 = _v256_setall_ps(0.f);
__m128 vr0 = *(__m128*)&t1, vr1 = vr0, vr2 = vr0, z = *(__m128*)&t2;
int CV_DECL_ALIGNED(16) maskbuf[FASCONV_BASE_VECSZ] = {0};
int rsz = blockSize % FASCONV_BASE_VECSZ;
for( int i = 0; i < rsz; i++ )
maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1;
__m128i mask = __lsx_vld((const float*)maskbuf, 0);
// now compute dot product of the weights
// and im2row-transformed part of the tensor
for( int i = 0; i < outCn; i += 3 )
{
const float* wptr0 = weights + i*wstep;
const float* wptr1 = wptr0 + wstep;
const float* wptr2 = wptr1 + wstep;
float* outptr0 = output + i*outPlaneSize;
float* outptr1 = outptr0 + outPlaneSize;
float* outptr2 = outptr1 + outPlaneSize;
float bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2];
if( i+2 >= outCn )
{
wptr2 = wptr1;
outptr2 = outptr1;
bias2 = bias1;
if( i+1 >= outCn )
{
wptr2 = wptr1 = wptr0;
outptr2 = outptr1 = outptr0;
bias2 = bias1 = bias0;
}
}
if( relu )
{
r0 = relu[i]; r1 = relu[i+1]; r2 = relu[i+2];
if( i+2 >= outCn )
{
r2 = r1;
if( i+1 >= outCn )
r2 = r1 = r0;
}
vr0 = _v256_extract_low(_v256_setall_ps(r0));
vr1 = _v256_extract_low(_v256_setall_ps(r1));
vr2 = _v256_extract_low(_v256_setall_ps(r2));
}
int j = 0;
for( ; j < blockSize; j += FASCONV_BASE_VECSZ )
{
bool tail = false;
if (j + FASCONV_BASE_VECSZ > blockSize)
{
if (j == 0)
break;
j = blockSize - FASCONV_BASE_VECSZ;
tail = true;
}
int k = 0;
const float* rptr = rowbuf + j*vecsize_aligned;
__m256i tmp;
__m256 vs00 = (__m256)__lasx_xvxor_v(tmp, tmp), vs01 = (__m256)__lasx_xvxor_v(tmp, tmp),
vs02 = (__m256)__lasx_xvxor_v(tmp, tmp), vs03 = (__m256)__lasx_xvxor_v(tmp, tmp),
vs10 = (__m256)__lasx_xvxor_v(tmp, tmp), vs11 = (__m256)__lasx_xvxor_v(tmp, tmp),
vs12 = (__m256)__lasx_xvxor_v(tmp, tmp), vs13 = (__m256)__lasx_xvxor_v(tmp, tmp),
vs20 = (__m256)__lasx_xvxor_v(tmp, tmp), vs21 = (__m256)__lasx_xvxor_v(tmp, tmp),
vs22 = (__m256)__lasx_xvxor_v(tmp, tmp), vs23 = (__m256)__lasx_xvxor_v(tmp, tmp);
for (; k < vecsize; k += 8, rptr += 8 )
{
__m256 w0 = (__m256)__lasx_xvld(wptr0 + k, 0);
__m256 w1 = (__m256)__lasx_xvld(wptr1 + k, 0);
__m256 w2 = (__m256)__lasx_xvld(wptr2 + k, 0);
__m256 r0 = (__m256)__lasx_xvld(rptr, 0);
vs00 = __lasx_xvfmadd_s(w0, r0, vs00);
vs10 = __lasx_xvfmadd_s(w1, r0, vs10);
vs20 = __lasx_xvfmadd_s(w2, r0, vs20);
r0 = (__m256)__lasx_xvld(rptr + vecsize_aligned, 0);
vs01 = __lasx_xvfmadd_s(w0, r0, vs01);
vs11 = __lasx_xvfmadd_s(w1, r0, vs11);
vs21 = __lasx_xvfmadd_s(w2, r0, vs21);
r0 = (__m256)__lasx_xvld(rptr + vecsize_aligned*2, 0);
vs02 = __lasx_xvfmadd_s(w0, r0, vs02);
vs12 = __lasx_xvfmadd_s(w1, r0, vs12);
vs22 = __lasx_xvfmadd_s(w2, r0, vs22);
r0 = (__m256)__lasx_xvld(rptr + vecsize_aligned*3, 0);
vs03 = __lasx_xvfmadd_s(w0, r0, vs03);
vs13 = __lasx_xvfmadd_s(w1, r0, vs13);
vs23 = __lasx_xvfmadd_s(w2, r0, vs23);
}
/*t0*/
__m256 vs00_perm = (__m256)__lasx_xvpermi_d(vs00, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs00_add_2w = __lasx_xvfadd_s(vs00, vs00_perm);
__m256 tmp00_srl = (__m256)__lasx_xvsrli_d(vs00_add_2w, 32);
__m256 vs00_add_4w = __lasx_xvfadd_s(vs00_add_2w, tmp00_srl);
__m256 vs01_perm = (__m256)__lasx_xvpermi_d(vs01, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs01_add_2w = __lasx_xvfadd_s(vs01, vs01_perm);
__m256 tmp01_srl = (__m256)__lasx_xvsrli_d(vs01_add_2w, 32);
__m256 vs01_add_4w = __lasx_xvfadd_s(vs01_add_2w, tmp01_srl);
__m256 vs02_perm = (__m256)__lasx_xvpermi_d(vs02, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs02_add_2w = __lasx_xvfadd_s(vs02, vs02_perm);
__m256 tmp02_srl = (__m256)__lasx_xvsrli_d(vs02_add_2w, 32);
__m256 vs02_add_4w = __lasx_xvfadd_s(vs02_add_2w, tmp02_srl);
__m256 vs03_perm = (__m256)__lasx_xvpermi_d(vs03, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs03_add_2w = __lasx_xvfadd_s(vs03, vs03_perm);
__m256 tmp03_srl = (__m256)__lasx_xvsrli_d(vs03_add_2w, 32);
__m256 vs03_add_4w = __lasx_xvfadd_s(vs03_add_2w, tmp03_srl);
__m256i vs01_vs00 = __lasx_xvpackev_w((__m256i)vs01_add_4w, (__m256i)vs00_add_4w);
__m256i vs03_vs02 = __lasx_xvpackev_w((__m256i)vs03_add_4w, (__m256i)vs02_add_4w);
__m256 t0 = (__m256)__lasx_xvpackev_d(vs03_vs02, vs01_vs00);
/*t1*/
__m256 vs10_perm = (__m256)__lasx_xvpermi_d(vs10, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs10_add_2w = __lasx_xvfadd_s(vs10, vs10_perm);
__m256 tmp10_srl = (__m256)__lasx_xvsrli_d(vs10_add_2w, 32);
__m256 vs10_add_4w = __lasx_xvfadd_s(vs10_add_2w, tmp10_srl);
__m256 vs11_perm = (__m256)__lasx_xvpermi_d(vs11, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs11_add_2w = __lasx_xvfadd_s(vs11, vs11_perm);
__m256 tmp11_srl = (__m256)__lasx_xvsrli_d(vs11_add_2w, 32);
__m256 vs11_add_4w = __lasx_xvfadd_s(vs11_add_2w, tmp11_srl);
__m256 vs12_perm = (__m256)__lasx_xvpermi_d(vs12, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs12_add_2w = __lasx_xvfadd_s(vs12, vs12_perm);
__m256 tmp12_srl = (__m256)__lasx_xvsrli_d(vs12_add_2w, 32);
__m256 vs12_add_4w = __lasx_xvfadd_s(vs12_add_2w, tmp12_srl);
__m256 vs13_perm = (__m256)__lasx_xvpermi_d(vs13, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs13_add_2w = __lasx_xvfadd_s(vs13, vs13_perm);
__m256 tmp13_srl = (__m256)__lasx_xvsrli_d(vs13_add_2w, 32);
__m256 vs13_add_4w = __lasx_xvfadd_s(vs13_add_2w, tmp13_srl);
__m256i vs11_vs10 = __lasx_xvpackev_w((__m256i)vs11_add_4w, (__m256i)vs10_add_4w);
__m256i vs13_vs12 = __lasx_xvpackev_w((__m256i)vs13_add_4w, (__m256i)vs12_add_4w);
__m256 t1 = (__m256)__lasx_xvpackev_d(vs13_vs12, vs11_vs10);
/*t2*/
__m256 vs20_perm = (__m256)__lasx_xvpermi_d(vs20, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs20_add_2w = __lasx_xvfadd_s(vs20, vs20_perm);
__m256 tmp20_srl = (__m256)__lasx_xvsrli_d(vs20_add_2w, 32);
__m256 vs20_add_4w = __lasx_xvfadd_s(vs20_add_2w, tmp20_srl);
__m256 vs21_perm = (__m256)__lasx_xvpermi_d(vs21, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs21_add_2w = __lasx_xvfadd_s(vs21, vs21_perm);
__m256 tmp21_srl = (__m256)__lasx_xvsrli_d(vs21_add_2w, 32);
__m256 vs21_add_4w = __lasx_xvfadd_s(vs21_add_2w, tmp21_srl);
__m256 vs22_perm = (__m256)__lasx_xvpermi_d(vs22, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs22_add_2w = __lasx_xvfadd_s(vs22, vs22_perm);
__m256 tmp22_srl = (__m256)__lasx_xvsrli_d(vs22_add_2w, 32);
__m256 vs22_add_4w = __lasx_xvfadd_s(vs22_add_2w, tmp22_srl);
__m256 vs23_perm = (__m256)__lasx_xvpermi_d(vs23, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs23_add_2w = __lasx_xvfadd_s(vs23, vs23_perm);
__m256 tmp23_srl = (__m256)__lasx_xvsrli_d(vs23_add_2w, 32);
__m256 vs23_add_4w = __lasx_xvfadd_s(vs23_add_2w, tmp23_srl);
__m256i vs21_vs20 = __lasx_xvpackev_w((__m256i)vs21_add_4w, (__m256i)vs20_add_4w);
__m256i vs23_vs22 = __lasx_xvpackev_w((__m256i)vs23_add_4w, (__m256i)vs22_add_4w);
__m256 t2 = (__m256)__lasx_xvpackev_d(vs23_vs22, vs21_vs20);
t0 = __lasx_xvfadd_s(t0, (__m256)__lasx_xvpermi_q(t0, t0, 1));
t1 = __lasx_xvfadd_s(t1, (__m256)__lasx_xvpermi_q(t1, t1, 1));
t2 = __lasx_xvfadd_s(t2, (__m256)__lasx_xvpermi_q(t2, t2, 1));
__m128 s0, s1, s2;
if( initOutput )
{
s0 = _v256_extract_low(_v256_setall_ps(bias0));
s1 = _v256_extract_low(_v256_setall_ps(bias1));
s2 = _v256_extract_low(_v256_setall_ps(bias2));
}
else
{
s0 = (__m128)__lsx_vld(outptr0 + j, 0);
s1 = (__m128)__lsx_vld(outptr1 + j, 0);
s2 = (__m128)__lsx_vld(outptr2 + j, 0);
}
s0 = __lsx_vfadd_s(s0, *(__m128*)&t0);
s1 = __lsx_vfadd_s(s1, *(__m128*)&t1);
s2 = __lsx_vfadd_s(s2, *(__m128*)&t2);
if( relu )
{
__m128i m0 = __lsx_vfcmp_clt_s(z, s0);
__m128i m1 = __lsx_vfcmp_clt_s(z, s1);
__m128i m2 = __lsx_vfcmp_clt_s(z, s2);
s0 = (__m128)__lsx_vbitsel_v((__m128i)__lsx_vfmul_s(s0, vr0), (__m128i)s0, m0);
s1 = (__m128)__lsx_vbitsel_v((__m128i)__lsx_vfmul_s(s1, vr1), (__m128i)s1, m1);
s2 = (__m128)__lsx_vbitsel_v((__m128i)__lsx_vfmul_s(s2, vr2), (__m128i)s2, m2);
}
if( tail )
{
s0 = (__m128)__lsx_vbitsel_v(__lsx_vld(outptr0 + j, 0), (__m128i)s0, mask);
s1 = (__m128)__lsx_vbitsel_v(__lsx_vld(outptr1 + j, 0), (__m128i)s1, mask);
s2 = (__m128)__lsx_vbitsel_v(__lsx_vld(outptr2 + j, 0), (__m128i)s2, mask);
}
__lsx_vst(s0, outptr0 + j, 0);
__lsx_vst(s1, outptr1 + j, 0);
__lsx_vst(s2, outptr2 + j, 0);
}
for( ; j <= blockSize - 2; j += 2 )
{
const float* rptr0 = rowbuf + j*vecsize_aligned;
const float* rptr1 = rowbuf + (j+1)*vecsize_aligned;
float s00, s01, s10, s11, s20, s21;
if( initOutput )
{
s00 = s01 = bias0;
s10 = s11 = bias1;
s20 = s21 = bias2;
}
else
{
s00 = outptr0[j]; s01 = outptr0[j+1];
s10 = outptr1[j]; s11 = outptr1[j+1];
s20 = outptr2[j]; s21 = outptr2[j+1];
}
for( int k = 0; k < vecsize; k++ )
{
float w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k];
float r = rptr0[k];
s00 += w0*r; s10 += w1*r; s20 += w2*r;
r = rptr1[k];
s01 += w0*r; s11 += w1*r; s21 += w2*r;
}
if( relu )
{
s00 = s00 > 0.f ? s00 : s00*r0;
s01 = s01 > 0.f ? s01 : s01*r0;
s10 = s10 > 0.f ? s10 : s10*r1;
s11 = s11 > 0.f ? s11 : s11*r1;
s20 = s20 > 0.f ? s20 : s20*r2;
s21 = s21 > 0.f ? s21 : s21*r2;
}
outptr0[j] = s00;
outptr0[j+1] = s01;
outptr1[j] = s10;
outptr1[j+1] = s11;
outptr2[j] = s20;
outptr2[j+1] = s21;
}
for( ; j < blockSize; j++ )
{
const float* rptr0 = rowbuf + j*vecsize_aligned;
float s00, s10, s20;
if( initOutput )
{
s00 = bias0;
s10 = bias1;
s20 = bias2;
}
else
{
s00 = outptr0[j];
s10 = outptr1[j];
s20 = outptr2[j];
}
for( int k = 0; k < vecsize; k++ )
{
float w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k];
float r = rptr0[k];
s00 += w0*r; s10 += w1*r; s20 += w2*r;
}
if( relu )
{
s00 = s00 > 0.f ? s00 : s00*r0;
s10 = s10 > 0.f ? s10 : s10*r1;
s20 = s20 > 0.f ? s20 : s20*r2;
}
outptr0[j] = s00;
outptr1[j] = s10;
outptr2[j] = s20;
}
}
}
static inline void _v256_load_deinterleave(const float* ptr, __m256& a, __m256& b)
{
__m256 t0 = (__m256)__lasx_xvld(ptr, 0);
__m256 t1 = (__m256)__lasx_xvld(ptr, 8*4);
__m256 lo = (__m256)__lasx_xvpermi_q(t0, t1, 2+0*16);
__m256 hi = (__m256)__lasx_xvpermi_q(t0, t1, 3+1*16);
a = (__m256)__lasx_xvpermi_w(hi, lo, 0x88);
b = (__m256)__lasx_xvpermi_w(hi, lo, 0xdd);
}
void fastDepthwiseConv( const float* wptr,
int kernel_h, int kernel_w,
int stride_h, int stride_w,
int dilation_h, int dilation_w,
int pad_t, int pad_l,
const float* biasptr, const float* relu,
const float* inptr_,
int height, int width,
float* outptr_,
int out_d, int outH, int outW )
{
const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2],
w10 = wptr[3], w11 = wptr[4], w12 = wptr[5],
w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8];
int outW1 = min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w);
float relu_coeff = relu ? relu[out_d] : 1.f, bias = biasptr[out_d];
for (int out_i = 0; out_i < outH; out_i++)
{
int in_i = out_i * stride_h - pad_t, out_j = 0;
const float* imgptr0 = inptr_ + in_i*width;
const float* imgptr1 = imgptr0 + dilation_h*width;
const float* imgptr2 = imgptr0 + (dilation_h*2)*width;
float out, w00 = w00_, w01 = w01_, w02 = w02_;
float w20 = w20_, w21 = w21_, w22 = w22_;
if (in_i < 0)
{
w00 = w01 = w02 = 0.f;
imgptr0 = imgptr1;
}
else if (in_i + dilation_h*(kernel_h-1) >= height)
{
w20 = w21 = w22 = 0.f;
imgptr2 = imgptr1;
}
float* outptr = outptr_ + out_i*outW;
if (pad_l > 0)
{
out = imgptr0[0]*w01 + imgptr0[dilation_w]*w02 +
imgptr1[0]*w11 + imgptr1[dilation_w]*w12 +
imgptr2[0]*w21 + imgptr2[dilation_w]*w22 + bias;
if (relu)
out = out > 0.f ? out : out*relu_coeff;
outptr[0] = out;
out_j = 1;
}
if (stride_w == 1 || (stride_w == 2 && dilation_w == 1))
{
const int VECSZ = 8;
__m256 vw00 = _v256_setall_ps(w00), vw01 = _v256_setall_ps(w01), vw02 = _v256_setall_ps(w02),
vw10 = _v256_setall_ps(w10), vw11 = _v256_setall_ps(w11), vw12 = _v256_setall_ps(w12),
vw20 = _v256_setall_ps(w20), vw21 = _v256_setall_ps(w21), vw22 = _v256_setall_ps(w22);
__m256 z = (__m256)__lasx_xvxor_v((__m256i)vw00, (__m256i)vw00),
vbias = _v256_setall_ps(bias), vrc = _v256_setall_ps(relu_coeff);
if( stride_w == 1 )
for( ; out_j < outW1; out_j += VECSZ )
{
if (out_j + VECSZ > outW1 && out_j > pad_l)
out_j = outW1 - VECSZ;
int in_j = out_j * stride_w - pad_l;
__m256 v00 = (__m256)__lasx_xvld(imgptr0 + in_j, 0),
v01 = (__m256)__lasx_xvld(imgptr0 + in_j + dilation_w, 0),
v02 = (__m256)__lasx_xvld(imgptr0 + in_j + dilation_w*2, 0),
v10 = (__m256)__lasx_xvld(imgptr1 + in_j, 0),
v11 = (__m256)__lasx_xvld(imgptr1 + in_j + dilation_w, 0),
v12 = (__m256)__lasx_xvld(imgptr1 + in_j + dilation_w*2, 0),
v20 = (__m256)__lasx_xvld(imgptr2 + in_j, 0),
v21 = (__m256)__lasx_xvld(imgptr2 + in_j + dilation_w, 0),
v22 = (__m256)__lasx_xvld(imgptr2 + in_j + dilation_w*2, 0);
__m256 vout0 = __lasx_xvfmadd_s(v00, vw00, vbias);
__m256 vout1 = __lasx_xvfmul_s(v01, vw01);
__m256 vout2 = __lasx_xvfmul_s(v02, vw02);
vout0 = __lasx_xvfmadd_s(v10, vw10, vout0);
vout1 = __lasx_xvfmadd_s(v11, vw11, vout1);
vout2 = __lasx_xvfmadd_s(v12, vw12, vout2);
vout0 = __lasx_xvfmadd_s(v20, vw20, vout0);
vout1 = __lasx_xvfmadd_s(v21, vw21, vout1);
vout2 = __lasx_xvfmadd_s(v22, vw22, vout2);
vout0 = __lasx_xvfadd_s(__lasx_xvfadd_s(vout0, vout1), vout2);
if (relu)
{
__m256i m = __lasx_xvfcmp_clt_s(z, vout0);
vout0 = (__m256)__lasx_xvbitsel_v((__m256i)__lasx_xvfmul_s(vout0, vrc), (__m256i)vout0, m);
}
__lasx_xvst(vout0, outptr + out_j, 0);
}
else
for( ; out_j < outW1; out_j += VECSZ )
{
if (out_j + VECSZ > outW1 && out_j > pad_l)
out_j = outW1 - VECSZ;
int in_j = out_j * stride_w - pad_l;
__m256 v00, v01, v02, v10, v11, v12, v20, v21, v22, unused;
_v256_load_deinterleave(imgptr0 + in_j, v00, v01);
_v256_load_deinterleave(imgptr0 + in_j + 2, v02, unused);
_v256_load_deinterleave(imgptr1 + in_j, v10, v11);
_v256_load_deinterleave(imgptr1 + in_j + 2, v12, unused);
_v256_load_deinterleave(imgptr2 + in_j, v20, v21);
_v256_load_deinterleave(imgptr2 + in_j + 2, v22, unused);
__m256 vout0 = __lasx_xvfmadd_s(v00, vw00, vbias);
__m256 vout1 = __lasx_xvfmul_s(v01, vw01);
__m256 vout2 = __lasx_xvfmul_s(v02, vw02);
vout0 = __lasx_xvfmadd_s(v10, vw10, vout0);
vout1 = __lasx_xvfmadd_s(v11, vw11, vout1);
vout2 = __lasx_xvfmadd_s(v12, vw12, vout2);
vout0 = __lasx_xvfmadd_s(v20, vw20, vout0);
vout1 = __lasx_xvfmadd_s(v21, vw21, vout1);
vout2 = __lasx_xvfmadd_s(v22, vw22, vout2);
vout0 = __lasx_xvfadd_s(__lasx_xvfadd_s(vout0, vout1), vout2);
if (relu)
{
__m256i m = __lasx_xvfcmp_clt_s(z, vout0);
vout0 = (__m256)__lasx_xvbitsel_v((__m256i)__lasx_xvfmul_s(vout0, vrc), (__m256i)vout0, m);
}
__lasx_xvst(vout0, outptr + out_j, 0);
}
}
for (; out_j < outW1; out_j++)
{
int in_j = out_j * stride_w - pad_l;
out = imgptr0[in_j]*w00 + imgptr0[in_j + dilation_w]*w01 + imgptr0[in_j + dilation_w*2]*w02 +
imgptr1[in_j]*w10 + imgptr1[in_j + dilation_w]*w11 + imgptr1[in_j + dilation_w*2]*w12 +
imgptr2[in_j]*w20 + imgptr2[in_j + dilation_w]*w21 + imgptr2[in_j + dilation_w*2]*w22 + bias;
if (relu)
out = out > 0.f ? out : out*relu_coeff;
outptr[out_j] = out;
}
for (; out_j < outW; out_j++ )
{
int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2;
float s0 = 1.f, s1 = 1.f, s2 = 1.f;
if (in_j0 >= width)
{
in_j0 = 0;
s0 = 0.f;
}
if (in_j1 >= width)
{
in_j1 = 0;
s1 = 0.f;
}
if (in_j2 >= width)
{
in_j2 = 0;
s2 = 0.f;
}
out = imgptr0[in_j0]*w00*s0 + imgptr0[in_j1]*w01*s1 + imgptr0[in_j2]*w02*s2 +
imgptr1[in_j0]*w10*s0 + imgptr1[in_j1]*w11*s1 + imgptr1[in_j2]*w12*s2 +
imgptr2[in_j0]*w20*s0 + imgptr2[in_j1]*w21*s1 + imgptr2[in_j2]*w22*s2 + bias;
if (relu)
out = out > 0.f ? out : out*relu_coeff;
outptr[out_j] = out;
}
}
}
// dst = vec * weights^t + bias
void fastGEMM1T( const float* vec, const float* weights,
size_t wstep, const float* bias,
float* dst, int nvecs, int vecsize )
{
int i = 0;
__m256i v256_tmp;
for( ; i <= nvecs - 8; i += 8 )
{
const float* wptr = weights + i*wstep;
__m256 vs0 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp), vs1 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp),
vs2 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp), vs3 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp),
vs4 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp), vs5 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp),
vs6 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp), vs7 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp);
for( int k = 0; k < vecsize; k += 8, wptr += 8 )
{
__m256 v = (__m256)__lasx_xvld(vec + k, 0);
vs0 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr, 0), v, vs0);
vs1 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr + wstep, 0), v, vs1);
vs2 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr + wstep*2, 0), v, vs2);
vs3 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr + wstep*3, 0), v, vs3);
vs4 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr + wstep*4, 0), v, vs4);
vs5 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr + wstep*5, 0), v, vs5);
vs6 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr + wstep*6, 0), v, vs6);
vs7 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr + wstep*7, 0), v, vs7);
}
/*s0*/
__m256 vs00_perm = (__m256)__lasx_xvpermi_d(vs0, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs00_add_2w = __lasx_xvfadd_s(vs0, vs00_perm);
__m256 tmp00_srl = (__m256)__lasx_xvsrli_d(vs00_add_2w, 32);
__m256 vs00_add_4w = __lasx_xvfadd_s(vs00_add_2w, tmp00_srl);
__m256 vs01_perm = (__m256)__lasx_xvpermi_d(vs1, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs01_add_2w = __lasx_xvfadd_s(vs1, vs01_perm);
__m256 tmp01_srl = (__m256)__lasx_xvsrli_d(vs01_add_2w, 32);
__m256 vs01_add_4w = __lasx_xvfadd_s(vs01_add_2w, tmp01_srl);
__m256 vs02_perm = (__m256)__lasx_xvpermi_d(vs2, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs02_add_2w = __lasx_xvfadd_s(vs2, vs02_perm);
__m256 tmp02_srl = (__m256)__lasx_xvsrli_d(vs02_add_2w, 32);
__m256 vs02_add_4w = __lasx_xvfadd_s(vs02_add_2w, tmp02_srl);
__m256 vs03_perm = (__m256)__lasx_xvpermi_d(vs3, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs03_add_2w = __lasx_xvfadd_s(vs3, vs03_perm);
__m256 tmp03_srl = (__m256)__lasx_xvsrli_d(vs03_add_2w, 32);
__m256 vs03_add_4w = __lasx_xvfadd_s(vs03_add_2w, tmp03_srl);
__m256i vs01_vs00 = __lasx_xvpackev_w((__m256i)vs01_add_4w, (__m256i)vs00_add_4w);
__m256i vs03_vs02 = __lasx_xvpackev_w((__m256i)vs03_add_4w, (__m256i)vs02_add_4w);
__m256 s0 = (__m256)__lasx_xvpackev_d(vs03_vs02, vs01_vs00);
/*s1*/
__m256 vs10_perm = (__m256)__lasx_xvpermi_d(vs4, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs10_add_2w = __lasx_xvfadd_s(vs4, vs10_perm);
__m256 tmp10_srl = (__m256)__lasx_xvsrli_d(vs10_add_2w, 32);
__m256 vs10_add_4w = __lasx_xvfadd_s(vs10_add_2w, tmp10_srl);
__m256 vs11_perm = (__m256)__lasx_xvpermi_d(vs5, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs11_add_2w = __lasx_xvfadd_s(vs5, vs11_perm);
__m256 tmp11_srl = (__m256)__lasx_xvsrli_d(vs11_add_2w, 32);
__m256 vs11_add_4w = __lasx_xvfadd_s(vs11_add_2w, tmp11_srl);
__m256 vs12_perm = (__m256)__lasx_xvpermi_d(vs6, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs12_add_2w = __lasx_xvfadd_s(vs6, vs12_perm);
__m256 tmp12_srl = (__m256)__lasx_xvsrli_d(vs12_add_2w, 32);
__m256 vs12_add_4w = __lasx_xvfadd_s(vs12_add_2w, tmp12_srl);
__m256 vs13_perm = (__m256)__lasx_xvpermi_d(vs7, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs13_add_2w = __lasx_xvfadd_s(vs7, vs13_perm);
__m256 tmp13_srl = (__m256)__lasx_xvsrli_d(vs13_add_2w, 32);
__m256 vs13_add_4w = __lasx_xvfadd_s(vs13_add_2w, tmp13_srl);
__m256i vs11_vs10 = __lasx_xvpackev_w((__m256i)vs11_add_4w, (__m256i)vs10_add_4w);
__m256i vs13_vs12 = __lasx_xvpackev_w((__m256i)vs13_add_4w, (__m256i)vs12_add_4w);
__m256 s1 = (__m256)__lasx_xvpackev_d(vs13_vs12, vs11_vs10);
s0 = __lasx_xvfadd_s(s0, (__m256)__lasx_xvpermi_q(s0, s0, 1));
s1 = __lasx_xvfadd_s(s1, (__m256)__lasx_xvpermi_q(s1, s1, 1));
s0 = __lasx_xvfadd_s(s0, (__m256)__lasx_xvld(bias + i, 0));
s1 = __lasx_xvfadd_s(s1, (__m256)__lasx_xvld(bias + i, 4*4));
__lsx_vst(*(__m128*)&s0, dst + i, 0);
__lsx_vst(*(__m128*)&s1, dst + i, 4*4);
}
float temp = 0.f;
for( ; i < nvecs; i++ )
{
const float* wptr = weights + i*wstep;
__m256 vs0 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp);
for( int k = 0; k < vecsize; k += 8, wptr += 8 )
{
__m256 v = (__m256)__lasx_xvld(vec + k, 0);
vs0 = __lasx_xvfmadd_s((__m256)__lasx_xvld(wptr, 0), v, vs0);
}
__m256i vs0_perm = __lasx_xvpermi_d(vs0, (2<<6) + (3<<4) + (0<<2) + 1);
__m256 vs0_add_2w = __lasx_xvfadd_s(vs0, (__m256)vs0_perm);
__m256i tmp_srl = __lasx_xvsrli_d(vs0_add_2w, 32);
__m256 vs0_add_4w = __lasx_xvfadd_s(vs0_add_2w, (__m256)tmp_srl);
temp = ((v8f32)vs0_add_4w)[0] + ((v8f32)vs0_add_4w)[4];
dst[i] = temp + bias[i];
}
}
void fastGEMM( const float* aptr, size_t astep, const float* bptr,
size_t bstep, float* cptr, size_t cstep,
int ma, int na, int nb )
{
int n = 0;
for( ; n <= nb - 16; n += 16 )
{
for( int m = 0; m < ma; m += 4 )
{
const float* aptr0 = aptr + astep*m;
const float* aptr1 = aptr + astep*std::min(m+1, ma-1);
const float* aptr2 = aptr + astep*std::min(m+2, ma-1);
const float* aptr3 = aptr + astep*std::min(m+3, ma-1);
float* cptr0 = cptr + cstep*m;
float* cptr1 = cptr + cstep*std::min(m+1, ma-1);
float* cptr2 = cptr + cstep*std::min(m+2, ma-1);
float* cptr3 = cptr + cstep*std::min(m+3, ma-1);
__m256i v256_tmp;
__m256 d00 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp), d01 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp);
__m256 d10 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp), d11 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp);
__m256 d20 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp), d21 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp);
__m256 d30 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp), d31 = (__m256)__lasx_xvxor_v(v256_tmp, v256_tmp);
for( int k = 0; k < na; k++ )
{
__m256 a0 = _v256_setall_ps(aptr0[k]);
__m256 a1 = _v256_setall_ps(aptr1[k]);
__m256 a2 = _v256_setall_ps(aptr2[k]);
__m256 a3 = _v256_setall_ps(aptr3[k]);
__m256 b0 = (__m256)__lasx_xvld(bptr + k*bstep + n, 0);
__m256 b1 = (__m256)__lasx_xvld(bptr + k*bstep + n + 8, 0);
d00 = __lasx_xvfmadd_s(a0, b0, d00);
d01 = __lasx_xvfmadd_s(a0, b1, d01);
d10 = __lasx_xvfmadd_s(a1, b0, d10);
d11 = __lasx_xvfmadd_s(a1, b1, d11);
d20 = __lasx_xvfmadd_s(a2, b0, d20);
d21 = __lasx_xvfmadd_s(a2, b1, d21);
d30 = __lasx_xvfmadd_s(a3, b0, d30);
d31 = __lasx_xvfmadd_s(a3, b1, d31);
}
__lasx_xvst(d00, cptr0 + n, 0);
__lasx_xvst(d01, cptr0 + n, 8*4);
__lasx_xvst(d10, cptr1 + n, 0);
__lasx_xvst(d11, cptr1 + n, 8*4);
__lasx_xvst(d20, cptr2 + n, 0);
__lasx_xvst(d21, cptr2 + n, 8*4);
__lasx_xvst(d30, cptr3 + n, 0);
__lasx_xvst(d31, cptr3 + n, 8*4);
}
}
for( ; n < nb; n++ )
{
for( int m = 0; m < ma; m++ )
{
const float* aptr0 = aptr + astep*m;
float* cptr0 = cptr + cstep*m;
float d0 = 0.f;
for( int k = 0; k < na; k++ )
d0 += aptr0[k]*bptr[k*bstep + n];
cptr0[n] = d0;
}
}
}
#endif // CV_LASX
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // namespace