diff --git a/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp index f592f79028..6926b9f79b 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv2_kernels.simd.hpp @@ -392,11 +392,189 @@ CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN #endif -static void conv32fC8(const void* inp__, const void* residual__, void* out__, - const ConvState& cs, const void* weights__, - const float* scale__, const float* bias__) +// Compute number of spatial chunks for load balancing across threads. +static int computeSpatChunks(int total_blocks, int planeblocks, int min_per_chunk = 16) { + int nSpatChunks = 1; + int nthreads = cv::getNumThreads(); + int target_tasks = nthreads * 8; + if (total_blocks < target_tasks && planeblocks > min_per_chunk) { + nSpatChunks = (target_tasks + total_blocks - 1) / total_blocks; + int max_chunks = planeblocks / min_per_chunk; + nSpatChunks = std::min(nSpatChunks, std::max(1, max_chunks)); + } + return nSpatChunks; +} + +static void setupActivation(const ConvState& cs, int K, + FastActivation& fastActivation, const float*& activParams, + ActivationFunc& activation, float& maxval, float& defaultAlpha) { + fastActivation = cs.fastActivation; + activParams = cs.activParams.data(); + activation = cs.activation; + maxval = FLT_MAX; + defaultAlpha = 0.f; + if (fastActivation == FAST_ACTIV_CLIP) { + CV_Assert(cs.activParams.size() == 2u); + maxval = activParams[1]; + } else if (fastActivation == FAST_ACTIV_RELU) { + CV_Assert(!activParams); + } else if (fastActivation == FAST_ACTIV_LEAKY_RELU) { + CV_Assert(cs.activParams.size() == 1u); + defaultAlpha = activParams[0]; + } else if (fastActivation == FAST_ACTIV_PRELU) { + CV_Assert(cs.activParams.size() == size_t(K)); + } else { + CV_Assert(fastActivation == FAST_ACTIV_NONE); + defaultAlpha = 1.f; + } +} + +static void fillCoeffBufs(FastActivation fastActivation, const float* activParams, float defaultAlpha, + int k_count, int k_base, + const float* scaleptr, const float* biasptr, + float* scalebuf, float* biasbuf, float* alphabuf) { + int kk = 0; + for (; kk < k_count; kk++) { + scalebuf[kk] = scaleptr ? scaleptr[k_base + kk] : 1.f; + biasbuf[kk] = biasptr ? biasptr[k_base + kk] : 0.f; + alphabuf[kk] = fastActivation == FAST_ACTIV_PRELU ? activParams[k_base + kk] : defaultAlpha; + } + for (; kk < 8; kk++) { + scalebuf[kk] = 0.f; + biasbuf[kk] = 0.f; + alphabuf[kk] = 0.f; + } +} + +#ifdef CONV_ENABLE_SIMD +static bool simdTailAdjust(int spat_block_size, int& p, int p0, int p1, int K0, + float*& outptr, const float*& resptr) { + if (p + spat_block_size > p1) { + if (p == p0) return true; + int _p_new = p1 - spat_block_size; + int _dp = _p_new - p; + outptr += _dp * K0; + if (resptr) resptr += _dp * K0; + p = _p_new; + } + return false; +} + +static void copyResidualBlock(bool aligned_k, int k_base, int k_count, int K0shift, + int K0, int planeblocks, int planesize, int spat_block_size, + float* tmpbuf, const float*& resptr) { + if (resptr) { + if (aligned_k) { + memcpy(tmpbuf, resptr + k_base*planeblocks, spat_block_size*K0*sizeof(float)); + } else { + for (int _kk = 0; _kk < k_count; ++_kk) { + const int _k = k_base + _kk; + int _kofs = (_k >> K0shift) * planesize + (_k & (K0-1)); + for (int _j = 0; _j < spat_block_size; _j++) + tmpbuf[_kk + _j*K0] = resptr[_kofs + _j*K0]; + } + } + resptr += spat_block_size*K0; + } +} + +static void scatterOutputBlock(bool aligned_k, int k_base, int k_count, int K0shift, + int K0, int planeblocks, int planesize, int spat_block_size, + float* outptr, const float* tmpbuf) { + if (!aligned_k) { + for (int _kk = 0; _kk < k_count; ++_kk) { + const int _k = k_base + _kk; + int _kofs = (_k >> K0shift) * planesize + (_k & (K0-1)); + for (int _j = 0; _j < spat_block_size; _j++) + outptr[_kofs + _j*K0] = tmpbuf[_kk + _j*K0]; + } + } +} +#endif // CONV_ENABLE_SIMD + +static void loadScalarResidual(const float* resptr, int k_base, int k_count, int K0shift, + int K0, int planesize, float* resbuf) { + if (resptr) { + for (int _kk = 0; _kk < k_count; ++_kk) { + const int _k = k_base + _kk; + int _kofs = (_k >> K0shift) * planesize + (_k & (K0-1)); + resbuf[_kk] = resptr[_kofs]; + } + } +} + +static void callActivationScalar(ActivationFunc activation, float* outbuf, int K0, const float* activParams) { + if (activation) activation(outbuf, outbuf, K0, activParams); +} + +static void scatterScalarOut(bool aligned_k, int k_base, int k_count, int K0shift, + int K0, int planesize, float* outptr, const float* outbuf) { + if (!aligned_k) { + for (int _kk = 0; _kk < k_count; _kk++) { + const int _k = k_base + _kk; + int _kofs = (_k >> K0shift) * planesize + (_k & (K0-1)); + outptr[_kofs] = outbuf[_kk]; + } + } +} + +// Architecture-specific: initialize scalar accumulators for a single output point. +#if CV_SIMD256 +#define CONV_INIT_SCALAR_SUMS() \ + v_float32x8 zz = v256_setzero_f32(); \ + v_float32x8 s0 = zz +#elif CV_SIMD128 +#define CONV_INIT_SCALAR_SUMS() \ + v_float32x4 zz = v_setzero_f32(); \ + v_float32x4 s0 = zz, s1 = zz +#else +#define CONV_INIT_SCALAR_SUMS() \ + for (int _ks = 0; _ks < K0; _ks++) tmpbuf[_ks] = 0.f +#endif + +#if CV_SIMD256 +#define CONV_FINALIZE_SCALAR_OUT(outbuf) \ + { \ + v_float32x8 _vsc = v256_load(scalebuf); \ + v_float32x8 _vbi = v256_load(biasbuf); \ + v_float32x8 _val = v256_load(alphabuf); \ + v_float32x8 _vmx = v256_setall_f32(maxval); \ + s0 = v_fma(s0, _vsc, _vbi); \ + s0 = v_add(s0, v256_load(resbuf)); \ + s0 = v_select(v_ge(s0, zz), s0, v_mul(s0, _val)); \ + s0 = v_min(s0, _vmx); \ + v_store((outbuf), s0); \ + } +#elif CV_SIMD128 +#define CONV_FINALIZE_SCALAR_OUT(outbuf) \ + { \ + v_float32x4 _vsc_lo = v_load(scalebuf), _vsc_hi = v_load(scalebuf + 4); \ + v_float32x4 _vbi_lo = v_load(biasbuf), _vbi_hi = v_load(biasbuf + 4); \ + v_float32x4 _val_lo = v_load(alphabuf), _val_hi = v_load(alphabuf + 4); \ + v_float32x4 _vmx = v_setall_f32(maxval); \ + s0 = v_fma(s0, _vsc_lo, _vbi_lo); \ + s1 = v_fma(s1, _vsc_hi, _vbi_hi); \ + s0 = v_add(s0, v_load(resbuf)); \ + s1 = v_add(s1, v_load(resbuf + 4)); \ + s0 = v_select(v_ge(s0, zz), s0, v_mul(s0, _val_lo)); \ + s1 = v_select(v_ge(s1, zz), s1, v_mul(s1, _val_hi)); \ + s0 = v_min(s0, _vmx); s1 = v_min(s1, _vmx); \ + v_store((outbuf), s0); v_store((outbuf) + 4, s1); \ + } +#else +#define CONV_FINALIZE_SCALAR_OUT(outbuf) \ + for (int _kk = 0; _kk < K0; _kk++) { \ + float _v = tmpbuf[_kk]*scalebuf[_kk] + biasbuf[_kk] + resbuf[_kk]; \ + _v = std::min(_v*(_v >= 0 ? 1.f : alphabuf[_kk]), maxval); \ + (outbuf)[_kk] = _v; \ + } +#endif + +// Specialized 1x1 convolution kernel with stride=1. +static void conv32fC8_1x1(const void* inp__, const void* residual__, void* out__, + const ConvState& cs, const void* weights__, + const float* scale__, const float* bias__) { - using FT = float; const MatShape& inpshape = cs.inpshape; const MatShape& outshape = cs.outshape; @@ -416,13 +594,13 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, int total_blocks = N * cs.ngroups * Kblk_; if ((K_/cs.ngroups) % inpshape.back() != 0) { - // if there could be 'padding' channels in the output, - // clear the output before the parallel loop - // to make sure that all the padding channels are cleared. - memset(out__, 0, outtotal*sizeof(FT)); + memset(out__, 0, outtotal*sizeof(float)); } - parallel_for_(Range(0, total_blocks), [&](const Range& range) { + int nSpatChunks_ = computeSpatChunks(total_blocks, planeblocks_); + int total_tasks = total_blocks * nSpatChunks_; + + parallel_for_(Range(0, total_tasks), [&](const Range& range) { constexpr int SPAT_BLOCK_SIZE = 10; constexpr int C0shift = 3, K0shift = C0shift; constexpr int C0 = 1 << C0shift, K0 = C0; @@ -434,6 +612,465 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, const int ngroups = cs.ngroups, Kblk = Kblk_, C1Max = C1Max_; const int Cg = C / ngroups; const int Kg = K / ngroups; + const int nSpatChunks = nSpatChunks_; + + int planeblocks = planeblocks_; + int planesize = planeblocks*K0; + int ndims = ndims_; + int Di = ndims >= 6 ? inpshape[ndims-4] : 1; + int Hi = ndims >= 5 ? inpshape[ndims-3] : 1; + int Wi = inpshape[ndims-2]; + int iplanesize = Di*Hi*Wi*C0; + + const float* scaleptr = (const float*)scale__; + const float* biasptr = (const float*)bias__; + + FastActivation fastActivation; + const float* activParams; + ActivationFunc activation; + float maxval, defaultAlpha; + float scalebuf[K0], biasbuf[K0], alphabuf[K0]; + setupActivation(cs, K, fastActivation, activParams, activation, maxval, defaultAlpha); + + for (int t = range.start; t < range.end; t++) { + const int block_id = t / nSpatChunks; + const int chunk_id = t % nSpatChunks; + const int p0 = chunk_id * planeblocks / nSpatChunks; + const int p1 = (chunk_id + 1) * planeblocks / nSpatChunks; + const int n = block_id / (ngroups * Kblk); + const int rem = block_id - n * (ngroups * Kblk); + const int g = rem / Kblk; + const int kblk = rem - g * Kblk; + + const int k_base = g * Kg + kblk * K0; + if (k_base >= K) continue; + + const int k_count = std::min(std::min(K0, Kg - kblk*K0), K - k_base); + bool aligned_k = (k_base & (K0-1)) == 0 && k_count == K0; + + const int c_start = g * Cg; + const int c00 = c_start & (C0-1); + const int c1_start = c_start >> C0shift; + const int cblocks = (c00 + Cg + C0 - 1) >> C0shift; + const float* inpbaseptr = (float*)inp__ + (n * C1 + c1_start) * iplanesize; + const float* wbaseptr = (float*)weights__ + (g*Kblk + kblk)*(1*C1Max*C0*K0); + + fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf); + + float* outptr = (float*)out__ + n*(K1*planesize) + p0*K0; + const float* resptr = residual__ ? (float*)residual__ + n*(K1*planesize) + p0*K0 : nullptr; + float tmpbuf[SPAT_BLOCK_SIZE*K0] = {}; + int p = p0; + + #ifdef CONV_ENABLE_SIMD + for (; p < p1; p += SPAT_BLOCK_SIZE, + outptr += SPAT_BLOCK_SIZE*K0) + { + if (simdTailAdjust(SPAT_BLOCK_SIZE, p, p0, p1, K0, outptr, resptr)) break; + copyResidualBlock(aligned_k, k_base, k_count, K0shift, K0, planeblocks, planesize, SPAT_BLOCK_SIZE, tmpbuf, resptr); + CONV_INIT_SUMS(); + + const float* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + inptr[j] = inpbaseptr + (size_t)(p + j) * C0; + inpstep[j] = iplanesize; + } + const float* wptr = wbaseptr; + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0*K0) { + CONV_UPDATE_LOOP_BODY(); + } + + float* outbuf = aligned_k ? outptr + k_base*planeblocks : tmpbuf; + CONV_FINALIZE_OUT_ALL(); + if (activation) { activation(outbuf, outbuf, SPAT_BLOCK_SIZE*K0, activParams); } + scatterOutputBlock(aligned_k, k_base, k_count, K0shift, K0, planeblocks, planesize, SPAT_BLOCK_SIZE, outptr, tmpbuf); + } + #endif + + float resbuf[K0] = {}; + + for (; p < p1; p++, outptr += K0, resptr += (resptr ? K0 : 0)) + { + CONV_INIT_SCALAR_SUMS(); + loadScalarResidual(resptr, k_base, k_count, K0shift, K0, planesize, resbuf); + + const float* inptr = inpbaseptr + (size_t)p * C0; + const float* wptr = wbaseptr; + + for (int c1 = 0; c1 < cblocks; ++c1, inptr += iplanesize, wptr += K0*C0) { + #if CV_SIMD256 + v_float32x8 w, x; + #undef CONV_UPDATE_BLOCK1 + #define CONV_UPDATE_BLOCK1(ofs) \ + w = v256_load(wptr + ofs*K0); \ + x = v256_setall_f32(inptr[ofs]); \ + s0 = v_fma(x, w, s0) + CONV_UPDATE_BLOCK1(0); CONV_UPDATE_BLOCK1(1); + CONV_UPDATE_BLOCK1(2); CONV_UPDATE_BLOCK1(3); + CONV_UPDATE_BLOCK1(4); CONV_UPDATE_BLOCK1(5); + CONV_UPDATE_BLOCK1(6); CONV_UPDATE_BLOCK1(7); + #elif CV_SIMD128 + v_float32x4 w0, w1, x; + #undef CONV_UPDATE_BLOCK1 + #define CONV_UPDATE_BLOCK1(ofs) \ + w0 = v_load(wptr + ofs*K0); w1 = v_load(wptr + ofs*K0 + 4); \ + x = v_setall_f32(inptr[ofs]); \ + s0 = v_fma(x, w0, s0); s1 = v_fma(x, w1, s1) + CONV_UPDATE_BLOCK1(0); CONV_UPDATE_BLOCK1(1); + CONV_UPDATE_BLOCK1(2); CONV_UPDATE_BLOCK1(3); + CONV_UPDATE_BLOCK1(4); CONV_UPDATE_BLOCK1(5); + CONV_UPDATE_BLOCK1(6); CONV_UPDATE_BLOCK1(7); + #else + for (int c0 = 0; c0 < C0; ++c0) { + const float xval = inptr[c0]; + for (int kk = 0; kk < K0; ++kk) + tmpbuf[kk] += xval * wptr[c0*K0 + kk]; + } + #endif + } + + float* outbuf = aligned_k ? outptr + k_base*planeblocks : tmpbuf; + CONV_FINALIZE_SCALAR_OUT(outbuf); + callActivationScalar(activation, outbuf, K0, activParams); + scatterScalarOut(aligned_k, k_base, k_count, K0shift, K0, planesize, outptr, outbuf); + } + } + }); +} + +// Specialized 2D 3x3 convolution kernel with stride=1, dilation=1. +static void conv32fC8_3x3s1(const void* inp__, const void* residual__, void* out__, + const ConvState& cs, const void* weights__, + const float* scale__, const float* bias__) +{ + const MatShape& inpshape = cs.inpshape; + const MatShape& outshape = cs.outshape; + + CV_Assert_N(inpshape.layout == DATA_LAYOUT_BLOCK, outshape.layout == DATA_LAYOUT_BLOCK); + + int K_ = outshape.channels(); + int ndims_ = outshape.dims; + int N = outshape[0]; + int H_ = ndims_ >= 5 ? outshape[ndims_ - 3] : 1; + int W_ = outshape[ndims_-2]; + int planeblocks_ = H_*W_; + size_t outtotal = outshape.total(); + + int Kblk_ = cs.wshape[1]; + int C1Max_ = cs.wshape[3]; + int total_blocks = N * cs.ngroups * Kblk_; + + if ((K_/cs.ngroups) % inpshape.back() != 0) { + memset(out__, 0, outtotal*sizeof(float)); + } + + int nSpatChunks_ = computeSpatChunks(total_blocks, planeblocks_); + int total_tasks = total_blocks * nSpatChunks_; + + parallel_for_(Range(0, total_tasks), [&](const Range& range) { + constexpr int SPAT_BLOCK_SIZE = 10; + constexpr int C0shift = 3, K0shift = C0shift; + constexpr int C0 = 1 << C0shift, K0 = C0; + + CV_Assert_N(inpshape.back() == C0, outshape.back() == K0); + + const int C = inpshape.channels(), K = outshape.channels(); + const int C1 = (C + C0 - 1)/C0, K1 = (K + K0 - 1)/K0; + const int ngroups = cs.ngroups, Kblk = Kblk_, C1Max = C1Max_; + const int Cg = C / ngroups; + const int Kg = K / ngroups; + const int nSpatChunks = nSpatChunks_; + int W = W_; + int Hi = ndims_ >= 5 ? inpshape[ndims_-3] : 1; + int Wi = inpshape[ndims_-2]; + const int padY = cs.pads[1], padX = cs.pads[2]; + const float* scaleptr = (const float*)scale__; + const float* biasptr = (const float*)bias__; + int planeblocks = planeblocks_; + int planesize = planeblocks*K0; + int iplanesize = Hi*Wi*C0; + + #ifdef CONV_ENABLE_SIMD + constexpr int MAX_CONV_DIMS = ConvState::MAX_CONV_DIMS; + int innerY0 = cs.inner[1], innerY1 = cs.inner[MAX_CONV_DIMS+1]; + int innerX0 = cs.inner[2], innerX1 = cs.inner[MAX_CONV_DIMS+2]; + #endif + + FastActivation fastActivation; + const float* activParams; + ActivationFunc activation; + float maxval, defaultAlpha; + float scalebuf[K0], biasbuf[K0], alphabuf[K0]; + setupActivation(cs, K, fastActivation, activParams, activation, maxval, defaultAlpha); + + for (int t = range.start; t < range.end; t++) { + const int block_id = t / nSpatChunks; + const int chunk_id = t % nSpatChunks; + const int p0 = chunk_id * planeblocks / nSpatChunks; + const int p1 = (chunk_id + 1) * planeblocks / nSpatChunks; + const int n = block_id / (ngroups * Kblk); + const int rem = block_id - n * (ngroups * Kblk); + const int g = rem / Kblk; + const int kblk = rem - g * Kblk; + + const int k_base = g * Kg + kblk * K0; + if (k_base >= K) continue; + + const int k_count = std::min(std::min(K0, Kg - kblk*K0), K - k_base); + bool aligned_k = (k_base & (K0-1)) == 0 && k_count == K0; + + const int c_start = g * Cg; + const int c00 = c_start & (C0-1); + const int c1_start = c_start >> C0shift; + const int cblocks = (c00 + Cg + C0 - 1) >> C0shift; + const float* inpbaseptr = (float*)inp__ + (n * C1 + c1_start) * iplanesize; + const float* wbaseptr = (float*)weights__ + (g*Kblk + kblk)*(9*C1Max*C0*K0); + + fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf); + + float* outptr = (float*)out__ + n*(K1*planesize) + p0*K0; + const float* resptr = residual__ ? (float*)residual__ + n*(K1*planesize) + p0*K0 : nullptr; + float tmpbuf[SPAT_BLOCK_SIZE*K0] = {}; + int p = p0; + + #ifdef CONV_ENABLE_SIMD + int inp_ofs[9]; + for (int ky = 0; ky < 3; ky++) + for (int kx = 0; kx < 3; kx++) + inp_ofs[ky*3 + kx] = (ky * Wi + kx) * C0; + float zbuf[C0] = {}; + for (; p < p1; p += SPAT_BLOCK_SIZE, + outptr += SPAT_BLOCK_SIZE*K0) + { + if (simdTailAdjust(SPAT_BLOCK_SIZE, p, p0, p1, K0, outptr, resptr)) break; + copyResidualBlock(aligned_k, k_base, k_count, K0shift, K0, planeblocks, planesize, SPAT_BLOCK_SIZE, tmpbuf, resptr); + + CONV_INIT_SUMS(); + + bool all_inner = false; + int yi_base, xi_base; + + if ((p % W) + SPAT_BLOCK_SIZE <= W) { + int yj = p / W; + int xj = p - yj * W; + yi_base = yj - padY; + xi_base = xj - padX; + bool y_inner = (yj >= innerY0 && yj < innerY1); + all_inner = y_inner && (xj >= innerX0) && + (xj + SPAT_BLOCK_SIZE - 1 < innerX1); + } else { + int yj = p / W; + int xj = p - yj * W; + yi_base = yj - padY; + xi_base = xj - padX; + } + + if (all_inner) { + const float* inp_yx_base0 = inpbaseptr + (yi_base * Wi + xi_base) * C0; + + const float* inp_pos[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) + inp_pos[j] = inp_yx_base0 + j * C0; + + for (int kpos = 0; kpos < 9; kpos++) { + const float* kwptr = wbaseptr + kpos * C1Max * C0 * K0; + const int kofs = inp_ofs[kpos]; + for (int c1 = 0; c1 < cblocks; c1++) { + const int c1_ofs = c1 * iplanesize; + const float* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + inptr[j] = inp_pos[j] + kofs + c1_ofs; + inpstep[j] = 0; + } + const float* wptr = kwptr + c1 * C0 * K0; + CONV_UPDATE_LOOP_BODY(); + } + } + } else { + int yi_arr[SPAT_BLOCK_SIZE], xi_arr[SPAT_BLOCK_SIZE]; + bool inner_arr[SPAT_BLOCK_SIZE]; + bool same_row = ((p % W) + SPAT_BLOCK_SIZE <= W); + + if (same_row) { + int yj = p / W; + int xj = p - yj * W; + bool y_inner = (yj >= innerY0 && yj < innerY1); + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + yi_arr[j] = yj - padY; + xi_arr[j] = xj + j - padX; + inner_arr[j] = y_inner && ((xj + j) >= innerX0 && (xj + j) < innerX1); + } + } else { + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int pj = p + j; + int yj = pj / W; + int xj = pj - yj * W; + yi_arr[j] = yj - padY; + xi_arr[j] = xj - padX; + inner_arr[j] = (yj >= innerY0 && yj < innerY1) && + (xj >= innerX0 && xj < innerX1); + } + } + int inp_spatial_ofs[9][SPAT_BLOCK_SIZE]; + for (int kpos = 0; kpos < 9; kpos++) { + int ky = kpos / 3, kx = kpos % 3; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int yij = yi_arr[j] + ky; + int xij = xi_arr[j] + kx; + if (inner_arr[j] || + (((unsigned)yij < (unsigned)Hi) & + ((unsigned)xij < (unsigned)Wi))) { + inp_spatial_ofs[kpos][j] = (yij * Wi + xij) * C0; + } else { + inp_spatial_ofs[kpos][j] = -1; + } + } + } + + for (int kpos = 0; kpos < 9; kpos++) { + const float* kwptr = wbaseptr + kpos * C1Max * C0 * K0; + for (int c1 = 0; c1 < cblocks; c1++) { + const float* inpbase_c1 = inpbaseptr + c1 * iplanesize; + const float* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int ofs = inp_spatial_ofs[kpos][j]; + inptr[j] = (ofs >= 0) ? inpbase_c1 + ofs : zbuf; + inpstep[j] = 0; + } + const float* wptr = kwptr + c1 * C0 * K0; + CONV_UPDATE_LOOP_BODY(); + } + } + } + + float* outbuf = aligned_k ? outptr + k_base*planeblocks : tmpbuf; + CONV_FINALIZE_OUT_ALL(); + if (activation) { activation(outbuf, outbuf, SPAT_BLOCK_SIZE*K0, activParams); } + scatterOutputBlock(aligned_k, k_base, k_count, K0shift, K0, planeblocks, planesize, SPAT_BLOCK_SIZE, outptr, tmpbuf); + } + #endif // CONV_ENABLE_SIMD + + float resbuf[K0] = {}; + + for (; p < p1; p++, outptr += K0, resptr += (resptr ? K0 : 0)) + { + int yj = p / W; + int xj = p - yj * W; + int yi_s = yj - padY; + int xi_s = xj - padX; + + CONV_INIT_SCALAR_SUMS(); + loadScalarResidual(resptr, k_base, k_count, K0shift, K0, planesize, resbuf); + + for (int ky = 0; ky < 3; ky++) { + int yi = yi_s + ky; + if ((unsigned)yi >= (unsigned)Hi) continue; + for (int kx = 0; kx < 3; kx++) { + int xi = xi_s + kx; + if ((unsigned)xi >= (unsigned)Wi) continue; + int kpos = ky*3 + kx; + const float* kwptr = wbaseptr + kpos * C1Max * C0 * K0; + const float* inptr_s = inpbaseptr; + for (int c1 = 0; c1 < cblocks; ++c1, inptr_s += iplanesize) { + const float* inptr = inptr_s + (yi*Wi + xi)*C0; + const float* wptr = kwptr + c1 * C0 * K0; + #if CV_SIMD256 + v_float32x8 w, xv; + #undef CONV_UPDATE_BLOCK1 + #define CONV_UPDATE_BLOCK1(ofs) \ + w = v256_load(wptr + ofs*K0); \ + xv = v256_setall_f32(inptr[ofs]); \ + s0 = v_fma(xv, w, s0) + CONV_UPDATE_BLOCK1(0); CONV_UPDATE_BLOCK1(1); + CONV_UPDATE_BLOCK1(2); CONV_UPDATE_BLOCK1(3); + CONV_UPDATE_BLOCK1(4); CONV_UPDATE_BLOCK1(5); + CONV_UPDATE_BLOCK1(6); CONV_UPDATE_BLOCK1(7); + #elif CV_SIMD128 + v_float32x4 w0, w1, xv; + #undef CONV_UPDATE_BLOCK1 + #define CONV_UPDATE_BLOCK1(ofs) \ + w0 = v_load(wptr + ofs*K0); w1 = v_load(wptr + ofs*K0 + 4); \ + xv = v_setall_f32(inptr[ofs]); \ + s0 = v_fma(xv, w0, s0); s1 = v_fma(xv, w1, s1) + CONV_UPDATE_BLOCK1(0); CONV_UPDATE_BLOCK1(1); + CONV_UPDATE_BLOCK1(2); CONV_UPDATE_BLOCK1(3); + CONV_UPDATE_BLOCK1(4); CONV_UPDATE_BLOCK1(5); + CONV_UPDATE_BLOCK1(6); CONV_UPDATE_BLOCK1(7); + #else + for (int c0 = 0; c0 < C0; ++c0) { + const float xval = inptr[c0]; + for (int kk = 0; kk < K0; ++kk) + tmpbuf[kk] += xval * wptr[c0*K0 + kk]; + } + #endif + } + } + } + + float* outbuf = aligned_k ? outptr + k_base*planeblocks : tmpbuf; + CONV_FINALIZE_SCALAR_OUT(outbuf); + callActivationScalar(activation, outbuf, K0, activParams); + scatterScalarOut(aligned_k, k_base, k_count, K0shift, K0, planesize, outptr, outbuf); + } + } + }); +} + +static void conv32fC8(const void* inp__, const void* residual__, void* out__, + const ConvState& cs, const void* weights__, + const float* scale__, const float* bias__) +{ + int ksize = cs.wshape[2]; + if (ksize == 1 && cs.strides[0]*cs.strides[1]*cs.strides[2] == 1) { + return conv32fC8_1x1(inp__, residual__, out__, cs, weights__, scale__, bias__); + } + if (ksize == 9 && cs.strides[1] == 1 && cs.strides[2] == 1 && + cs.dilations[1] == 1 && cs.dilations[2] == 1 && + cs.outshape.dims <= 5) { + return conv32fC8_3x3s1(inp__, residual__, out__, cs, weights__, scale__, bias__); + } + + const MatShape& inpshape = cs.inpshape; + const MatShape& outshape = cs.outshape; + + CV_Assert_N(inpshape.layout == DATA_LAYOUT_BLOCK, outshape.layout == DATA_LAYOUT_BLOCK); + + int K_ = outshape.channels(); + int ndims_ = outshape.dims; + int N = outshape[0]; + int D_ = ndims_ >= 6 ? outshape[ndims_ - 4] : 1; + int H_ = ndims_ >= 5 ? outshape[ndims_ - 3] : 1; + int W_ = outshape[ndims_-2]; + int planeblocks_ = D_*H_*W_; + size_t outtotal = outshape.total(); + + int Kblk_ = cs.wshape[1]; + int C1Max_ = cs.wshape[3]; + int total_blocks = N * cs.ngroups * Kblk_; + + if ((K_/cs.ngroups) % inpshape.back() != 0) { + memset(out__, 0, outtotal*sizeof(float)); + } + + int nSpatChunksGen_ = computeSpatChunks(total_blocks, planeblocks_); + int total_tasks_gen = total_blocks * nSpatChunksGen_; + + parallel_for_(Range(0, total_tasks_gen), [&](const Range& range) { + constexpr int SPAT_BLOCK_SIZE = 10; + constexpr int C0shift = 3, K0shift = C0shift; + constexpr int C0 = 1 << C0shift, K0 = C0; + + CV_Assert_N(inpshape.back() == C0, outshape.back() == K0); + + const int C = inpshape.channels(), K = outshape.channels(); + const int C1 = (C + C0 - 1)/C0, K1 = (K + K0 - 1)/K0; + const int ngroups = cs.ngroups, Kblk = Kblk_, C1Max = C1Max_; + const int Cg = C / ngroups; + const int Kg = K / ngroups; + const int nSpatChunks = nSpatChunksGen_; int ksize = cs.wshape[2]; int ndims = ndims_; int D = D_, H = H_, W = W_; @@ -457,29 +1094,16 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, float zbuf[C0] = {}; #endif - FastActivation fastActivation = cs.fastActivation; - const float* activParams = cs.activParams.data(); - ActivationFunc activation = cs.activation; - float maxval = FLT_MAX, defaultAlpha = 0.f; + FastActivation fastActivation; + const float* activParams; + ActivationFunc activation; + float maxval, defaultAlpha; float scalebuf[K0], biasbuf[K0], alphabuf[K0]; - if (fastActivation == FAST_ACTIV_CLIP) { - CV_Assert(cs.activParams.size() == 2u); - maxval = activParams[1]; - } else if (fastActivation == FAST_ACTIV_RELU) { - CV_Assert(!activParams); - } else if (fastActivation == FAST_ACTIV_LEAKY_RELU) { - CV_Assert(cs.activParams.size() == 1u); - defaultAlpha = activParams[0]; - } else if (fastActivation == FAST_ACTIV_PRELU) { - CV_Assert(cs.activParams.size() == size_t(K)); - } else { - CV_Assert(fastActivation == FAST_ACTIV_NONE); - defaultAlpha = 1.f; - } + setupActivation(cs, K, fastActivation, activParams, activation, maxval, defaultAlpha); // 1x1x1 convolution with (1,1,1) strides: - // flatten input/output tensors in this case to accelerate address computations - if (ksize == 1 && Sz*Sy*Sx == 1) { + bool is_1x1s1 = (ksize == 1 && Sz*Sy*Sx == 1); + if (is_1x1s1) { W *= D*H; Wi *= Di*Hi; D = Di = H = Hi = 1; @@ -490,9 +1114,12 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, } for (int t = range.start; t < range.end; t++) { - const int p0 = 0, p1 = planeblocks; - const int n = t / (ngroups * Kblk); - const int rem = t - n * (ngroups * Kblk); + const int block_id = t / nSpatChunks; + const int chunk_id = t % nSpatChunks; + const int p0 = chunk_id * planeblocks / nSpatChunks; + const int p1 = (chunk_id + 1) * planeblocks / nSpatChunks; + const int n = block_id / (ngroups * Kblk); + const int rem = block_id - n * (ngroups * Kblk); const int g = rem / Kblk; const int kblk = rem - g * Kblk; @@ -509,19 +1136,7 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, const float* inpbaseptr = (float*)inp__ + (n * C1 + c1_start) * iplanesize; const float* wbaseptr = (float*)weights__ + (g*Kblk + kblk)*(ksize*C1Max*C0*K0); - { - int kk = 0; - for (; kk < k_count; kk++) { - scalebuf[kk] = scaleptr ? scaleptr[k_base + kk] : 1.f; - biasbuf[kk] = biasptr ? biasptr[k_base + kk] : 0.f; - alphabuf[kk] = fastActivation == FAST_ACTIV_PRELU ? activParams[k_base + kk] : defaultAlpha; - } - for (; kk < K0; kk++) { - scalebuf[kk] = 0.f; - biasbuf[kk] = 0.f; - alphabuf[kk] = 0.f; - } - } + fillCoeffBufs(fastActivation, activParams, defaultAlpha, k_count, k_base, scaleptr, biasptr, scalebuf, biasbuf, alphabuf); float* outptr = (float*)out__ + n*(K1*planesize) + p0*K0; const float* resptr = residual__ ? (float*)residual__ + n*(K1*planesize) + p0*K0 : nullptr; @@ -529,12 +1144,50 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, int p = p0; #ifdef CONV_ENABLE_SIMD + if (is_1x1s1) { for (; p < p1; p += SPAT_BLOCK_SIZE, outptr += SPAT_BLOCK_SIZE*K0) { - Vec3i pt[SPAT_BLOCK_SIZE]; - bool inner[SPAT_BLOCK_SIZE]; + if (simdTailAdjust(SPAT_BLOCK_SIZE, p, p0, p1, K0, outptr, resptr)) break; + copyResidualBlock(aligned_k, k_base, k_count, K0shift, K0, planeblocks, planesize, SPAT_BLOCK_SIZE, tmpbuf, resptr); + CONV_INIT_SUMS(); + const float* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + inptr[j] = inpbaseptr + (size_t)(p + j) * C0; + inpstep[j] = iplanesize; + } + const float* wptr = wbaseptr; + for (int c1 = 0; c1 < cblocks; c1++, wptr += C0*K0) { + CONV_UPDATE_LOOP_BODY(); + } + + float* outbuf = aligned_k ? outptr + k_base*planeblocks : tmpbuf; + CONV_FINALIZE_OUT_ALL(); + if (activation) { activation(outbuf, outbuf, SPAT_BLOCK_SIZE*K0, activParams); } + scatterOutputBlock(aligned_k, k_base, k_count, K0shift, K0, planeblocks, planesize, SPAT_BLOCK_SIZE, outptr, tmpbuf); + } + } else { + cv::AutoBuffer kofs_tab(ksize); + for (int i = 0; i < ksize; i++) { + kofs_tab[i] = ((ofsZYX[i*3] * Hi + ofsZYX[i*3+1]) * Wi + ofsZYX[i*3+2]) * C0; + } + int x_step = Sx * C0; + + // Precompute initial (z, y, x) from p to avoid repeated division + int cur_z, cur_y, cur_x; + if (D == 1) { + cur_z = 0; cur_y = p / W; cur_x = p - cur_y * W; + } else { + cur_z = p / (H * W); + int yxj = p - cur_z * (H * W); + cur_y = yxj / W; cur_x = yxj - cur_y * W; + } + + for (; p < p1; p += SPAT_BLOCK_SIZE, + outptr += SPAT_BLOCK_SIZE*K0) + { if (p + SPAT_BLOCK_SIZE > p1) { if (p == p0) break; @@ -543,90 +1196,120 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, outptr += dp*K0; resptr += (resptr ? dp*K0 : 0); p = p_new; - } - - if (resptr) { - if (aligned_k) { - memcpy(tmpbuf, resptr + k_base*planeblocks, SPAT_BLOCK_SIZE*K0*sizeof(FT)); + // Recompute coordinates for the adjusted position + if (D == 1) { + cur_z = 0; cur_y = p / W; cur_x = p - cur_y * W; } else { - for (int kk = 0; kk < k_count; ++kk) { - const int k = k_base + kk; - int kofs = (k >> K0shift) * planesize + (k & (K0-1)); - for (int j = 0; j < SPAT_BLOCK_SIZE; j++) - tmpbuf[kk + j*K0] = resptr[kofs + j*K0]; - } - } - resptr += SPAT_BLOCK_SIZE*K0; - } - - if ((p % W) + SPAT_BLOCK_SIZE <= W) { - int zj = p / (H*W); - int yxj = p - zj*(H*W); - int yj = yxj / W; - int x = yxj - yj*W; - const bool zy_inner = (zj >= innerZ0 && zj < innerZ1) && (yj >= innerY0 && yj < innerY1); - for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { - int xj = x + j; - pt[j] = Vec3i(zj*Sz - padZ, yj*Sy - padY, xj*Sx - padX); - inner[j] = zy_inner && (xj >= innerX0 && xj < innerX1); - } - } else { - for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { - int pj = p + j; - int zj = pj / (H*W); - int yxj = pj - zj*(H*W); - int yj = yxj / W; - int xj = yxj - yj*W; - pt[j] = Vec3i(zj*Sz - padZ, yj*Sy - padY, xj*Sx - padX); - inner[j] = (zj >= innerZ0 && zj < innerZ1) && - (yj >= innerY0 && yj < innerY1) && - (xj >= innerX0 && xj < innerX1); + cur_z = p / (H * W); + int yxj = p - cur_z * (H * W); + cur_y = yxj / W; cur_x = yxj - cur_y * W; } } + copyResidualBlock(aligned_k, k_base, k_count, K0shift, K0, planeblocks, planesize, SPAT_BLOCK_SIZE, tmpbuf, resptr); CONV_INIT_SUMS(); - for (int i = 0; i < ksize; i++) { - const float* inptr[SPAT_BLOCK_SIZE]; - int inpstep[SPAT_BLOCK_SIZE]; - for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { - Vec3i ptj = pt[j]; - int zij = ptj[0] + ofsZYX[i*3 + 0]; - int yij = ptj[1] + ofsZYX[i*3 + 1]; - int xij = ptj[2] + ofsZYX[i*3 + 2]; - if (inner[j] || ((((unsigned)zij < (unsigned)Di)& - ((unsigned)yij < (unsigned)Hi)& - ((unsigned)xij < (unsigned)Wi)) != 0)) { - inptr[j] = inpbaseptr + (((zij * Hi) + yij) * Wi + xij) * C0; - inpstep[j] = iplanesize; - } else { - inptr[j] = zbuf; - inpstep[j] = 0; + bool same_row = (cur_x + SPAT_BLOCK_SIZE <= W); + bool all_inner = same_row && + (cur_z >= innerZ0 && cur_z < innerZ1) && + (cur_y >= innerY0 && cur_y < innerY1) && + (cur_x >= innerX0) && (cur_x + SPAT_BLOCK_SIZE - 1 < innerX1); + + if (all_inner) { + int zi_base = cur_z*Sz - padZ; + int yi_base = cur_y*Sy - padY; + int xi_base = cur_x*Sx - padX; + int base_ofs = (zi_base * Hi + yi_base) * Wi * C0 + xi_base * C0; + + // ksize-outer, c1-inner: matches weight layout [Kblk][ksize][C1Max][C0*K0] + for (int i = 0; i < ksize; i++) { + const float* kwptr = wbaseptr + i * C1Max * C0 * K0; + for (int c1 = 0; c1 < cblocks; c1++) { + const float* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + const float* inp_ki = inpbaseptr + c1 * iplanesize + base_ofs + kofs_tab[i]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + inptr[j] = inp_ki + j * x_step; + inpstep[j] = 0; + } + const float* wptr = kwptr + c1 * C0 * K0; + CONV_UPDATE_LOOP_BODY(); } } - const float* wptr = wbaseptr + i*C1Max*K0*C0; + } else { + Vec3i pt[SPAT_BLOCK_SIZE]; + bool inner[SPAT_BLOCK_SIZE]; - for (int c1 = 0; c1 < cblocks; c1++, wptr += C0*K0) { - CONV_UPDATE_LOOP_BODY(); + if (same_row) { + const bool zy_inner = (cur_z >= innerZ0 && cur_z < innerZ1) && (cur_y >= innerY0 && cur_y < innerY1); + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + pt[j] = Vec3i(cur_z*Sz - padZ, cur_y*Sy - padY, (cur_x+j)*Sx - padX); + inner[j] = zy_inner && ((cur_x+j) >= innerX0 && (cur_x+j) < innerX1); + } + } else { + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + int pj = p + j; + int zj, yj, xj; + if (D == 1) { + zj = 0; yj = pj / W; xj = pj - yj * W; + } else { + zj = pj / (H*W); + int yxj = pj - zj*(H*W); + yj = yxj / W; xj = yxj - yj*W; + } + pt[j] = Vec3i(zj*Sz - padZ, yj*Sy - padY, xj*Sx - padX); + inner[j] = (zj >= innerZ0 && zj < innerZ1) && + (yj >= innerY0 && yj < innerY1) && + (xj >= innerX0 && xj < innerX1); + } + } + + // ksize-outer, c1-inner: matches weight layout [Kblk][ksize][C1Max][C0*K0] + for (int i = 0; i < ksize; i++) { + const float* kwptr = wbaseptr + i * C1Max * C0 * K0; + for (int c1 = 0; c1 < cblocks; c1++) { + const float* inpbase_c1 = inpbaseptr + c1 * iplanesize; + const float* inptr[SPAT_BLOCK_SIZE]; + int inpstep[SPAT_BLOCK_SIZE]; + for (int j = 0; j < SPAT_BLOCK_SIZE; j++) { + Vec3i ptj = pt[j]; + int zij = ptj[0] + ofsZYX[i*3 + 0]; + int yij = ptj[1] + ofsZYX[i*3 + 1]; + int xij = ptj[2] + ofsZYX[i*3 + 2]; + if (inner[j] || ((((unsigned)zij < (unsigned)Di)& + ((unsigned)yij < (unsigned)Hi)& + ((unsigned)xij < (unsigned)Wi)) != 0)) { + inptr[j] = inpbase_c1 + (((zij * Hi) + yij) * Wi + xij) * C0; + inpstep[j] = 0; + } else { + inptr[j] = zbuf; + inpstep[j] = 0; + } + } + const float* wptr = kwptr + c1 * C0 * K0; + CONV_UPDATE_LOOP_BODY(); + } } } float* outbuf = aligned_k ? outptr + k_base*planeblocks : tmpbuf; CONV_FINALIZE_OUT_ALL(); + if (activation) { activation(outbuf, outbuf, SPAT_BLOCK_SIZE*K0, activParams); } + scatterOutputBlock(aligned_k, k_base, k_count, K0shift, K0, planeblocks, planesize, SPAT_BLOCK_SIZE, outptr, tmpbuf); - if (activation) { - activation(outbuf, outbuf, SPAT_BLOCK_SIZE*K0, activParams); - } - - if (!aligned_k) { - for (int kk = 0; kk < k_count; ++kk) { - const int k = k_base + kk; - int kofs = (k >> K0shift) * planesize + (k & (K0-1)); - for (int j = 0; j < SPAT_BLOCK_SIZE; j++) - outptr[kofs + j*K0] = tmpbuf[kk + j*K0]; + cur_x += SPAT_BLOCK_SIZE; + if (cur_x >= W) { + int pn = p + SPAT_BLOCK_SIZE; + if (D == 1) { + cur_z = 0; cur_y = pn / W; cur_x = pn - cur_y * W; + } else { + cur_z = pn / (H*W); + int yxj = pn - cur_z*(H*W); + cur_y = yxj / W; cur_x = yxj - cur_y * W; } } } + } #endif float resbuf[K0] = {}; @@ -641,26 +1324,10 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, int yi_base = yj*Sy - padY; int xi_base = xj*Sx - padX; - #if CV_SIMD256 - v_float32x8 zz = v256_setzero_f32(); - v_float32x8 s0 = zz; - #elif CV_SIMD128 - v_float32x4 zz = v_setzero_f32(); - v_float32x4 s0 = zz, s1 = zz; - #else - for (int kk = 0; kk < K0; kk++) { - tmpbuf[kk] = 0.f; - } - #endif - - if (resptr) { - for (int kk = 0; kk < k_count; ++kk) { - const int k = k_base + kk; - int kofs = (k >> K0shift) * planesize + (k & (K0-1)); - resbuf[kk] = resptr[kofs]; - } - } + CONV_INIT_SCALAR_SUMS(); + loadScalarResidual(resptr, k_base, k_count, K0shift, K0, planesize, resbuf); + // ksize-outer, c1-inner (matching weight layout [Kblk][ksize][C1Max][C0*K0]) for (int i = 0; i < ksize; i++) { int zi = zi_base + ofsZYX[i*3 + 0]; int yi = yi_base + ofsZYX[i*3 + 1]; @@ -671,10 +1338,10 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, ((unsigned)xi >= (unsigned)Wi)) != 0) continue; - const float* inptr = inpbaseptr + (((zi * Hi) + yi) * Wi + xi) * C0; - const float* wptr = wbaseptr + i*C1Max*K0*C0; - - for (int c1 = 0; c1 < cblocks; ++c1, inptr += iplanesize, wptr += K0*C0) { + const float* kwptr = wbaseptr + i * C1Max * K0 * C0; + for (int c1 = 0; c1 < cblocks; ++c1) { + const float* inptr = inpbaseptr + c1 * iplanesize + (((zi * Hi) + yi) * Wi + xi) * C0; + const float* wptr = kwptr + c1 * K0 * C0; #if CV_SIMD256 v_float32x8 w, x; #undef CONV_UPDATE_BLOCK1 @@ -716,53 +1383,9 @@ static void conv32fC8(const void* inp__, const void* residual__, void* out__, } float* outbuf = aligned_k ? outptr + k_base*planeblocks : tmpbuf; - - #if CV_SIMD256 - v_float32x8 vscale = v256_load(scalebuf); - v_float32x8 vbias = v256_load(biasbuf); - v_float32x8 valpha = v256_load(alphabuf); - v_float32x8 vmaxval = v256_setall_f32(maxval); - - s0 = v_fma(s0, vscale, vbias); - s0 = v_add(s0, v256_load(resbuf)); - s0 = v_select(v_ge(s0, zz), s0, v_mul(s0, valpha)); - s0 = v_min(s0, vmaxval); - v_store(outbuf, s0); - #elif CV_SIMD128 - v_float32x4 vscale_lo = v_load(scalebuf), vscale_hi = v_load(scalebuf + 4); - v_float32x4 vbias_lo = v_load(biasbuf), vbias_hi = v_load(biasbuf + 4); - v_float32x4 valpha_lo = v_load(alphabuf), valpha_hi = v_load(alphabuf + 4); - v_float32x4 vmaxval = v_setall_f32(maxval); - - s0 = v_fma(s0, vscale_lo, vbias_lo); - s1 = v_fma(s1, vscale_hi, vbias_hi); - s0 = v_add(s0, v_load(resbuf)); - s1 = v_add(s1, v_load(resbuf + 4)); - s0 = v_select(v_ge(s0, zz), s0, v_mul(s0, valpha_lo)); - s1 = v_select(v_ge(s1, zz), s1, v_mul(s1, valpha_hi)); - s0 = v_min(s0, vmaxval); - s1 = v_min(s1, vmaxval); - v_store(outbuf, s0); - v_store(outbuf + 4, s1); - #else - for (int kk = 0; kk < K0; kk++) { - float v = tmpbuf[kk]*scalebuf[kk] + biasbuf[kk] + resbuf[kk]; - v = std::min(v*(v >= 0 ? 1.f : alphabuf[kk]), maxval); - outbuf[kk] = v; - } - #endif - - if (activation) { - activation(outbuf, outbuf, K0, activParams); - } - - if (!aligned_k) { - for (int kk = 0; kk < k_count; kk++) { - const int k = k_base + kk; - int kofs = (k >> K0shift) * planesize + (k & (K0-1)); - outptr[kofs] = tmpbuf[kk]; - } - } + CONV_FINALIZE_SCALAR_OUT(outbuf); + callActivationScalar(activation, outbuf, K0, activParams); + scatterScalarOut(aligned_k, k_base, k_count, K0shift, K0, planesize, outptr, outbuf); } } });