diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 382379cb1c..cd83684f77 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -4,8 +4,8 @@ endif() set(the_description "Deep neural network module. It allows to load models from different frameworks and to make forward pass") -ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV LASX) -ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX RVV LASX) +ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV LASX NEON) +ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX RVV LASX NEON) ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_block" AVX AVX2 NEON NEON_FP16) ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_depthwise" AVX AVX2 RVV LASX) ocv_add_dispatched_file("layers/cpu_kernels/conv_winograd_f63" AVX AVX2 NEON NEON_FP16) diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index 8882168ce8..0da90156b6 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -53,6 +53,95 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, size_t bstep, float* cptr, size_t cstep, int ma, int na, int nb ); +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_NEON + +static const uint32_t tailMaskArray[7] = { + 0u, 0u, 0u, 0u, + 0xffffffffu, 0xffffffffu, 0xffffffffu +}; + +void fastGEMM1T( const float* vec, const float* weights, + size_t wstep, const float* bias, + float* dst, int nvecs, int vecsize ) +{ + int i = 0; + CV_Assert(vecsize >= 4 || vecsize == 0); + const uint32_t* tailMaskPtr = tailMaskArray + (vecsize % 4); + const uint32x4_t tailMaskU = vld1q_u32(tailMaskPtr); + const float32x4_t tailMask = vreinterpretq_f32_u32(tailMaskU); + + for ( ; i <= nvecs - 4; i += 4 ) + { + const float* wptr = weights + i * wstep; + float32x4_t vs0 = vdupq_n_f32(0.0f); + float32x4_t vs1 = vdupq_n_f32(0.0f); + float32x4_t vs2 = vdupq_n_f32(0.0f); + float32x4_t vs3 = vdupq_n_f32(0.0f); + int k = 0; + for ( ; k <= vecsize - 4; k += 4, wptr += 4 ) + { + float32x4_t v = vld1q_f32(vec + k); + + vs0 = vmlaq_f32(vs0, vld1q_f32(wptr), v); + vs1 = vmlaq_f32(vs1, vld1q_f32(wptr + wstep), v); + vs2 = vmlaq_f32(vs2, vld1q_f32(wptr + wstep * 2), v); + vs3 = vmlaq_f32(vs3, vld1q_f32(wptr + wstep * 3), v); + } + if (k != vecsize) + { + k = vecsize - 4; + wptr = weights + i * wstep + k; + float32x4_t v = vld1q_f32(vec + k); + v = vreinterpretq_f32_u32( vandq_u32(vreinterpretq_u32_f32(v), vreinterpretq_u32_f32(tailMask)) ); + float32x4_t w0 = vreinterpretq_f32_u32( vandq_u32(vreinterpretq_u32_f32(vld1q_f32(wptr)), vreinterpretq_u32_f32(tailMask)) ); + float32x4_t w1 = vreinterpretq_f32_u32( vandq_u32(vreinterpretq_u32_f32(vld1q_f32(wptr + wstep)), vreinterpretq_u32_f32(tailMask)) ); + float32x4_t w2 = vreinterpretq_f32_u32( vandq_u32(vreinterpretq_u32_f32(vld1q_f32(wptr + wstep * 2)), vreinterpretq_u32_f32(tailMask)) ); + float32x4_t w3 = vreinterpretq_f32_u32( vandq_u32(vreinterpretq_u32_f32(vld1q_f32(wptr + wstep * 3)), vreinterpretq_u32_f32(tailMask)) ); + vs0 = vmlaq_f32(vs0, w0, v); + vs1 = vmlaq_f32(vs1, w1, v); + vs2 = vmlaq_f32(vs2, w2, v); + vs3 = vmlaq_f32(vs3, w3, v); + } + + auto hsumq_f32 = [](float32x4_t x) -> float { + float32x2_t s2 = vadd_f32(vget_low_f32(x), vget_high_f32(x)); + s2 = vpadd_f32(s2, s2); + return vget_lane_f32(s2, 0); + }; + + dst[i + 0] = hsumq_f32(vs0) + bias[i + 0]; + dst[i + 1] = hsumq_f32(vs1) + bias[i + 1]; + dst[i + 2] = hsumq_f32(vs2) + bias[i + 2]; + dst[i + 3] = hsumq_f32(vs3) + bias[i + 3]; + } + + for ( ; i < nvecs; i++ ) + { + const float* wptr = weights + i * wstep; + float32x4_t vs0 = vdupq_n_f32(0.0f); + int k = 0; + for ( ; k <= vecsize - 4; k += 4, wptr += 4 ) + { + float32x4_t v = vld1q_f32(vec + k); + vs0 = vmlaq_f32(vs0, vld1q_f32(wptr), v); + } + if (k != vecsize) + { + k = vecsize - 4; + wptr = weights + i * wstep + k; + float32x4_t v = vld1q_f32(vec + k); + v = vreinterpretq_f32_u32( vandq_u32(vreinterpretq_u32_f32(v), vreinterpretq_u32_f32(tailMask)) ); + float32x4_t w0 = vreinterpretq_f32_u32( vandq_u32(vreinterpretq_u32_f32(vld1q_f32(wptr)), vreinterpretq_u32_f32(tailMask)) ); + vs0 = vmlaq_f32(vs0, w0, v); + } + float32x2_t s2 = vadd_f32(vget_low_f32(vs0), vget_high_f32(vs0)); + s2 = vpadd_f32(s2, s2); + dst[i] = vget_lane_f32(s2, 0) + bias[i]; + } +} + +#endif //CV_NEON + #if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX #if !CV_FMA3 // AVX workaround diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index ad45a8a2a9..202933b4ca 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -138,6 +138,9 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer #if CV_TRY_AVX2 bool useAVX2; #endif +#if CV_TRY_NEON + bool useNEON; +#endif // CUDA needs input blobs to be rearranged in a specific way, but some transformations // in ONNXImporter are destructive, so we keep a copy. @@ -152,6 +155,9 @@ public: #endif #if CV_TRY_AVX2 , useAVX2(checkHardwareSupport(CPU_AVX2)) +#endif +#if CV_TRY_NEON + , useNEON(checkHardwareSupport(CPU_NEON)) #endif { setParamsFrom(params); @@ -489,6 +495,14 @@ public: && Wh.depth() == CV_32F && hInternal.depth() == CV_32F && gates.depth() == CV_32F && Wh.cols >= 8; #endif +#if CV_TRY_NEON + bool canUseNeon = gates.isContinuous() && bias.isContinuous() + && Wx.depth() == CV_32F && gates.depth() == CV_32F + && bias.depth() == CV_32F && Wx.cols >= 4; + bool canUseNeon_hInternal = hInternal.isContinuous() && gates.isContinuous() && bias.isContinuous() + && Wh.depth() == CV_32F && hInternal.depth() == CV_32F && gates.depth() == CV_32F + && Wh.cols >= 4; +#endif int tsStart, tsEnd, tsInc; if (reverse || i == 1) { @@ -539,6 +553,23 @@ public: } } else +#endif +#if CV_TRY_NEON + if (useNEON && canUseNeon && xCurr.isContinuous()) + { + for (int n = 0; n < xCurr.rows; n++) { + opt_NEON::fastGEMM1T( + xCurr.ptr(n), + Wx.ptr(), + Wx.step1(), + bias.ptr(), + gates.ptr(n), + Wx.rows, + Wx.cols + ); + } + } + else #endif { gemm(xCurr, Wx, 1, gates, 0, gates, GEMM_2_T); // Wx * x_t @@ -578,6 +609,23 @@ public: } } else +#endif +#if CV_TRY_NEON + if (useNEON && canUseNeon_hInternal) + { + for (int n = 0; n < hInternal.rows; n++) { + opt_NEON::fastGEMM1T( + hInternal.ptr(n), + Wh.ptr(), + Wh.step1(), + gates.ptr(n), + gates.ptr(n), + Wh.rows, + Wh.cols + ); + } + } + else #endif { gemm(hInternal, Wh, 1, gates, 1, gates, GEMM_2_T); //+Wh * h_{t-1}