potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 15:53:03 +04:00
Code Packages Releases Wiki Activity

Merge pull request #28889 from abhishek-gola:simd_kernel_speedup

Browse Source 
SIMD Kernel speedup for DNN layers #28889

This PR add following speedups for Grounding Dino tiny model.

For Device:  Intel(R) Core(TM) i9-14900KS, x86, 32 Cores, ubuntu 24.04,
| Model | `ENGINE_NEW (Before)` | `ENGINE_NEW (After)` | `ENGINE_ORT` |
| :--- | :--- | :--- | :--- | 
| **Grounding Dino Tiny** | 3130 ms | 1872.06 ms| 1800.18 ms|

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
2026-05-21 18:23:15 +05:30
committed by GitHub
parent 1d54398f07
commit 0a8603b3ac
21 changed files with 1850 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/core/CMakeLists.txt
+1
View File
@@ -16,6 +16,7 @@ ocv_add_dispatched_file(split SSE2 AVX2 LASX)
ocv_add_dispatched_file(sum SSE2 AVX2 LASX)
ocv_add_dispatched_file(reduce SSE2 SSSE3 AVX2 NEON_DOTPROD)
ocv_add_dispatched_file(norm SSE2 SSE4_1 AVX AVX2 NEON_DOTPROD LASX)
ocv_add_dispatched_file(transpose AVX AVX2 NEON RVV LASX)
# dispatching for accuracy tests
ocv_add_dispatched_file_force_all(test_intrin128 TEST SSE2 SSE3 SSSE3 SSE4_1 SSE4_2 AVX FP16 AVX2 AVX512_SKX)
modules/core/src/matrix_transform.cpp
+7 -32
View File
@@ -7,6 +7,9 @@
#include "hal_replacement.hpp"
#include "opencv2/core/detail/dispatch_helper.impl.hpp"
#include "transpose.simd.hpp"
#include "transpose.simd_declarations.hpp"
#include <algorithm> // std::swap_ranges
#include <numeric> // std::accumulate
@@ -179,38 +182,10 @@ static void transpose_16bit_simd(const uchar* src, size_t sstep, uchar* dst, siz
static void transpose_32bit_simd(const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size sz)
{
const uint32_t* src32 = reinterpret_cast<const uint32_t*>(src);
uint32_t* dst32 = reinterpret_cast<uint32_t*>(dst);
const size_t sstep_e = sstep / sizeof(uint32_t);
const size_t dstep_e = dstep / sizeof(uint32_t);
const int m = sz.width, n = sz.height;
int i = 0;
for (; i <= m - 4; i += 4)
{
int j = 0;
for (; j <= n - 4; j += 4)
{
v_uint32x4 r0 = v_load(src32 + i + sstep_e*(j+0));
v_uint32x4 r1 = v_load(src32 + i + sstep_e*(j+1));
v_uint32x4 r2 = v_load(src32 + i + sstep_e*(j+2));
v_uint32x4 r3 = v_load(src32 + i + sstep_e*(j+3));
v_uint32x4 o0, o1, o2, o3;
v_transpose4x4(r0, r1, r2, r3, o0, o1, o2, o3);
v_store(dst32 + dstep_e*(i+0) + j, o0);
v_store(dst32 + dstep_e*(i+1) + j, o1);
v_store(dst32 + dstep_e*(i+2) + j, o2);
v_store(dst32 + dstep_e*(i+3) + j, o3);
}
for (; j < n; j++)
for (int k = 0; k < 4; k++)
dst32[dstep_e*(i+k) + j] = src32[i + sstep_e*j + k];
}
for (; i < m; i++)
for (int j = 0; j < n; j++)
dst32[dstep_e*i + j] = src32[i + sstep_e*j];
// Cache-blocked + AVX2-aware path lives in transpose.simd.hpp; pick the
// best variant available at runtime.
CV_CPU_DISPATCH(transpose_32bit_blocks_simd, (src, sstep, dst, dstep, sz),
CV_CPU_DISPATCH_MODES_ALL);
}
static void transpose_48bit_simd(const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size sz)
modules/core/src/transpose.simd.hpp
+131
View File
@@ -0,0 +1,131 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "opencv2/core/hal/intrin.hpp"
namespace cv {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
void transpose_32bit_blocks_simd(const uchar* src, size_t sstep,
uchar* dst, size_t dstep, Size sz);
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
static inline void transpose_32bit_blocks_tile(const uint32_t* src32, size_t sstep_e,
uint32_t* dst32, size_t dstep_e,
int i_lo, int i_hi, int j_lo, int j_hi)
{
int i = i_lo;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (VTraits<v_uint32>::vlanes() == 8)
{
for (; i + 8 <= i_hi; i += 8)
{
int j = j_lo;
for (; j + 8 <= j_hi; j += 8)
{
v_uint32 r0 = vx_load(src32 + i + sstep_e*(j+0));
v_uint32 r1 = vx_load(src32 + i + sstep_e*(j+1));
v_uint32 r2 = vx_load(src32 + i + sstep_e*(j+2));
v_uint32 r3 = vx_load(src32 + i + sstep_e*(j+3));
v_uint32 r4 = vx_load(src32 + i + sstep_e*(j+4));
v_uint32 r5 = vx_load(src32 + i + sstep_e*(j+5));
v_uint32 r6 = vx_load(src32 + i + sstep_e*(j+6));
v_uint32 r7 = vx_load(src32 + i + sstep_e*(j+7));
v_uint32 a0, a1, a2, a3, a4, a5, a6, a7;
v_transpose4x4(r0, r1, r2, r3, a0, a1, a2, a3);
v_transpose4x4(r4, r5, r6, r7, a4, a5, a6, a7);
v_uint32 b0 = v_combine_low (a0, a4);
v_uint32 b1 = v_combine_low (a1, a5);
v_uint32 b2 = v_combine_low (a2, a6);
v_uint32 b3 = v_combine_low (a3, a7);
v_uint32 b4 = v_combine_high(a0, a4);
v_uint32 b5 = v_combine_high(a1, a5);
v_uint32 b6 = v_combine_high(a2, a6);
v_uint32 b7 = v_combine_high(a3, a7);
vx_store(dst32 + dstep_e*(i+0) + j, b0);
vx_store(dst32 + dstep_e*(i+1) + j, b1);
vx_store(dst32 + dstep_e*(i+2) + j, b2);
vx_store(dst32 + dstep_e*(i+3) + j, b3);
vx_store(dst32 + dstep_e*(i+4) + j, b4);
vx_store(dst32 + dstep_e*(i+5) + j, b5);
vx_store(dst32 + dstep_e*(i+6) + j, b6);
vx_store(dst32 + dstep_e*(i+7) + j, b7);
}
for (; j < j_hi; j++)
for (int k = 0; k < 8; k++)
dst32[dstep_e*(i+k) + j] = src32[i + sstep_e*j + k];
}
}
else if (VTraits<v_uint32>::vlanes() == 4)
{
for (; i + 4 <= i_hi; i += 4)
{
int j = j_lo;
for (; j + 4 <= j_hi; j += 4)
{
v_uint32 r0 = vx_load(src32 + i + sstep_e*(j+0));
v_uint32 r1 = vx_load(src32 + i + sstep_e*(j+1));
v_uint32 r2 = vx_load(src32 + i + sstep_e*(j+2));
v_uint32 r3 = vx_load(src32 + i + sstep_e*(j+3));
v_uint32 o0, o1, o2, o3;
v_transpose4x4(r0, r1, r2, r3, o0, o1, o2, o3);
vx_store(dst32 + dstep_e*(i+0) + j, o0);
vx_store(dst32 + dstep_e*(i+1) + j, o1);
vx_store(dst32 + dstep_e*(i+2) + j, o2);
vx_store(dst32 + dstep_e*(i+3) + j, o3);
}
for (; j < j_hi; j++)
for (int k = 0; k < 4; k++)
dst32[dstep_e*(i+k) + j] = src32[i + sstep_e*j + k];
}
}
#endif
for (; i < i_hi; i++)
for (int j = j_lo; j < j_hi; j++)
dst32[dstep_e*i + j] = src32[i + sstep_e*j];
}
void transpose_32bit_blocks_simd(const uchar* src, size_t sstep,
uchar* dst, size_t dstep, Size sz)
{
const uint32_t* src32 = reinterpret_cast<const uint32_t*>(src);
uint32_t* dst32 = reinterpret_cast<uint32_t*>(dst);
const size_t sstep_e = sstep / sizeof(uint32_t);
const size_t dstep_e = dstep / sizeof(uint32_t);
const int m = sz.width, n = sz.height;
// Single-threaded for small inputs to avoid parallel_for_ overhead.
const int64_t kMinParallelElements = 64 * 1024;
if ((int64_t)m * n < kMinParallelElements)
{
transpose_32bit_blocks_tile(src32, sstep_e, dst32, dstep_e, 0, m, 0, n);
return;
}
const int TILE = 32;
const int mtiles = (m + TILE - 1) / TILE;
const int ntiles = (n + TILE - 1) / TILE;
const int64_t totalTiles = (int64_t)mtiles * ntiles;
parallel_for_(Range(0, (int)totalTiles), [&](const Range& r)
{
for (int idx = r.start; idx < r.end; idx++)
{
int ti = idx / ntiles;
int tj = idx - ti * ntiles;
int i_lo = ti * TILE, i_hi = std::min(i_lo + TILE, m);
int j_lo = tj * TILE, j_hi = std::min(j_lo + TILE, n);
transpose_32bit_blocks_tile(src32, sstep_e, dst32, dstep_e, i_lo, i_hi, j_lo, j_hi);
}
});
}
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
CV_CPU_OPTIMIZATION_NAMESPACE_END
} // namespace cv
modules/dnn/CMakeLists.txt
+4
View File
@@ -14,6 +14,10 @@ ocv_add_dispatched_file("layers/cpu_kernels/conv2_depthwise" AVX AVX2 NEON NEON_
ocv_add_dispatched_file("layers/cpu_kernels/conv2_kernels" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file_force_all("int8layers/conv2_int8_kernels" AVX2)
ocv_add_dispatched_file("layers/cpu_kernels/activation_kernels" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/reduce2_kernels" AVX AVX2 NEON RVV LASX)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/transpose_kernels" AVX AVX2 NEON RVV LASX)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/gridsample_kernels" AVX AVX2 NEON RVV LASX)
ocv_add_dispatched_file_force_all("layers/cpu_kernels/nary_eltwise_kernels" AVX AVX2 NEON RVV LASX)
ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js)
modules/dnn/src/layers/clip_layer.cpp
+42
View File
@@ -5,6 +5,23 @@
// Copyright (C) 2025, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#define CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#include "cpu_kernels/activation_kernels.simd.hpp"
#include "layers/cpu_kernels/activation_kernels.simd_declarations.hpp"
#undef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
namespace cv { namespace dnn { namespace {
static inline void clampFloatChunkDispatch(const float* src, float* dst,
size_t n, float lo, float hi) {
CV_CPU_DISPATCH(clampFloatChunk_, (src, dst, n, lo, hi),
CV_CPU_DISPATCH_MODES_ALL);
}
}}}
#define CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN namespace cpu_baseline {
#define CV_CPU_OPTIMIZATION_NAMESPACE_END }
#undef CV_CPU_DISPATCH_MODES_ALL
#include "layers_common.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include <opencv2/core/hal/interface.h>
@@ -67,6 +84,9 @@ public:
return backendId == DNN_BACKEND_OPENCV;
}
// Clip rewrites values in place of the input layout.
virtual bool alwaysSupportInplace() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
@@ -122,6 +142,28 @@ public:
double actualMax = dynMax ? getScalar(inputs[2]) : (hasMax ? maxValue : typeMax(data.depth()));
CV_Assert(actualMin <= actualMax);
// Fused single-pass clamp for contiguous CV_32F. Halves memory traffic
// vs the cv::max + cv::min pair, and lets the allocator run it in-place.
if (data.depth() == CV_32F && data.isContinuous() && dst.isContinuous() &&
data.total() == dst.total()) {
const float lo = (float)actualMin;
const float hi = (float)actualMax;
const size_t total = data.total();
const float* src = data.ptr<float>();
float* out = dst.ptr<float>();
const size_t CHUNK = 16384;
int nChunks = (int)((total + CHUNK - 1) / CHUNK);
parallel_for_(Range(0, nChunks), [&](const Range& r) {
for (int c = r.start; c < r.end; c++) {
size_t start = (size_t)c * CHUNK;
size_t end = std::min(start + CHUNK, total);
clampFloatChunkDispatch(src + start, out + start,
end - start, lo, hi);
}
});
return;
}
Scalar lowS = Scalar::all(actualMin);
Scalar highS = Scalar::all(actualMax);
cv::max(data, lowS, dst);
modules/dnn/src/layers/cpu_kernels/activation_kernels.simd.hpp
+136
View File
@@ -5,8 +5,12 @@
// Third party copyrights are property of their respective owners.
#include <opencv2/dnn/all_layers.hpp>
#include <opencv2/dnn/shape_utils.hpp>
#include "opencv2/core/hal/intrin.hpp"
#include "opencv2/core/fast_math.hpp"
#include <math.h>
#include <cfloat>
#include <algorithm>
namespace cv {
namespace dnn {
@@ -14,6 +18,12 @@ CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
cv::dnn::ActivationFunc getActivationFunc_(int type);
// Per-row softmax over a contiguous Mat axis.
void softmax_(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep, float scale);
// Fused clamp on a single contiguous chunk, 4x unrolled. Used by clip_layer.
void clampFloatChunk_(const float* src, float* dst, size_t n, float lo, float hi);
CV_CPU_OPTIMIZATION_NAMESPACE_END
}}
@@ -300,6 +310,132 @@ static void activationClip(const void* input, void* output,
}
}
void clampFloatChunk_(const float* src, float* dst, size_t n, float lo, float hi) {
size_t i = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int lanes = VTraits<v_float32>::vlanes();
v_float32 vlo = vx_setall_f32(lo);
v_float32 vhi = vx_setall_f32(hi);
for (; i + lanes * 4 <= n; i += lanes * 4) {
v_store(dst + i, v_min(v_max(vx_load(src + i), vlo), vhi));
v_store(dst + i + lanes, v_min(v_max(vx_load(src + i + lanes), vlo), vhi));
v_store(dst + i + lanes * 2, v_min(v_max(vx_load(src + i + lanes * 2), vlo), vhi));
v_store(dst + i + lanes * 3, v_min(v_max(vx_load(src + i + lanes * 3), vlo), vhi));
}
for (; i + lanes <= n; i += lanes)
v_store(dst + i, v_min(v_max(vx_load(src + i), vlo), vhi));
#endif
for (; i < n; i++)
dst[i] = std::min(std::max(src[i], lo), hi);
}
void softmax_(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep, float scale) {
CV_Assert(src.type() == CV_32F);
CV_Assert(src.isContinuous() && dst.isContinuous());
CV_Assert(src.size == dst.size);
axis = normalize_axis(axis, src.dims);
size_t outerSize = src.total(0, axis),
innerSize = src.total(axis + 1);
const float *srcPtr = src.ptr<float>();
float *dstPtr = dst.ptr<float>();
size_t outerStep = src.total(axis);
size_t cnStep = src.total(axis + 1);
// multi-threads: weight by axisStep so axis=-1 with small outerSize*innerSize
// (e.g. [4,256,13294] -> 1024 tasks of 13294 elems) still parallelizes.
size_t totalTasks = outerSize * innerSize;
double nstripes = (double) totalTasks * (double) axisStep / 8192.0;
if (nstripes < 1.0) nstripes = 1.0;
size_t channelAxis = (axisStep + 7) & -8;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int nlanes = VTraits<v_float32>::vlanes();
#endif
parallel_for_(Range(0, (int) totalTasks), [&](const Range &range) {
AutoBuffer<float> axisBuf_(channelAxis);
float *axisBuf = axisBuf_.data();
for (size_t i = range.start; i < range.end; i++) {
size_t outerDim = i / innerSize;
size_t innerDim = i % innerSize;
size_t srcOffset = outerDim * outerStep + innerDim;
size_t _cnDim = 0;
#if CV_ENABLE_UNROLLED && defined(_M_ARM64)
for (; _cnDim + 3 < axisStep; _cnDim += 4) {
axisBuf[_cnDim + 0] = srcPtr[srcOffset + (_cnDim + 0 + axisBias) * cnStep];
axisBuf[_cnDim + 1] = srcPtr[srcOffset + (_cnDim + 1 + axisBias) * cnStep];
axisBuf[_cnDim + 2] = srcPtr[srcOffset + (_cnDim + 2 + axisBias) * cnStep];
axisBuf[_cnDim + 3] = srcPtr[srcOffset + (_cnDim + 3 + axisBias) * cnStep];
}
#endif
for (; _cnDim < axisStep; _cnDim++)
axisBuf[_cnDim] = srcPtr[srcOffset + (_cnDim + axisBias) * cnStep];
float maxVal = -FLT_MAX;
int cnDim = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 vmax = vx_setall_f32(-FLT_MAX);
for (; cnDim < axisStep; cnDim += nlanes) {
if (cnDim > axisStep - nlanes) {
if (cnDim == 0) { break; }
cnDim = axisStep - nlanes;
}
v_float32 val = vx_load(axisBuf + cnDim);
vmax = v_max(vmax, val);
}
maxVal = v_reduce_max(vmax);
#endif
for (; cnDim < axisStep; cnDim++) {
maxVal = std::max(maxVal, axisBuf[cnDim]);
}
// Fuse the input scale into the centered exp: softmax(x*scale) where
// exp((x - maxVal) * scale) is numerically equivalent to pre-multiplying.
float s = 0.f;
cnDim = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 vs = vx_setzero_f32();
vmax = vx_setall_f32(maxVal);
v_float32 vscale = vx_setall_f32(scale);
for (; cnDim <= axisStep - nlanes; cnDim += nlanes) {
v_float32 val = vx_load(axisBuf + cnDim);
val = v_mul(v_sub(val, vmax), vscale);
val = v_exp(val);
vs = v_add(vs, val);
v_store(axisBuf + cnDim, val);
}
s = v_reduce_sum(vs);
#endif
for (; cnDim < axisStep; cnDim++) {
axisBuf[cnDim] = expf((axisBuf[cnDim] - maxVal) * scale);
s += axisBuf[cnDim];
}
_cnDim = 0;
if (s == 0.f || cvIsInf(1.f / s)) {
for (; _cnDim < axisStep; _cnDim++)
dstPtr[srcOffset + (_cnDim + axisBias) * cnStep] = 0.f;
} else {
s = 1.f / s;
#if CV_ENABLE_UNROLLED && defined(_M_ARM64)
for (; _cnDim + 3 < axisStep; _cnDim += 4) {
dstPtr[srcOffset + (_cnDim + 0 + axisBias) * cnStep] = axisBuf[_cnDim + 0] * s;
dstPtr[srcOffset + (_cnDim + 1 + axisBias) * cnStep] = axisBuf[_cnDim + 1] * s;
dstPtr[srcOffset + (_cnDim + 2 + axisBias) * cnStep] = axisBuf[_cnDim + 2] * s;
dstPtr[srcOffset + (_cnDim + 3 + axisBias) * cnStep] = axisBuf[_cnDim + 3] * s;
}
#endif
for (; _cnDim < axisStep; _cnDim++)
dstPtr[srcOffset + (_cnDim + axisBias) * cnStep] = axisBuf[_cnDim] * s;
}
}
}, nstripes);
}
ActivationFunc getActivationFunc_(int type)
{
switch (type) {
modules/dnn/src/layers/cpu_kernels/gridsample_kernels.simd.hpp
+695
View File
@@ -0,0 +1,695 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include <opencv2/core.hpp>
#include "opencv2/core/hal/intrin.hpp"
#include <cfloat>
#include <algorithm>
#include <cmath>
namespace cv { namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
// padding values: 0 = zeros, 1 = border, 2 = reflection.
void gridSampleBilinear2D_f32_(
const float* X, const float* G, float* Y,
int N, int C, int H, int W, int Hout, int Wout,
bool align_corners, int padding);
void gridSampleNearest2D_f32_(
const float* X, const float* G, float* Y,
int N, int C, int H, int W, int Hout, int Wout,
bool align_corners, int padding);
void gridSampleBicubic2D_f32_(
const float* X, const float* G, float* Y,
int N, int C, int H, int W, int Hout, int Wout,
bool align_corners, int padding, float cubic_alpha);
void gridSampleBilinear3D_f32_(
const float* X, const float* G, float* Y,
int N, int C, int D, int H, int W, int Dout, int Hout, int Wout,
bool align_corners, int padding);
void gridSampleNearest3D_f32_(
const float* X, const float* G, float* Y,
int N, int C, int D, int H, int W, int Dout, int Hout, int Wout,
bool align_corners, int padding);
CV_CPU_OPTIMIZATION_NAMESPACE_END
}}
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
namespace cv { namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
namespace gs_simd_internal {
enum { GS_ZEROS = 0, GS_BORDER = 1, GS_REFLECTION = 2 };
static inline float gs_reflect(float x, int limit, bool align_corners)
{
if (limit <= 1) return 0.f;
const float minv = align_corners ? 0.f : -0.5f;
const float maxv = align_corners ? float(limit - 1) : float(limit) - 0.5f;
const float m = maxv - minv;
const float two_m = 2.f * m;
float t = fmodf(x - minv, two_m);
if (t < 0) t += two_m;
if (t > m) t = two_m - t;
return t + minv;
}
static inline void gs_cubic_coeffs(float x, float A, float* c)
{
c[0] = ((A*(x + 1.0f) - 5.0f*A)*(x + 1.0f) + 8.0f*A)*(x + 1.0f) - 4.0f*A;
c[1] = ((A + 2.0f)*x - (A + 3.0f))*x*x + 1.0f;
c[2] = ((A + 2.0f)*(1.0f - x) - (A + 3.0f))*(1.0f - x)*(1.0f - x) + 1.0f;
c[3] = 1.0f - c[0] - c[1] - c[2];
}
static inline void gs_xy_params(int W, int H, bool align_corners,
float& xs, float& ys, float& xd, float& yd)
{
const float delta = align_corners ? 1.f : 0.f;
xs = 0.5f * (W - delta);
ys = 0.5f * (H - delta);
xd = 0.5f * (W - delta) + 0.5f * (delta - 1.f);
yd = 0.5f * (H - delta) + 0.5f * (delta - 1.f);
}
static inline void gs_xyz_params(int W, int H, int D, bool align_corners,
float& xs, float& ys, float& zs,
float& xd, float& yd, float& zd)
{
const float delta = align_corners ? 1.f : 0.f;
xs = 0.5f * (W - delta);
ys = 0.5f * (H - delta);
zs = 0.5f * (D - delta);
xd = 0.5f * (W - delta) + 0.5f * (delta - 1.f);
yd = 0.5f * (H - delta) + 0.5f * (delta - 1.f);
zd = 0.5f * (D - delta) + 0.5f * (delta - 1.f);
}
template<int PAD>
static inline float gs_fetch2D(const float* baseNC, int yy, int xx,
int H, int W, size_t xHStride, bool align_corners)
{
int xi = xx, yi = yy;
if (PAD == GS_BORDER) {
xi = std::max(0, std::min(W - 1, xi));
yi = std::max(0, std::min(H - 1, yi));
} else if (PAD == GS_REFLECTION) {
xi = saturate_cast<int>(std::floor(gs_reflect((float)xi, W, align_corners) + 0.5f));
yi = saturate_cast<int>(std::floor(gs_reflect((float)yi, H, align_corners) + 0.5f));
}
if (xi < 0 || yi < 0 || xi >= W || yi >= H) return 0.f;
return baseNC[(size_t)yi * xHStride + (size_t)xi];
}
template<int PAD>
static inline float gs_fetch3D(const float* baseNC, int zz, int yy, int xx,
int D, int H, int W,
size_t xDStride, size_t xHStride, bool align_corners)
{
int xi = xx, yi = yy, zi = zz;
if (PAD == GS_BORDER) {
xi = std::max(0, std::min(W - 1, xi));
yi = std::max(0, std::min(H - 1, yi));
zi = std::max(0, std::min(D - 1, zi));
} else if (PAD == GS_REFLECTION) {
xi = saturate_cast<int>(std::floor(gs_reflect((float)xi, W, align_corners) + 0.5f));
yi = saturate_cast<int>(std::floor(gs_reflect((float)yi, H, align_corners) + 0.5f));
zi = saturate_cast<int>(std::floor(gs_reflect((float)zi, D, align_corners) + 0.5f));
}
if (xi < 0 || yi < 0 || zi < 0 || xi >= W || yi >= H || zi >= D) return 0.f;
return baseNC[(size_t)zi * xDStride + (size_t)yi * xHStride + (size_t)xi];
}
template<int PAD>
static inline void bilinear2D(const float* X, const float* G, float* Y,
int N, int C, int H, int W,
int Hout, int Wout, bool align_corners)
{
const size_t xNStride = (size_t)C * H * W;
const size_t xCStride = (size_t)H * W;
const size_t xHStride = (size_t)W;
const size_t gNStride = (size_t)Hout * Wout * 2;
const size_t gHStride = (size_t)Wout * 2;
const size_t yNStride = (size_t)C * Hout * Wout;
const size_t yCStride = (size_t)Hout * Wout;
const size_t yHStride = (size_t)Wout;
float xscale, yscale, xdelta, ydelta;
gs_xy_params(W, H, align_corners, xscale, yscale, xdelta, ydelta);
parallel_for_(Range(0, N * Hout), [&](const Range& r) {
for (int idx = r.start; idx < r.end; idx++) {
int n = idx / Hout, h = idx - n * Hout;
const float* baseN = X + (size_t)n * xNStride;
const float* gRow = G + (size_t)n * gNStride + (size_t)h * gHStride;
float* outBase = Y + (size_t)n * yNStride + (size_t)h * yHStride;
for (int w = 0; w < Wout; w++) {
float xf = gRow[w * 2 + 0] * xscale + xdelta;
float yf = gRow[w * 2 + 1] * yscale + ydelta;
int x0 = saturate_cast<int>(floorf(xf));
int y0 = saturate_cast<int>(floorf(yf));
float fx = xf - x0, fy = yf - y0;
float w00 = (1.f - fx) * (1.f - fy);
float w01 = fx * (1.f - fy);
float w10 = (1.f - fx) * fy;
float w11 = fx * fy;
bool interior = (x0 >= 0 && y0 >= 0 && x0 + 1 < W && y0 + 1 < H);
int c = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (interior) {
const int L = VTraits<v_float32>::vlanes();
const size_t off00 = (size_t)y0 * xHStride + (size_t)x0;
const size_t off01 = off00 + 1;
const size_t off10 = off00 + xHStride;
const size_t off11 = off10 + 1;
const v_float32 vw00 = vx_setall_f32(w00);
const v_float32 vw01 = vx_setall_f32(w01);
const v_float32 vw10 = vx_setall_f32(w10);
const v_float32 vw11 = vx_setall_f32(w11);
for (; c + L <= C; c += L) {
float CV_DECL_ALIGNED(32) b00[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b01[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b10[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b11[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) bo[VTraits<v_float32>::max_nlanes];
for (int k = 0; k < L; k++) {
const float* pNC = baseN + (size_t)(c + k) * xCStride;
b00[k] = pNC[off00];
b01[k] = pNC[off01];
b10[k] = pNC[off10];
b11[k] = pNC[off11];
}
v_float32 V00 = vx_load_aligned(b00);
v_float32 V01 = vx_load_aligned(b01);
v_float32 V10 = vx_load_aligned(b10);
v_float32 V11 = vx_load_aligned(b11);
v_float32 R = v_add(v_add(v_mul(V00, vw00), v_mul(V01, vw01)),
v_add(v_mul(V10, vw10), v_mul(V11, vw11)));
v_store_aligned(bo, R);
for (int k = 0; k < L; k++) {
outBase[(size_t)(c + k) * yCStride + (size_t)w] = bo[k];
}
}
}
#endif
for (; c < C; c++) {
const float* baseNC = baseN + (size_t)c * xCStride;
float v00, v01, v10, v11;
if (interior) {
const float* py0 = baseNC + (size_t)y0 * xHStride;
const float* py1 = py0 + xHStride;
v00 = py0[x0]; v01 = py0[x0 + 1];
v10 = py1[x0]; v11 = py1[x0 + 1];
} else {
v00 = gs_fetch2D<PAD>(baseNC, y0, x0, H, W, xHStride, align_corners);
v01 = gs_fetch2D<PAD>(baseNC, y0, x0 + 1, H, W, xHStride, align_corners);
v10 = gs_fetch2D<PAD>(baseNC, y0 + 1, x0, H, W, xHStride, align_corners);
v11 = gs_fetch2D<PAD>(baseNC, y0 + 1, x0 + 1, H, W, xHStride, align_corners);
}
outBase[(size_t)c * yCStride + (size_t)w] = w00 * v00 + w01 * v01 + w10 * v10 + w11 * v11;
}
}
}
});
}
template<int PAD>
static inline void nearest2D(const float* X, const float* G, float* Y,
int N, int C, int H, int W,
int Hout, int Wout, bool align_corners)
{
const size_t xNStride = (size_t)C * H * W;
const size_t xCStride = (size_t)H * W;
const size_t xHStride = (size_t)W;
const size_t gNStride = (size_t)Hout * Wout * 2;
const size_t gHStride = (size_t)Wout * 2;
const size_t yNStride = (size_t)C * Hout * Wout;
const size_t yCStride = (size_t)Hout * Wout;
const size_t yHStride = (size_t)Wout;
float xscale, yscale, xdelta, ydelta;
gs_xy_params(W, H, align_corners, xscale, yscale, xdelta, ydelta);
parallel_for_(Range(0, N * Hout), [&](const Range& r) {
for (int idx = r.start; idx < r.end; idx++) {
int n = idx / Hout, h = idx - n * Hout;
const float* baseN = X + (size_t)n * xNStride;
const float* gRow = G + (size_t)n * gNStride + (size_t)h * gHStride;
float* outBase = Y + (size_t)n * yNStride + (size_t)h * yHStride;
for (int w = 0; w < Wout; w++) {
float xf = gRow[w * 2 + 0] * xscale + xdelta;
float yf = gRow[w * 2 + 1] * yscale + ydelta;
int xi = cvRound(xf);
int yi = cvRound(yf);
bool interior = (xi >= 0 && yi >= 0 && xi < W && yi < H);
int c = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (interior) {
const int L = VTraits<v_float32>::vlanes();
const size_t off = (size_t)yi * xHStride + (size_t)xi;
for (; c + L <= C; c += L) {
float CV_DECL_ALIGNED(32) b[VTraits<v_float32>::max_nlanes];
for (int k = 0; k < L; k++) {
b[k] = baseN[(size_t)(c + k) * xCStride + off];
}
for (int k = 0; k < L; k++) {
outBase[(size_t)(c + k) * yCStride + (size_t)w] = b[k];
}
}
}
#endif
for (; c < C; c++) {
const float* baseNC = baseN + (size_t)c * xCStride;
float v;
if (interior) v = baseNC[(size_t)yi * xHStride + (size_t)xi];
else v = gs_fetch2D<PAD>(baseNC, yi, xi, H, W, xHStride, align_corners);
outBase[(size_t)c * yCStride + (size_t)w] = v;
}
}
}
});
}
template<int PAD>
static inline void bicubic2D(const float* X, const float* G, float* Y,
int N, int C, int H, int W,
int Hout, int Wout, bool align_corners,
float cubic_alpha)
{
const size_t xNStride = (size_t)C * H * W;
const size_t xCStride = (size_t)H * W;
const size_t xHStride = (size_t)W;
const size_t gNStride = (size_t)Hout * Wout * 2;
const size_t gHStride = (size_t)Wout * 2;
const size_t yNStride = (size_t)C * Hout * Wout;
const size_t yCStride = (size_t)Hout * Wout;
const size_t yHStride = (size_t)Wout;
float xscale, yscale, xdelta, ydelta;
gs_xy_params(W, H, align_corners, xscale, yscale, xdelta, ydelta);
parallel_for_(Range(0, N * Hout), [&](const Range& r) {
for (int idx = r.start; idx < r.end; idx++) {
int n = idx / Hout, h = idx - n * Hout;
const float* baseN = X + (size_t)n * xNStride;
const float* gRow = G + (size_t)n * gNStride + (size_t)h * gHStride;
float* outBase = Y + (size_t)n * yNStride + (size_t)h * yHStride;
for (int w = 0; w < Wout; w++) {
float xf = gRow[w * 2 + 0] * xscale + xdelta;
float yf = gRow[w * 2 + 1] * yscale + ydelta;
int x1 = saturate_cast<int>(floorf(xf));
int y1 = saturate_cast<int>(floorf(yf));
float tx = xf - x1, ty = yf - y1;
float wx[4], wy[4];
gs_cubic_coeffs(tx, cubic_alpha, wx);
gs_cubic_coeffs(ty, cubic_alpha, wy);
bool interior = (x1 >= 1 && y1 >= 1 && x1 + 2 < W && y1 + 2 < H);
int c = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (interior) {
const int L = VTraits<v_float32>::vlanes();
const v_float32 vwx0 = vx_setall_f32(wx[0]);
const v_float32 vwx1 = vx_setall_f32(wx[1]);
const v_float32 vwx2 = vx_setall_f32(wx[2]);
const v_float32 vwx3 = vx_setall_f32(wx[3]);
const v_float32 vwy0 = vx_setall_f32(wy[0]);
const v_float32 vwy1 = vx_setall_f32(wy[1]);
const v_float32 vwy2 = vx_setall_f32(wy[2]);
const v_float32 vwy3 = vx_setall_f32(wy[3]);
for (; c + L <= C; c += L) {
float CV_DECL_ALIGNED(32) a[16][VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) bo[VTraits<v_float32>::max_nlanes];
for (int k = 0; k < L; k++) {
const float* pNC = baseN + (size_t)(c + k) * xCStride;
for (int j = 0; j < 4; j++) {
const float* p = pNC + (size_t)(y1 - 1 + j) * xHStride + (size_t)(x1 - 1);
a[j*4 + 0][k] = p[0];
a[j*4 + 1][k] = p[1];
a[j*4 + 2][k] = p[2];
a[j*4 + 3][k] = p[3];
}
}
v_float32 row0, row1, row2, row3;
v_float32 mn, mx;
{
v_float32 c0 = vx_load_aligned(a[0]);
v_float32 c1 = vx_load_aligned(a[1]);
v_float32 c2 = vx_load_aligned(a[2]);
v_float32 c3 = vx_load_aligned(a[3]);
row0 = v_add(v_add(v_mul(c0, vwx0), v_mul(c1, vwx1)),
v_add(v_mul(c2, vwx2), v_mul(c3, vwx3)));
mn = v_min(v_min(c0, c1), v_min(c2, c3));
mx = v_max(v_max(c0, c1), v_max(c2, c3));
}
{
v_float32 c0 = vx_load_aligned(a[4]);
v_float32 c1 = vx_load_aligned(a[5]);
v_float32 c2 = vx_load_aligned(a[6]);
v_float32 c3 = vx_load_aligned(a[7]);
row1 = v_add(v_add(v_mul(c0, vwx0), v_mul(c1, vwx1)),
v_add(v_mul(c2, vwx2), v_mul(c3, vwx3)));
mn = v_min(mn, v_min(v_min(c0, c1), v_min(c2, c3)));
mx = v_max(mx, v_max(v_max(c0, c1), v_max(c2, c3)));
}
{
v_float32 c0 = vx_load_aligned(a[8]);
v_float32 c1 = vx_load_aligned(a[9]);
v_float32 c2 = vx_load_aligned(a[10]);
v_float32 c3 = vx_load_aligned(a[11]);
row2 = v_add(v_add(v_mul(c0, vwx0), v_mul(c1, vwx1)),
v_add(v_mul(c2, vwx2), v_mul(c3, vwx3)));
mn = v_min(mn, v_min(v_min(c0, c1), v_min(c2, c3)));
mx = v_max(mx, v_max(v_max(c0, c1), v_max(c2, c3)));
}
{
v_float32 c0 = vx_load_aligned(a[12]);
v_float32 c1 = vx_load_aligned(a[13]);
v_float32 c2 = vx_load_aligned(a[14]);
v_float32 c3 = vx_load_aligned(a[15]);
row3 = v_add(v_add(v_mul(c0, vwx0), v_mul(c1, vwx1)),
v_add(v_mul(c2, vwx2), v_mul(c3, vwx3)));
mn = v_min(mn, v_min(v_min(c0, c1), v_min(c2, c3)));
mx = v_max(mx, v_max(v_max(c0, c1), v_max(c2, c3)));
}
v_float32 R = v_add(v_add(v_mul(row0, vwy0), v_mul(row1, vwy1)),
v_add(v_mul(row2, vwy2), v_mul(row3, vwy3)));
R = v_max(mn, v_min(R, mx));
v_store_aligned(bo, R);
for (int k = 0; k < L; k++) {
outBase[(size_t)(c + k) * yCStride + (size_t)w] = bo[k];
}
}
}
#endif
for (; c < C; c++) {
const float* baseNC = baseN + (size_t)c * xCStride;
float a[4][4];
float minv = FLT_MAX, maxv = -FLT_MAX;
if (interior) {
const float* p = baseNC + (size_t)(y1 - 1) * xHStride + (size_t)(x1 - 1);
for (int j = 0; j < 4; j++) {
a[j][0] = p[0]; a[j][1] = p[1]; a[j][2] = p[2]; a[j][3] = p[3];
p += xHStride;
}
} else {
for (int j = 0; j < 4; j++) {
a[j][0] = gs_fetch2D<PAD>(baseNC, y1 - 1 + j, x1 - 1, H, W, xHStride, align_corners);
a[j][1] = gs_fetch2D<PAD>(baseNC, y1 - 1 + j, x1, H, W, xHStride, align_corners);
a[j][2] = gs_fetch2D<PAD>(baseNC, y1 - 1 + j, x1 + 1, H, W, xHStride, align_corners);
a[j][3] = gs_fetch2D<PAD>(baseNC, y1 - 1 + j, x1 + 2, H, W, xHStride, align_corners);
}
}
float rowv[4];
for (int j = 0; j < 4; j++) {
rowv[j] = a[j][0] * wx[0] + a[j][1] * wx[1] + a[j][2] * wx[2] + a[j][3] * wx[3];
for (int i = 0; i < 4; i++) {
minv = std::min(minv, a[j][i]);
maxv = std::max(maxv, a[j][i]);
}
}
float outv = rowv[0] * wy[0] + rowv[1] * wy[1] + rowv[2] * wy[2] + rowv[3] * wy[3];
outv = std::max(minv, std::min(outv, maxv));
outBase[(size_t)c * yCStride + (size_t)w] = outv;
}
}
}
});
}
template<int PAD>
static inline void bilinear3D(const float* X, const float* G, float* Y,
int N, int C, int D, int H, int W,
int Dout, int Hout, int Wout, bool align_corners)
{
const size_t xNStride = (size_t)C * D * H * W;
const size_t xCStride = (size_t)D * H * W;
const size_t xDStride = (size_t)H * W;
const size_t xHStride = (size_t)W;
const size_t gNStride = (size_t)Dout * Hout * Wout * 3;
const size_t gDStride = (size_t)Hout * Wout * 3;
const size_t gHStride = (size_t)Wout * 3;
const size_t yNStride = (size_t)C * Dout * Hout * Wout;
const size_t yCStride = (size_t)Dout * Hout * Wout;
const size_t yDStride = (size_t)Hout * Wout;
const size_t yHStride = (size_t)Wout;
float xscale, yscale, zscale, xdelta, ydelta, zdelta;
gs_xyz_params(W, H, D, align_corners, xscale, yscale, zscale, xdelta, ydelta, zdelta);
parallel_for_(Range(0, N * Dout * Hout), [&](const Range& r) {
for (int idx = r.start; idx < r.end; idx++) {
int n = idx / (Dout * Hout);
int rem = idx - n * Dout * Hout;
int d = rem / Hout;
int h = rem - d * Hout;
const float* baseN = X + (size_t)n * xNStride;
const float* gRow = G + (size_t)n * gNStride + (size_t)d * gDStride + (size_t)h * gHStride;
float* outBase = Y + (size_t)n * yNStride + (size_t)d * yDStride + (size_t)h * yHStride;
for (int w = 0; w < Wout; w++) {
float xf = gRow[w * 3 + 0] * xscale + xdelta;
float yf = gRow[w * 3 + 1] * yscale + ydelta;
float zf = gRow[w * 3 + 2] * zscale + zdelta;
int x0 = saturate_cast<int>(floorf(xf));
int y0 = saturate_cast<int>(floorf(yf));
int z0 = saturate_cast<int>(floorf(zf));
float dx = xf - x0, dy = yf - y0, dz = zf - z0;
float w000 = (1.f - dx) * (1.f - dy) * (1.f - dz);
float w001 = dx * (1.f - dy) * (1.f - dz);
float w010 = (1.f - dx) * dy * (1.f - dz);
float w011 = dx * dy * (1.f - dz);
float w100 = (1.f - dx) * (1.f - dy) * dz;
float w101 = dx * (1.f - dy) * dz;
float w110 = (1.f - dx) * dy * dz;
float w111 = dx * dy * dz;
bool interior = (x0 >= 0 && y0 >= 0 && z0 >= 0 &&
x0 + 1 < W && y0 + 1 < H && z0 + 1 < D);
int c = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (interior) {
const int L = VTraits<v_float32>::vlanes();
const size_t off000 = (size_t)z0 * xDStride + (size_t)y0 * xHStride + (size_t)x0;
const size_t off001 = off000 + 1;
const size_t off010 = off000 + xHStride;
const size_t off011 = off010 + 1;
const size_t off100 = off000 + xDStride;
const size_t off101 = off100 + 1;
const size_t off110 = off100 + xHStride;
const size_t off111 = off110 + 1;
const v_float32 vw000 = vx_setall_f32(w000);
const v_float32 vw001 = vx_setall_f32(w001);
const v_float32 vw010 = vx_setall_f32(w010);
const v_float32 vw011 = vx_setall_f32(w011);
const v_float32 vw100 = vx_setall_f32(w100);
const v_float32 vw101 = vx_setall_f32(w101);
const v_float32 vw110 = vx_setall_f32(w110);
const v_float32 vw111 = vx_setall_f32(w111);
for (; c + L <= C; c += L) {
float CV_DECL_ALIGNED(32) b000[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b001[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b010[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b011[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b100[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b101[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b110[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) b111[VTraits<v_float32>::max_nlanes];
float CV_DECL_ALIGNED(32) bo[VTraits<v_float32>::max_nlanes];
for (int k = 0; k < L; k++) {
const float* pNC = baseN + (size_t)(c + k) * xCStride;
b000[k] = pNC[off000]; b001[k] = pNC[off001];
b010[k] = pNC[off010]; b011[k] = pNC[off011];
b100[k] = pNC[off100]; b101[k] = pNC[off101];
b110[k] = pNC[off110]; b111[k] = pNC[off111];
}
v_float32 V000 = vx_load_aligned(b000);
v_float32 V001 = vx_load_aligned(b001);
v_float32 V010 = vx_load_aligned(b010);
v_float32 V011 = vx_load_aligned(b011);
v_float32 V100 = vx_load_aligned(b100);
v_float32 V101 = vx_load_aligned(b101);
v_float32 V110 = vx_load_aligned(b110);
v_float32 V111 = vx_load_aligned(b111);
v_float32 R = v_add(
v_add(v_add(v_mul(V000, vw000), v_mul(V001, vw001)),
v_add(v_mul(V010, vw010), v_mul(V011, vw011))),
v_add(v_add(v_mul(V100, vw100), v_mul(V101, vw101)),
v_add(v_mul(V110, vw110), v_mul(V111, vw111))));
v_store_aligned(bo, R);
for (int k = 0; k < L; k++) {
outBase[(size_t)(c + k) * yCStride + (size_t)w] = bo[k];
}
}
}
#endif
for (; c < C; c++) {
const float* baseNC = baseN + (size_t)c * xCStride;
float v000, v001, v010, v011, v100, v101, v110, v111;
if (interior) {
const float* pz0 = baseNC + (size_t)z0 * xDStride;
const float* pz1 = pz0 + xDStride;
const float* p00 = pz0 + (size_t)y0 * xHStride;
const float* p01 = pz0 + (size_t)(y0 + 1) * xHStride;
const float* p10 = pz1 + (size_t)y0 * xHStride;
const float* p11 = pz1 + (size_t)(y0 + 1) * xHStride;
v000 = p00[x0]; v001 = p00[x0 + 1];
v010 = p01[x0]; v011 = p01[x0 + 1];
v100 = p10[x0]; v101 = p10[x0 + 1];
v110 = p11[x0]; v111 = p11[x0 + 1];
} else {
v000 = gs_fetch3D<PAD>(baseNC, z0, y0, x0, D, H, W, xDStride, xHStride, align_corners);
v001 = gs_fetch3D<PAD>(baseNC, z0, y0, x0 + 1, D, H, W, xDStride, xHStride, align_corners);
v010 = gs_fetch3D<PAD>(baseNC, z0, y0 + 1, x0, D, H, W, xDStride, xHStride, align_corners);
v011 = gs_fetch3D<PAD>(baseNC, z0, y0 + 1, x0 + 1, D, H, W, xDStride, xHStride, align_corners);
v100 = gs_fetch3D<PAD>(baseNC, z0 + 1, y0, x0, D, H, W, xDStride, xHStride, align_corners);
v101 = gs_fetch3D<PAD>(baseNC, z0 + 1, y0, x0 + 1, D, H, W, xDStride, xHStride, align_corners);
v110 = gs_fetch3D<PAD>(baseNC, z0 + 1, y0 + 1, x0, D, H, W, xDStride, xHStride, align_corners);
v111 = gs_fetch3D<PAD>(baseNC, z0 + 1, y0 + 1, x0 + 1, D, H, W, xDStride, xHStride, align_corners);
}
outBase[(size_t)c * yCStride + (size_t)w] =
w000 * v000 + w001 * v001 + w010 * v010 + w011 * v011 +
w100 * v100 + w101 * v101 + w110 * v110 + w111 * v111;
}
}
}
});
}
template<int PAD>
static inline void nearest3D(const float* X, const float* G, float* Y,
int N, int C, int D, int H, int W,
int Dout, int Hout, int Wout, bool align_corners)
{
const size_t xNStride = (size_t)C * D * H * W;
const size_t xCStride = (size_t)D * H * W;
const size_t xDStride = (size_t)H * W;
const size_t xHStride = (size_t)W;
const size_t gNStride = (size_t)Dout * Hout * Wout * 3;
const size_t gDStride = (size_t)Hout * Wout * 3;
const size_t gHStride = (size_t)Wout * 3;
const size_t yNStride = (size_t)C * Dout * Hout * Wout;
const size_t yCStride = (size_t)Dout * Hout * Wout;
const size_t yDStride = (size_t)Hout * Wout;
const size_t yHStride = (size_t)Wout;
float xscale, yscale, zscale, xdelta, ydelta, zdelta;
gs_xyz_params(W, H, D, align_corners, xscale, yscale, zscale, xdelta, ydelta, zdelta);
parallel_for_(Range(0, N * Dout * Hout), [&](const Range& r) {
for (int idx = r.start; idx < r.end; idx++) {
int n = idx / (Dout * Hout);
int rem = idx - n * Dout * Hout;
int d = rem / Hout;
int h = rem - d * Hout;
const float* baseN = X + (size_t)n * xNStride;
const float* gRow = G + (size_t)n * gNStride + (size_t)d * gDStride + (size_t)h * gHStride;
float* outBase = Y + (size_t)n * yNStride + (size_t)d * yDStride + (size_t)h * yHStride;
for (int w = 0; w < Wout; w++) {
float xf = gRow[w * 3 + 0] * xscale + xdelta;
float yf = gRow[w * 3 + 1] * yscale + ydelta;
float zf = gRow[w * 3 + 2] * zscale + zdelta;
int xi = cvRound(xf), yi = cvRound(yf), zi = cvRound(zf);
bool interior = (xi >= 0 && yi >= 0 && zi >= 0 && xi < W && yi < H && zi < D);
int c = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (interior) {
const int L = VTraits<v_float32>::vlanes();
const size_t off = (size_t)zi * xDStride + (size_t)yi * xHStride + (size_t)xi;
for (; c + L <= C; c += L) {
float CV_DECL_ALIGNED(32) b[VTraits<v_float32>::max_nlanes];
for (int k = 0; k < L; k++) {
b[k] = baseN[(size_t)(c + k) * xCStride + off];
}
for (int k = 0; k < L; k++) {
outBase[(size_t)(c + k) * yCStride + (size_t)w] = b[k];
}
}
}
#endif
for (; c < C; c++) {
const float* baseNC = baseN + (size_t)c * xCStride;
float v;
if (interior) v = baseNC[(size_t)zi * xDStride + (size_t)yi * xHStride + (size_t)xi];
else v = gs_fetch3D<PAD>(baseNC, zi, yi, xi, D, H, W, xDStride, xHStride, align_corners);
outBase[(size_t)c * yCStride + (size_t)w] = v;
}
}
}
});
}
} // namespace gs_simd_internal
void gridSampleBilinear2D_f32_(const float* X, const float* G, float* Y,
int N, int C, int H, int W, int Hout, int Wout,
bool align_corners, int padding)
{
using namespace gs_simd_internal;
if (padding == GS_ZEROS) bilinear2D<GS_ZEROS>(X, G, Y, N, C, H, W, Hout, Wout, align_corners);
else if (padding == GS_BORDER) bilinear2D<GS_BORDER>(X, G, Y, N, C, H, W, Hout, Wout, align_corners);
else bilinear2D<GS_REFLECTION>(X, G, Y, N, C, H, W, Hout, Wout, align_corners);
}
void gridSampleNearest2D_f32_(const float* X, const float* G, float* Y,
int N, int C, int H, int W, int Hout, int Wout,
bool align_corners, int padding)
{
using namespace gs_simd_internal;
if (padding == GS_ZEROS) nearest2D<GS_ZEROS>(X, G, Y, N, C, H, W, Hout, Wout, align_corners);
else if (padding == GS_BORDER) nearest2D<GS_BORDER>(X, G, Y, N, C, H, W, Hout, Wout, align_corners);
else nearest2D<GS_REFLECTION>(X, G, Y, N, C, H, W, Hout, Wout, align_corners);
}
void gridSampleBicubic2D_f32_(const float* X, const float* G, float* Y,
int N, int C, int H, int W, int Hout, int Wout,
bool align_corners, int padding, float cubic_alpha)
{
using namespace gs_simd_internal;
if (padding == GS_ZEROS) bicubic2D<GS_ZEROS>(X, G, Y, N, C, H, W, Hout, Wout, align_corners, cubic_alpha);
else if (padding == GS_BORDER) bicubic2D<GS_BORDER>(X, G, Y, N, C, H, W, Hout, Wout, align_corners, cubic_alpha);
else bicubic2D<GS_REFLECTION>(X, G, Y, N, C, H, W, Hout, Wout, align_corners, cubic_alpha);
}
void gridSampleBilinear3D_f32_(const float* X, const float* G, float* Y,
int N, int C, int D, int H, int W, int Dout, int Hout, int Wout,
bool align_corners, int padding)
{
using namespace gs_simd_internal;
if (padding == GS_ZEROS) bilinear3D<GS_ZEROS>(X, G, Y, N, C, D, H, W, Dout, Hout, Wout, align_corners);
else if (padding == GS_BORDER) bilinear3D<GS_BORDER>(X, G, Y, N, C, D, H, W, Dout, Hout, Wout, align_corners);
else bilinear3D<GS_REFLECTION>(X, G, Y, N, C, D, H, W, Dout, Hout, Wout, align_corners);
}
void gridSampleNearest3D_f32_(const float* X, const float* G, float* Y,
int N, int C, int D, int H, int W, int Dout, int Hout, int Wout,
bool align_corners, int padding)
{
using namespace gs_simd_internal;
if (padding == GS_ZEROS) nearest3D<GS_ZEROS>(X, G, Y, N, C, D, H, W, Dout, Hout, Wout, align_corners);
else if (padding == GS_BORDER) nearest3D<GS_BORDER>(X, G, Y, N, C, D, H, W, Dout, Hout, Wout, align_corners);
else nearest3D<GS_REFLECTION>(X, G, Y, N, C, D, H, W, Dout, Hout, Wout, align_corners);
}
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // cv::dnn
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
modules/dnn/src/layers/cpu_kernels/nary_eltwise_kernels.simd.hpp
+121
View File
@@ -0,0 +1,121 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include <opencv2/core.hpp>
#include "opencv2/core/hal/intrin.hpp"
namespace cv { namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
// Op codes that simd_binop_f32_ understands.
enum SimdBinOp { SIMD_BIN_ADD = 0, SIMD_BIN_SUB = 1, SIMD_BIN_MUL = 2, SIMD_BIN_DIV = 3 };
// Apply binary op on n contiguous floats.
int simd_binop_f32_(const float* a, const float* b, float* out, int n, int op);
CV_CPU_OPTIMIZATION_NAMESPACE_END
}}
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
namespace cv { namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
int simd_binop_f32_(const float* a, const float* b, float* out, int n, int op) {
int i = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int lanes = VTraits<v_float32>::vlanes();
if (op == SIMD_BIN_ADD) {
for (; i <= n - lanes * 4; i += lanes * 4) {
v_float32 a0 = vx_load(a + i), b0 = vx_load(b + i);
v_float32 a1 = vx_load(a + i + lanes), b1 = vx_load(b + i + lanes);
v_float32 a2 = vx_load(a + i + lanes * 2), b2 = vx_load(b + i + lanes * 2);
v_float32 a3 = vx_load(a + i + lanes * 3), b3 = vx_load(b + i + lanes * 3);
v_float32 r0 = v_add(a0, b0), r1 = v_add(a1, b1);
v_float32 r2 = v_add(a2, b2), r3 = v_add(a3, b3);
vx_store(out + i, r0); vx_store(out + i + lanes, r1);
vx_store(out + i + lanes * 2, r2); vx_store(out + i + lanes * 3, r3);
}
for (; i <= n - lanes; i += lanes)
vx_store(out + i, v_add(vx_load(a + i), vx_load(b + i)));
if (i < n && n >= lanes) {
i = n - lanes;
vx_store(out + i, v_add(vx_load(a + i), vx_load(b + i)));
i = n;
}
} else if (op == SIMD_BIN_MUL) {
for (; i <= n - lanes * 4; i += lanes * 4) {
v_float32 a0 = vx_load(a + i), b0 = vx_load(b + i);
v_float32 a1 = vx_load(a + i + lanes), b1 = vx_load(b + i + lanes);
v_float32 a2 = vx_load(a + i + lanes * 2), b2 = vx_load(b + i + lanes * 2);
v_float32 a3 = vx_load(a + i + lanes * 3), b3 = vx_load(b + i + lanes * 3);
v_float32 r0 = v_mul(a0, b0), r1 = v_mul(a1, b1);
v_float32 r2 = v_mul(a2, b2), r3 = v_mul(a3, b3);
vx_store(out + i, r0); vx_store(out + i + lanes, r1);
vx_store(out + i + lanes * 2, r2); vx_store(out + i + lanes * 3, r3);
}
for (; i <= n - lanes; i += lanes)
vx_store(out + i, v_mul(vx_load(a + i), vx_load(b + i)));
if (i < n && n >= lanes) {
i = n - lanes;
vx_store(out + i, v_mul(vx_load(a + i), vx_load(b + i)));
i = n;
}
} else if (op == SIMD_BIN_SUB) {
for (; i <= n - lanes * 4; i += lanes * 4) {
v_float32 a0 = vx_load(a + i), b0 = vx_load(b + i);
v_float32 a1 = vx_load(a + i + lanes), b1 = vx_load(b + i + lanes);
v_float32 a2 = vx_load(a + i + lanes * 2), b2 = vx_load(b + i + lanes * 2);
v_float32 a3 = vx_load(a + i + lanes * 3), b3 = vx_load(b + i + lanes * 3);
v_float32 r0 = v_sub(a0, b0), r1 = v_sub(a1, b1);
v_float32 r2 = v_sub(a2, b2), r3 = v_sub(a3, b3);
vx_store(out + i, r0); vx_store(out + i + lanes, r1);
vx_store(out + i + lanes * 2, r2); vx_store(out + i + lanes * 3, r3);
}
for (; i <= n - lanes; i += lanes)
vx_store(out + i, v_sub(vx_load(a + i), vx_load(b + i)));
if (i < n && n >= lanes) {
i = n - lanes;
vx_store(out + i, v_sub(vx_load(a + i), vx_load(b + i)));
i = n;
}
} else if (op == SIMD_BIN_DIV) {
for (; i <= n - lanes * 4; i += lanes * 4) {
v_float32 a0 = vx_load(a + i), b0 = vx_load(b + i);
v_float32 a1 = vx_load(a + i + lanes), b1 = vx_load(b + i + lanes);
v_float32 a2 = vx_load(a + i + lanes * 2), b2 = vx_load(b + i + lanes * 2);
v_float32 a3 = vx_load(a + i + lanes * 3), b3 = vx_load(b + i + lanes * 3);
v_float32 r0 = v_div(a0, b0), r1 = v_div(a1, b1);
v_float32 r2 = v_div(a2, b2), r3 = v_div(a3, b3);
vx_store(out + i, r0); vx_store(out + i + lanes, r1);
vx_store(out + i + lanes * 2, r2); vx_store(out + i + lanes * 3, r3);
}
for (; i <= n - lanes; i += lanes)
vx_store(out + i, v_div(vx_load(a + i), vx_load(b + i)));
if (i < n && n >= lanes) {
i = n - lanes;
vx_store(out + i, v_div(vx_load(a + i), vx_load(b + i)));
i = n;
}
}
vx_cleanup();
#endif
if (op == SIMD_BIN_ADD) {
for (; i < n; i++) out[i] = a[i] + b[i];
} else if (op == SIMD_BIN_MUL) {
for (; i < n; i++) out[i] = a[i] * b[i];
} else if (op == SIMD_BIN_SUB) {
for (; i < n; i++) out[i] = a[i] - b[i];
} else {
for (; i < n; i++) out[i] = a[i] / b[i];
}
return n;
}
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // cv::dnn
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
modules/dnn/src/layers/cpu_kernels/reduce2_kernels.simd.hpp
+299
View File
@@ -0,0 +1,299 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include <opencv2/core.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/dnn/all_layers.hpp>
#include "opencv2/core/hal/intrin.hpp"
#include <cfloat>
#include <cmath>
#include <vector>
#include <algorithm>
namespace cv { namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
// Reduce-all over a contiguous CV_32F src
void reduceAllFloatParallel_(const Mat& src, Mat& dst, int reduce_type);
// Reduce a contiguous trailing block of axes for CV_32F src.
void reduceLastAxesFloatParallel_(const Mat& src, Mat& dst, size_t innerLen, int reduce_type);
CV_CPU_OPTIMIZATION_NAMESPACE_END
}}
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
namespace cv { namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
void reduceAllFloatParallel_(const Mat& src, Mat& dst, int reduce_type_int) {
const Reduce2Layer::ReduceType rt = (Reduce2Layer::ReduceType)reduce_type_int;
const float* p = src.ptr<const float>();
const size_t total = src.total();
const int nThreads = std::max(1, cv::getNumThreads());
const int stripes = std::max(1, std::min<int>(
(int)((total + 16383) / 16384), nThreads));
float init_f = 0.0f;
if (rt == Reduce2Layer::ReduceType::MAX) init_f = -FLT_MAX;
else if (rt == Reduce2Layer::ReduceType::MIN) init_f = FLT_MAX;
else if (rt == Reduce2Layer::ReduceType::PROD) init_f = 1.0f;
std::vector<float> partial_f(stripes, init_f);
parallel_for_(Range(0, stripes), [&](const Range& r) {
for (int s = r.start; s < r.end; s++) {
size_t start = (size_t)s * total / stripes;
size_t end = (size_t)(s + 1) * total / stripes;
const float* q = p + start;
size_t n = end - start, i = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int L = VTraits<v_float32>::vlanes();
#endif
switch (rt) {
case Reduce2Layer::ReduceType::MAX: {
float acc = -FLT_MAX;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setall_f32(-FLT_MAX);
for (; i + L <= n; i += L) va = v_max(va, vx_load(q + i));
acc = v_reduce_max(va);
#endif
for (; i < n; i++) acc = acc > q[i] ? acc : q[i];
partial_f[s] = acc;
break;
}
case Reduce2Layer::ReduceType::MIN: {
float acc = FLT_MAX;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setall_f32(FLT_MAX);
for (; i + L <= n; i += L) va = v_min(va, vx_load(q + i));
acc = v_reduce_min(va);
#endif
for (; i < n; i++) acc = acc < q[i] ? acc : q[i];
partial_f[s] = acc;
break;
}
case Reduce2Layer::ReduceType::SUM:
case Reduce2Layer::ReduceType::MEAN:
case Reduce2Layer::ReduceType::LOG_SUM: {
float acc = 0.0f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setzero_f32();
for (; i + L <= n; i += L) va = v_add(va, vx_load(q + i));
acc += v_reduce_sum(va);
#endif
for (; i < n; i++) acc += q[i];
partial_f[s] = acc;
break;
}
case Reduce2Layer::ReduceType::L1: {
float acc = 0.0f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setzero_f32();
for (; i + L <= n; i += L) va = v_add(va, v_abs(vx_load(q + i)));
acc += v_reduce_sum(va);
#endif
for (; i < n; i++) acc += std::fabs(q[i]);
partial_f[s] = acc;
break;
}
case Reduce2Layer::ReduceType::L2:
case Reduce2Layer::ReduceType::SUM_SQUARE: {
float acc = 0.0f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setzero_f32();
for (; i + L <= n; i += L) {
v_float32 x = vx_load(q + i);
va = v_add(va, v_mul(x, x));
}
acc += v_reduce_sum(va);
#endif
for (; i < n; i++) { float x = q[i]; acc += x * x; }
partial_f[s] = acc;
break;
}
case Reduce2Layer::ReduceType::PROD: {
float acc = 1.0f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 va = vx_setall_f32(1.0f);
for (; i + L <= n; i += L) va = v_mul(va, vx_load(q + i));
float buf[VTraits<v_float32>::max_nlanes];
v_store(buf, va);
for (int k = 0; k < L; k++) acc *= buf[k];
#endif
for (; i < n; i++) acc *= q[i];
partial_f[s] = acc;
break;
}
default: CV_Error(Error::StsInternal, "reduceAllFloatParallel_: unhandled type");
}
}
});
float* out = dst.ptr<float>();
switch (rt) {
case Reduce2Layer::ReduceType::MAX: {
float m = -FLT_MAX;
for (int s = 0; s < stripes; s++) m = m > partial_f[s] ? m : partial_f[s];
*out = m;
break;
}
case Reduce2Layer::ReduceType::MIN: {
float m = FLT_MAX;
for (int s = 0; s < stripes; s++) m = m < partial_f[s] ? m : partial_f[s];
*out = m;
break;
}
case Reduce2Layer::ReduceType::SUM: {
float acc = 0.0f; for (int s = 0; s < stripes; s++) acc += partial_f[s];
*out = acc;
break;
}
case Reduce2Layer::ReduceType::MEAN: {
float acc = 0.0f; for (int s = 0; s < stripes; s++) acc += partial_f[s];
*out = total > 0 ? (float)((double)acc / (double)total) : 0.0f;
break;
}
case Reduce2Layer::ReduceType::L1:
case Reduce2Layer::ReduceType::SUM_SQUARE: {
float acc = 0.0f; for (int s = 0; s < stripes; s++) acc += partial_f[s];
*out = acc;
break;
}
case Reduce2Layer::ReduceType::L2: {
float acc = 0.0f; for (int s = 0; s < stripes; s++) acc += partial_f[s];
*out = std::sqrt(acc);
break;
}
case Reduce2Layer::ReduceType::LOG_SUM: {
float acc = 0.0f; for (int s = 0; s < stripes; s++) acc += partial_f[s];
*out = total > 0 ? std::log(acc) : -std::numeric_limits<float>::infinity();
break;
}
case Reduce2Layer::ReduceType::PROD: {
float acc = 1.0f; for (int s = 0; s < stripes; s++) acc *= partial_f[s];
*out = acc;
break;
}
default: CV_Error(Error::StsInternal, "reduceAllFloatParallel_: unhandled type");
}
}
void reduceLastAxesFloatParallel_(const Mat& src, Mat& dst, size_t innerLen, int reduce_type_int) {
const Reduce2Layer::ReduceType rt = (Reduce2Layer::ReduceType)reduce_type_int;
const float* p = src.ptr<const float>();
float* q = dst.ptr<float>();
const size_t nOut = src.total() / innerLen;
const float inv_inner = innerLen > 0 ? 1.0f / (float)innerLen : 0.0f;
parallel_for_(Range(0, (int)nOut), [&](const Range& r) {
for (int row = r.start; row < r.end; row++) {
const float* s0 = p + (size_t)row * innerLen;
size_t i = 0, n = innerLen;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int L = VTraits<v_float32>::vlanes();
#endif
switch (rt) {
case Reduce2Layer::ReduceType::MAX: {
float acc = -FLT_MAX;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (n >= (size_t)L) {
v_float32 va = vx_setall_f32(-FLT_MAX);
for (; i + L <= n; i += L) va = v_max(va, vx_load(s0 + i));
acc = v_reduce_max(va);
}
#endif
for (; i < n; i++) acc = acc > s0[i] ? acc : s0[i];
q[row] = acc;
break;
}
case Reduce2Layer::ReduceType::MIN: {
float acc = FLT_MAX;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (n >= (size_t)L) {
v_float32 va = vx_setall_f32(FLT_MAX);
for (; i + L <= n; i += L) va = v_min(va, vx_load(s0 + i));
acc = v_reduce_min(va);
}
#endif
for (; i < n; i++) acc = acc < s0[i] ? acc : s0[i];
q[row] = acc;
break;
}
case Reduce2Layer::ReduceType::SUM:
case Reduce2Layer::ReduceType::MEAN:
case Reduce2Layer::ReduceType::LOG_SUM: {
float acc = 0.0f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (n >= (size_t)L) {
v_float32 va = vx_setzero_f32();
for (; i + L <= n; i += L) va = v_add(va, vx_load(s0 + i));
acc += v_reduce_sum(va);
}
#endif
for (; i < n; i++) acc += s0[i];
if (rt == Reduce2Layer::ReduceType::MEAN) q[row] = acc * inv_inner;
else if (rt == Reduce2Layer::ReduceType::LOG_SUM) q[row] = std::log(acc);
else q[row] = acc;
break;
}
case Reduce2Layer::ReduceType::L1: {
float acc = 0.0f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (n >= (size_t)L) {
v_float32 va = vx_setzero_f32();
for (; i + L <= n; i += L) va = v_add(va, v_abs(vx_load(s0 + i)));
acc += v_reduce_sum(va);
}
#endif
for (; i < n; i++) acc += std::fabs(s0[i]);
q[row] = acc;
break;
}
case Reduce2Layer::ReduceType::L2:
case Reduce2Layer::ReduceType::SUM_SQUARE: {
float acc = 0.0f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (n >= (size_t)L) {
v_float32 va = vx_setzero_f32();
for (; i + L <= n; i += L) {
v_float32 x = vx_load(s0 + i);
va = v_add(va, v_mul(x, x));
}
acc += v_reduce_sum(va);
}
#endif
for (; i < n; i++) { float x = s0[i]; acc += x * x; }
q[row] = (rt == Reduce2Layer::ReduceType::L2) ? std::sqrt(acc) : acc;
break;
}
case Reduce2Layer::ReduceType::PROD: {
float acc = 1.0f;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (n >= (size_t)L) {
v_float32 va = vx_setall_f32(1.0f);
for (; i + L <= n; i += L) va = v_mul(va, vx_load(s0 + i));
float buf[VTraits<v_float32>::max_nlanes];
v_store(buf, va);
for (int k = 0; k < L; k++) acc *= buf[k];
}
#endif
for (; i < n; i++) acc *= s0[i];
q[row] = acc;
break;
}
default: CV_Error(Error::StsInternal, "reduceLastAxesFloatParallel_: unhandled type");
}
}
}, (double)nOut * (double)innerLen / 16384.0);
}
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // cv::dnn
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
modules/dnn/src/layers/cpu_kernels/softmax.cpp
+9 -126
View File
@@ -11,138 +11,21 @@
#include "../../precomp.hpp"
#include "softmax.hpp"
#include "opencv2/core/fast_math.hpp"
#define CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
#include "activation_kernels.simd.hpp"
#include "layers/cpu_kernels/activation_kernels.simd_declarations.hpp"
#undef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
namespace cv { namespace dnn {
void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep, float scale){
CV_Assert(src.type() == CV_32F);
CV_Assert(src.isContinuous() && dst.isContinuous());
CV_Assert(src.size == dst.size);
axis = normalize_axis(axis, src.dims);
const bool scaled = scale != 1.f;
size_t outerSize = src.total(0, axis),
innerSize = src.total(axis + 1);
const float *srcPtr = src.ptr<float>();
float *dstPtr = dst.ptr<float>();
size_t outerStep = src.total(axis);
size_t cnStep = src.total(axis + 1);
// multi-threads
size_t totalTasks = outerSize * innerSize;
double nstripes = (double) totalTasks / 1024.0;
// make the channel axis to be multiple of 8
size_t channelAxis = (axisStep + 7) & -8;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int nlanes = VTraits<v_float32>::vlanes();
#endif
parallel_for_(Range(0, (int) totalTasks), [&](const Range &range) {
AutoBuffer<float> axisBuf_(channelAxis);
float *axisBuf = axisBuf_.data();
for (size_t i = range.start; i < range.end; i++) {
size_t outerDim = i / innerSize;
size_t innerDim = i % innerSize;
size_t srcOffset = outerDim * outerStep + innerDim;
// copy data from src to buf along axis, since the data may not be continuous
size_t _cnDim = 0;
#if CV_ENABLE_UNROLLED && defined(_M_ARM64)
for (; _cnDim + 3 < axisStep; _cnDim += 4) {
axisBuf[_cnDim + 0] = srcPtr[srcOffset + (_cnDim + 0 + axisBias) * cnStep];
axisBuf[_cnDim + 1] = srcPtr[srcOffset + (_cnDim + 1 + axisBias) * cnStep];
axisBuf[_cnDim + 2] = srcPtr[srcOffset + (_cnDim + 2 + axisBias) * cnStep];
axisBuf[_cnDim + 3] = srcPtr[srcOffset + (_cnDim + 3 + axisBias) * cnStep];
}
#endif
for (; _cnDim < axisStep; _cnDim++)
axisBuf[_cnDim] = srcPtr[srcOffset + (_cnDim + axisBias) * cnStep];
// Bake the fused pre-Softmax scale into the buffer once. axisBuf
// already lives in L1 from the load above, so this pass is short.
if (scaled) {
int sk = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
v_float32 vscale = vx_setall_f32(scale);
for (; sk <= axisStep - nlanes; sk += nlanes) {
v_float32 val = vx_load(axisBuf + sk);
val = v_mul(val, vscale);
v_store(axisBuf + sk, val);
}
#endif
for (; sk < axisStep; sk++)
axisBuf[sk] *= scale;
}
float maxVal = -FLT_MAX;
int cnDim = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
// calculate the max value along the axis
v_float32 vmax = vx_setall_f32(-FLT_MAX);
for (; cnDim < axisStep; cnDim += nlanes) {
if (cnDim > axisStep - nlanes) {
if (cnDim == 0) { break; }
cnDim = axisStep - nlanes;
}
v_float32 val = vx_load(axisBuf + cnDim);
vmax = v_max(vmax, val);
}
maxVal = v_reduce_max(vmax);
#endif
for (; cnDim < axisStep; cnDim++) {
maxVal = std::max(maxVal, axisBuf[cnDim]);
}
float s = 0.f;
cnDim = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
// calculate the exp value along the axis
v_float32 vs = vx_setzero_f32();
vmax = vx_setall_f32(maxVal);
// calculate and sum all data along axis
for (; cnDim <= axisStep - nlanes; cnDim += nlanes) {
// cannot apply halide trick here due to axisBuf is constantly updated
v_float32 val = vx_load(axisBuf + cnDim);
val = v_sub(val, vmax);
val = v_exp(val);
vs = v_add(vs, val);
v_store(axisBuf + cnDim, val);
}
s = v_reduce_sum(vs);
#endif
for (; cnDim < axisStep; cnDim++) {
axisBuf[cnDim] = expf(axisBuf[cnDim] - maxVal);
s += axisBuf[cnDim];
}
// copy back the result to src
_cnDim = 0;
if (s == 0.f || cvIsInf(1.f / s)) {
for (; _cnDim < axisStep; _cnDim++)
dstPtr[srcOffset + (_cnDim + axisBias) * cnStep] = 0.f;
} else {
s = 1.f / s;
#if CV_ENABLE_UNROLLED && defined(_M_ARM64)
for (; _cnDim + 3 < axisStep; _cnDim += 4) {
dstPtr[srcOffset + (_cnDim + 0 + axisBias) * cnStep] = axisBuf[_cnDim + 0] * s;
dstPtr[srcOffset + (_cnDim + 1 + axisBias) * cnStep] = axisBuf[_cnDim + 1] * s;
dstPtr[srcOffset + (_cnDim + 2 + axisBias) * cnStep] = axisBuf[_cnDim + 2] * s;
dstPtr[srcOffset + (_cnDim + 3 + axisBias) * cnStep] = axisBuf[_cnDim + 3] * s;
}
#endif
for (; _cnDim < axisStep; _cnDim++)
dstPtr[srcOffset + (_cnDim + axisBias) * cnStep] = axisBuf[_cnDim] * s;
}
}
}, nstripes);
void softmax(Mat &dst, const Mat &src, int axis, int axisBias, int axisStep, float scale) {
CV_CPU_DISPATCH(softmax_, (dst, src, axis, axisBias, axisStep, scale),
CV_CPU_DISPATCH_MODES_ALL);
}
void softmax(Mat &dst, const Mat &src, int axis) {
softmax(dst, src, axis, 0, src.size[axis]);
softmax(dst, src, axis, 0, src.size[axis], 1.f);
}
void softmax(Mat &dst, const Mat &src, int axis, float scale) {
modules/dnn/src/layers/cpu_kernels/transpose_kernels.simd.hpp
+123
View File
@@ -0,0 +1,123 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include <opencv2/core.hpp>
#include <opencv2/core/utility.hpp>
#include "opencv2/core/hal/intrin.hpp"
#include <cstdint>
#include <algorithm>
namespace cv { namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
// Blocked 2D float transpose.
void transpose2D_f32_(const float* inp, float* out,
int64_t outer, int64_t rows, int64_t cols);
CV_CPU_OPTIMIZATION_NAMESPACE_END
}}
#ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
namespace cv { namespace dnn {
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
#if (CV_SIMD || CV_SIMD_SCALABLE)
// AVX2 8x8 f32 in-register transpose. v_transpose4x4 transposes the two
// 128-bit halves independently; v_combine_low/high then exchange halves so
// element k of each target row comes from the right source row.
static inline void transpose_8x8_f32_avx2_(const float* src, int64_t src_stride,
float* dst, int64_t dst_stride)
{
v_float32 r0 = vx_load(src + 0 * src_stride);
v_float32 r1 = vx_load(src + 1 * src_stride);
v_float32 r2 = vx_load(src + 2 * src_stride);
v_float32 r3 = vx_load(src + 3 * src_stride);
v_float32 r4 = vx_load(src + 4 * src_stride);
v_float32 r5 = vx_load(src + 5 * src_stride);
v_float32 r6 = vx_load(src + 6 * src_stride);
v_float32 r7 = vx_load(src + 7 * src_stride);
v_float32 a0, a1, a2, a3, a4, a5, a6, a7;
v_transpose4x4(r0, r1, r2, r3, a0, a1, a2, a3);
v_transpose4x4(r4, r5, r6, r7, a4, a5, a6, a7);
v_float32 b0 = v_combine_low (a0, a4);
v_float32 b1 = v_combine_low (a1, a5);
v_float32 b2 = v_combine_low (a2, a6);
v_float32 b3 = v_combine_low (a3, a7);
v_float32 b4 = v_combine_high(a0, a4);
v_float32 b5 = v_combine_high(a1, a5);
v_float32 b6 = v_combine_high(a2, a6);
v_float32 b7 = v_combine_high(a3, a7);
v_store(dst + 0 * dst_stride, b0);
v_store(dst + 1 * dst_stride, b1);
v_store(dst + 2 * dst_stride, b2);
v_store(dst + 3 * dst_stride, b3);
v_store(dst + 4 * dst_stride, b4);
v_store(dst + 5 * dst_stride, b5);
v_store(dst + 6 * dst_stride, b6);
v_store(dst + 7 * dst_stride, b7);
}
#endif
void transpose2D_f32_(const float* inp, float* out,
int64_t outer, int64_t rows, int64_t cols)
{
const int TILE = 32;
int64_t batchStride = rows * cols;
int64_t rtiles = (rows + TILE - 1) / TILE;
int64_t ctiles = (cols + TILE - 1) / TILE;
int64_t total = outer * rtiles * ctiles;
parallel_for_(Range(0, (int)total), [&](const Range& range) {
for (int64_t idx = range.start; idx < range.end; idx++) {
int64_t rem = idx;
int64_t b = rem / (rtiles * ctiles);
rem -= b * rtiles * ctiles;
int64_t ti = rem / ctiles;
int64_t tj = rem - ti * ctiles;
int64_t r0 = ti * TILE, r1 = std::min(r0 + TILE, rows);
int64_t c0 = tj * TILE, c1 = std::min(c0 + TILE, cols);
const float* inB = inp + b * batchStride;
float* outB = out + b * batchStride;
#if (CV_SIMD || CV_SIMD_SCALABLE)
if (VTraits<v_float32>::vlanes() == 8) {
int64_t r = r0;
for (; r + 8 <= r1; r += 8) {
int64_t c = c0;
for (; c + 8 <= c1; c += 8) {
transpose_8x8_f32_avx2_(inB + c * rows + r, rows,
outB + r * cols + c, cols);
}
for (; c < c1; c++) {
for (int k = 0; k < 8; k++)
outB[(r + k) * cols + c] = inB[c * rows + (r + k)];
}
}
for (; r < r1; r++) {
for (int64_t c = c0; c < c1; c++)
outB[r * cols + c] = inB[c * rows + r];
}
continue;
}
#endif
for (int64_t r = r0; r < r1; r++) {
for (int64_t c = c0; c < c1; c++)
outB[r * cols + c] = inB[c * rows + r];
}
}
});
}
CV_CPU_OPTIMIZATION_NAMESPACE_END
}} // cv::dnn
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
modules/dnn/src/layers/gridsample_layer.cpp
+44
View File
@@ -5,6 +5,14 @@
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
// Dispatch infrastructure for the SIMD GridSample kernels.
#include "cpu_kernels/gridsample_kernels.simd.hpp"
#include "layers/cpu_kernels/gridsample_kernels.simd_declarations.hpp"
#define CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN namespace cpu_baseline {
#define CV_CPU_OPTIMIZATION_NAMESPACE_END }
#undef CV_CPU_DISPATCH_MODES_ALL
#include "layers_common.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include <opencv2/core/utility.hpp>
@@ -234,6 +242,7 @@ static inline void gridSampleComputeRows(
});
}
template<typename T>
static inline void gridSampleDispatch(
const T* Xptr,
@@ -245,6 +254,24 @@ static inline void gridSampleDispatch(
int mode, int padding,
float cubic_alpha)
{
if (std::is_same<T, float>::value) {
const float* X = reinterpret_cast<const float*>(Xptr);
float* Y = reinterpret_cast<float*>(Yptr);
if (mode == M_NEAREST) {
CV_CPU_DISPATCH(gridSampleNearest2D_f32_,
(X, Gptr, Y, N, C, H, W, Hout, Wout, align_corners, padding),
NEON, AVX2, AVX, BASELINE);
} else if (mode == M_BILINEAR) {
CV_CPU_DISPATCH(gridSampleBilinear2D_f32_,
(X, Gptr, Y, N, C, H, W, Hout, Wout, align_corners, padding),
NEON, AVX2, AVX, BASELINE);
} else {
CV_CPU_DISPATCH(gridSampleBicubic2D_f32_,
(X, Gptr, Y, N, C, H, W, Hout, Wout, align_corners, padding, cubic_alpha),
NEON, AVX2, AVX, BASELINE);
}
return;
}
if (mode == M_NEAREST) {
if (padding == P_ZEROS) gridSampleComputeRows<T, M_NEAREST, P_ZEROS>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, cubic_alpha);
else if (padding == P_BORDER) gridSampleComputeRows<T, M_NEAREST, P_BORDER>(Xptr, Gptr, Yptr, N, C, H, W, Hout, Wout, align_corners, cubic_alpha);
@@ -387,6 +414,7 @@ static inline void gridSampleCompute3D(
});
}
template<typename T>
static inline void gridSampleDispatch3D(
const T* Xptr,
@@ -397,6 +425,22 @@ static inline void gridSampleDispatch3D(
bool align_corners,
int mode, int padding)
{
if (std::is_same<T, float>::value) {
const float* X = reinterpret_cast<const float*>(Xptr);
float* Y = reinterpret_cast<float*>(Yptr);
if (mode == M_NEAREST) {
CV_CPU_DISPATCH(gridSampleNearest3D_f32_,
(X, Gptr, Y, N, C, D, H, W, Dout, Hout, Wout, align_corners, padding),
NEON, AVX2, AVX, BASELINE);
} else if (mode == M_BILINEAR) {
CV_CPU_DISPATCH(gridSampleBilinear3D_f32_,
(X, Gptr, Y, N, C, D, H, W, Dout, Hout, Wout, align_corners, padding),
NEON, AVX2, AVX, BASELINE);
} else {
CV_Error(Error::StsNotImplemented, "GridSample bicubic mode is not supported for 5D inputs");
}
return;
}
if (mode == M_NEAREST) {
if (padding == P_ZEROS) gridSampleCompute3D<T, M_NEAREST, P_ZEROS>(Xptr, Gptr, Yptr, N, C, D, H, W, Dout, Hout, Wout, align_corners);
else if (padding == P_BORDER) gridSampleCompute3D<T, M_NEAREST, P_BORDER>(Xptr, Gptr, Yptr, N, C, D, H, W, Dout, Hout, Wout, align_corners);
modules/dnn/src/layers/nary_eltwise_layers.cpp
+94 -76
View File
@@ -3,6 +3,13 @@
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include "cpu_kernels/nary_eltwise_kernels.simd.hpp"
#include "layers/cpu_kernels/nary_eltwise_kernels.simd_declarations.hpp"
#define CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN namespace cpu_baseline {
#define CV_CPU_OPTIMIZATION_NAMESPACE_END }
#undef CV_CPU_DISPATCH_MODES_ALL
#include "../net_impl.hpp"
#include "layers_common.hpp"
#include "../op_cuda.hpp"
@@ -36,6 +43,13 @@ static int _mod(int x, int y) {
return res;
}
// Wrapper that turns the CV_CPU_DISPATCH return-chain into a normal call.
static inline int simd_binop_f32_dispatch(const float* a, const float* b,
float* out, int n, int op) {
CV_CPU_DISPATCH(simd_binop_f32_, (a, b, out, n, op),
NEON, AVX2, AVX, BASELINE);
}
}
class NaryEltwiseHelper CV_FINAL
@@ -450,10 +464,32 @@ public:
}
template <typename T, typename RESULT_T, typename Functor>
void binary_forward_impl(const Functor& op, int ndims, const std::vector<int>& shape,
const char* data1, const std::vector<size_t>& step1,
const char* data2, const std::vector<size_t>& step2,
char* data, const std::vector<size_t>& step, size_t block_size) {
void binary_forward_impl(const Functor& op, int ndims, const std::vector<int>& shape_in,
const char* data1, const std::vector<size_t>& step1_in,
const char* data2, const std::vector<size_t>& step2_in,
char* data, const std::vector<size_t>& step_in, size_t block_size) {
// Collapse consecutive dims that are contiguous in all three tensors.
std::vector<int> shape(shape_in.begin(), shape_in.begin() + ndims);
std::vector<size_t> step1(step1_in.begin(), step1_in.begin() + ndims);
std::vector<size_t> step2(step2_in.begin(), step2_in.begin() + ndims);
std::vector<size_t> step (step_in .begin(), step_in .begin() + ndims);
for (int k = (int)shape.size() - 2; k >= 0; k--) {
size_t e1 = (size_t)shape[k + 1] * step1[k + 1];
size_t e2 = (size_t)shape[k + 1] * step2[k + 1];
size_t e = (size_t)shape[k + 1] * step [k + 1];
if (step1[k] == e1 && step2[k] == e2 && step[k] == e) {
shape[k] *= shape[k + 1];
step1[k] = step1[k + 1];
step2[k] = step2[k + 1];
step [k] = step [k + 1];
shape.erase(shape.begin() + k + 1);
step1.erase(step1.begin() + k + 1);
step2.erase(step2.begin() + k + 1);
step .erase(step .begin() + k + 1);
}
}
ndims = (int)shape.size();
size_t dp1 = 0, dp2 = 0, dp = 0;
int plane_size = 1;
size_t nplanes = 1;
@@ -469,46 +505,35 @@ public:
}
}
#if CV_SIMD
// Fast path: fully contiguous float Add → flatten + SIMD + parallel_for_
bool is_add = (this->op == OPERATION::SUM || this->op == OPERATION::ADD);
if (is_add && std::is_same<T, float>::value && std::is_same<RESULT_T, float>::value &&
dp1 == 1 && dp2 == 1 && dp == 1 && ndims >= 1) {
bool contiguous = true;
for (int k = ndims - 2; k >= 0; k--) {
if (shape[k] <= 1) continue; // size-1 dims have stride 0, skip
size_t expected = (size_t)shape[k + 1] * step1[k + 1];
if (step1[k] != expected || step2[k] != expected || step[k] != expected) {
contiguous = false;
break;
const bool is_add = (this->op == OPERATION::SUM || this->op == OPERATION::ADD);
const bool is_sub = (this->op == OPERATION::SUB);
const bool is_mul = (this->op == OPERATION::PROD);
const bool is_div = (this->op == OPERATION::DIV);
const bool simd_f32 = std::is_same<T, float>::value && std::is_same<RESULT_T, float>::value &&
(is_add || is_sub || is_mul || is_div);
const int simd_bin_op = is_add ? cpu_baseline::SIMD_BIN_ADD :
is_sub ? cpu_baseline::SIMD_BIN_SUB :
is_mul ? cpu_baseline::SIMD_BIN_MUL :
is_div ? cpu_baseline::SIMD_BIN_DIV : -1;
// Fast path: fully-flat contiguous float binop -> one big parallel SIMD loop.
if (simd_f32 && ndims == 1 && dp1 == 1 && dp2 == 1 && dp == 1) {
int64_t total = (int64_t)nplanes * plane_size;
const float* p1 = (const float*)data1;
const float* p2 = (const float*)data2;
float* po = (float*)data;
const int64_t chunk = 4096;
int64_t nchunks = (total + chunk - 1) / chunk;
parallel_for_(Range(0, (int)nchunks), [&](const Range& r) {
for (int c = r.start; c < r.end; c++) {
int64_t start = c * chunk;
int64_t end = std::min(start + chunk, total);
int n = (int)(end - start);
simd_binop_f32_dispatch(p1 + start, p2 + start, po + start, n, simd_bin_op);
}
}
if (contiguous) {
int64_t total = (int64_t)nplanes * plane_size;
const float* p1 = (const float*)data1;
const float* p2 = (const float*)data2;
float* po = (float*)data;
const int64_t chunk = 1024;
int64_t nchunks = (total + chunk - 1) / chunk;
parallel_for_(Range(0, (int)nchunks), [&](const Range& r) {
for (int c = r.start; c < r.end; c++) {
int64_t start = c * chunk;
int64_t end = std::min(start + chunk, total);
int64_t i = start;
for (; i <= end - (int64_t)VTraits<v_float32>::nlanes * 4; i += VTraits<v_float32>::nlanes * 4) {
v_store(po + i, v_add(vx_load(p1 + i), vx_load(p2 + i)));
v_store(po + i + VTraits<v_float32>::nlanes, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes), vx_load(p2 + i + VTraits<v_float32>::nlanes)));
v_store(po + i + VTraits<v_float32>::nlanes*2, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes*2), vx_load(p2 + i + VTraits<v_float32>::nlanes*2)));
v_store(po + i + VTraits<v_float32>::nlanes*3, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes*3), vx_load(p2 + i + VTraits<v_float32>::nlanes*3)));
}
for (; i < end; i++)
po[i] = p1[i] + p2[i];
}
});
return;
}
});
return;
}
#endif
if (nplanes == 1) { // parallelize within the plane
const T* ptr1 = (const T*)data1;
@@ -516,24 +541,14 @@ public:
RESULT_T* ptr = (RESULT_T*)data;
auto worker = [&](const Range &r) {
if (dp1 == 1 && dp2 == 1 && dp == 1) {
int i = r.start;
#if CV_SIMD
if (is_add && std::is_same<T, float>::value && std::is_same<RESULT_T, float>::value) {
if (simd_f32) {
const float* p1 = (const float*)(const void*)&ptr1[r.start];
const float* p2 = (const float*)(const void*)&ptr2[r.start];
float* po = (float*)(void*)&ptr[r.start];
int len = r.end - r.start, j = 0;
for (; j <= len - VTraits<v_float32>::nlanes * 4; j += VTraits<v_float32>::nlanes * 4) {
v_store(po + j, v_add(vx_load(p1 + j), vx_load(p2 + j)));
v_store(po + j + VTraits<v_float32>::nlanes, v_add(vx_load(p1 + j + VTraits<v_float32>::nlanes), vx_load(p2 + j + VTraits<v_float32>::nlanes)));
v_store(po + j + VTraits<v_float32>::nlanes*2, v_add(vx_load(p1 + j + VTraits<v_float32>::nlanes*2), vx_load(p2 + j + VTraits<v_float32>::nlanes*2)));
v_store(po + j + VTraits<v_float32>::nlanes*3, v_add(vx_load(p1 + j + VTraits<v_float32>::nlanes*3), vx_load(p2 + j + VTraits<v_float32>::nlanes*3)));
}
i = r.start + j;
}
#endif
for(; i < r.end; i++) {
ptr[i] = op(ptr1[i], ptr2[i]);
simd_binop_f32_dispatch(p1, p2, po, r.end - r.start, simd_bin_op);
} else {
for (int i = r.start; i < r.end; i++)
ptr[i] = op(ptr1[i], ptr2[i]);
}
} else if (dp1 == 1 && dp2 == 0 && dp == 1){
T x2 = *ptr2;
@@ -574,22 +589,14 @@ public:
const T* ptr2 = (const T*)ptr2_;
RESULT_T* ptr = (RESULT_T*)ptr_;
if (dp1 == 1 && dp2 == 1 && dp == 1) {
int i = 0;
#if CV_SIMD
if (is_add && std::is_same<T, float>::value && std::is_same<RESULT_T, float>::value) {
const float* p1 = (const float*)(const void*)ptr1;
const float* p2 = (const float*)(const void*)ptr2;
float* po = (float*)(void*)ptr;
for (; i <= plane_size - VTraits<v_float32>::nlanes * 4; i += VTraits<v_float32>::nlanes * 4) {
v_store(po + i, v_add(vx_load(p1 + i), vx_load(p2 + i)));
v_store(po + i + VTraits<v_float32>::nlanes, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes), vx_load(p2 + i + VTraits<v_float32>::nlanes)));
v_store(po + i + VTraits<v_float32>::nlanes*2, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes*2), vx_load(p2 + i + VTraits<v_float32>::nlanes*2)));
v_store(po + i + VTraits<v_float32>::nlanes*3, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes*3), vx_load(p2 + i + VTraits<v_float32>::nlanes*3)));
}
}
#endif
for(; i < plane_size; i++) {
ptr[i] = op(ptr1[i], ptr2[i]);
if (simd_f32) {
simd_binop_f32_dispatch((const float*)(const void*)ptr1,
(const float*)(const void*)ptr2,
(float*)(void*)ptr,
plane_size, simd_bin_op);
} else {
for (int i = 0; i < plane_size; i++)
ptr[i] = op(ptr1[i], ptr2[i]);
}
} else if (dp1 == 1 && dp2 == 0 && dp == 1){
T x2 = *ptr2;
@@ -608,7 +615,14 @@ public:
}
}
};
double nstripes = nplanes * (1.0 / double(block_size));
// Use total work (planes * plane_size) so large broadcasts parallelize
// even when nplanes alone is below the old nstripes threshold. Target
// ~16KB of output work per stripe so even tiny plane_size ops (e.g.
// 2-element broadcast rows) spread across threads instead of running
// serially.
double nstripes = (double)nplanes * plane_size * sizeof(T) / 16384.0;
if (nstripes < 1.0) nstripes = 1.0;
(void)block_size;
parallel_for_(Range(0, nplanes), worker, nstripes);
}
}
@@ -741,7 +755,9 @@ public:
}
}
};
double nstripes = nplanes * (1.0 / double(block_size));
double nstripes = (double)nplanes * plane_size * sizeof(T) / 16384.0;
if (nstripes < 1.0) nstripes = 1.0;
(void)block_size;
parallel_for_(Range(0, nplanes), worker, nstripes);
}
}
@@ -880,7 +896,9 @@ public:
}
}
};
double nstripes = nplanes * (1.0 / double(block_size));
double nstripes = (double)nplanes * plane_size * (sizeof(T_INP1) + sizeof(T_OUT)) / 16384.0;
if (nstripes < 1.0) nstripes = 1.0;
(void)block_size;
parallel_for_(Range(0, nplanes), worker, nstripes);
}
}
modules/dnn/src/layers/nonzero_layer.cpp
+1 -1
View File
@@ -147,7 +147,7 @@ public:
{
CV_Assert(in_arr.size().area() == 1);
const Mat X = in_arr.getMat(0);
CV_Assert(X.data != nullptr);
CV_Assert(X.total() == 0 || X.data != nullptr);
const int rank = X.dims;
std::vector<int> dims(rank), strides(rank);
modules/dnn/src/layers/reduce2_layer.cpp
+40
View File
@@ -5,6 +5,13 @@
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "cpu_kernels/reduce2_kernels.simd.hpp"
#include "layers/cpu_kernels/reduce2_kernels.simd_declarations.hpp"
#define CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN namespace cpu_baseline {
#define CV_CPU_OPTIMIZATION_NAMESPACE_END }
#undef CV_CPU_DISPATCH_MODES_ALL
#include <opencv2/dnn/shape_utils.hpp>
#include "../net_impl.hpp"
#include "../op_cann.hpp"
@@ -521,9 +528,42 @@ public:
outputs_arr.getMatVector(outputs);
Mat& dst = outputs[0];
if (src.depth() == CV_32F && src.isContinuous() && dst.isContinuous() &&
reduce_type != ReduceType::LOG_SUM_EXP) {
if (dst.total() == 1) {
CV_CPU_DISPATCH(reduceAllFloatParallel_, (src, dst, (int)reduce_type),
NEON, AVX2, AVX, BASELINE);
return;
}
size_t innerLen = 1;
if (reduceTrailingAxesLen(src, axes, innerLen) && innerLen > 1) {
CV_CPU_DISPATCH(reduceLastAxesFloatParallel_, (src, dst, innerLen, (int)reduce_type),
NEON, AVX2, AVX, BASELINE);
return;
}
}
typeDispatch(dst.type(), src, dst, axes, noop_with_empty_axes);
}
static bool reduceTrailingAxesLen(const Mat& src, const std::vector<int>& axes,
size_t& innerLen)
{
MatShape s = shape(src);
int nd = s.dims;
if (axes.empty() || (int)axes.size() > nd) return false;
std::vector<int> sorted_axes(axes.begin(), axes.end());
std::sort(sorted_axes.begin(), sorted_axes.end());
// Must be the trailing [nd - k .. nd - 1] block.
int k = (int)sorted_axes.size();
for (int i = 0; i < k; i++)
if (sorted_axes[i] != nd - k + i) return false;
innerLen = 1;
for (int i = nd - k; i < nd; i++) innerLen *= (size_t)s[i];
return true;
}
virtual std::ostream& dumpAttrs(std::ostream& strm, int indent) const CV_OVERRIDE
{
prindent(strm, indent);
modules/dnn/src/layers/reshape2_layer.cpp
+4
View File
@@ -132,6 +132,10 @@ public:
return backendId == DNN_BACKEND_OPENCV;
}
// Reshape just re-interprets the same contiguous buffer. Let the graph
// buffer allocator alias input and output so the memcpy in forward() is a no-op.
virtual bool alwaysSupportInplace() const CV_OVERRIDE { return true; }
bool haveShapeSpec() const
{
return newShapeDesc.dims >= 0;
modules/dnn/src/layers/squeeze_layer.cpp
+2
View File
@@ -48,6 +48,8 @@ public:
return backendId == DNN_BACKEND_OPENCV;
}
virtual bool alwaysSupportInplace() const CV_OVERRIDE { return true; }
MatShape getOutShape(const MatShape& inpShape, const std::vector<int>& axes_) const
{
bool squeezeMask[MatShape::MAX_DIMS];
modules/dnn/src/layers/transpose_layer.cpp
+85
View File
@@ -3,6 +3,12 @@
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include "cpu_kernels/transpose_kernels.simd.hpp"
#include "layers/cpu_kernels/transpose_kernels.simd_declarations.hpp"
#define CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN namespace cpu_baseline {
#define CV_CPU_OPTIMIZATION_NAMESPACE_END }
#undef CV_CPU_DISPATCH_MODES_ALL
#include "layers_common.hpp"
#include "../net_impl.hpp"
//#include "../op_cuda.hpp"
@@ -26,6 +32,26 @@ namespace dnn
Opset's 1 to 23 are covered.
*/
// Detect a "swap innermost two dims" permutation, collapsing leading dims.
// Returns true and fills outer/rows/cols when applicable.
static bool isSwapLastTwo(const MatShape& inpShape,
const std::vector<int>& perm,
int esz,
int64_t& outer, int64_t& rows, int64_t& cols)
{
if (esz != 4) return false;
int nd = inpShape.dims;
if (nd < 2 || (int)perm.size() != nd) return false;
for (int i = 0; i < nd - 2; i++)
if (perm[i] != i) return false;
if (perm[nd - 2] != nd - 1 || perm[nd - 1] != nd - 2) return false;
outer = 1;
for (int i = 0; i < nd - 2; i++) outer *= inpShape[i];
rows = inpShape[nd - 1]; // output rows = input inner dim
cols = inpShape[nd - 2]; // output cols = input outer-of-pair dim
return true;
}
static void transpose(const Mat& inp, const std::vector<int>& perm, Mat& out)
{
enum {TRANSPOSE_MAX_DIMS=7};
@@ -35,6 +61,65 @@ static void transpose(const Mat& inp, const std::vector<int>& perm, Mat& out)
size_t esz = inp.elemSize();
CV_Assert(esz == 1 || esz == 2 || esz == 4 || esz == 8);
int64_t outer = 1, rows = 0, cols = 0;
if (isSwapLastTwo(inpShape, perm, (int)esz, outer, rows, cols)
&& inp.isContinuous() && out.isContinuous()) {
CV_CPU_DISPATCH(transpose2D_f32_,
(inp.ptr<float>(), out.ptr<float>(), outer, rows, cols),
NEON, AVX2, AVX, BASELINE);
return;
}
if (inp.isContinuous() && out.isContinuous() &&
ndims >= 2 && (int)perm.size() == ndims &&
perm[ndims - 1] == ndims - 1)
{
std::vector<int64_t> inStride(ndims, 0);
inStride[ndims - 1] = 1;
for (int i = ndims - 2; i >= 0; i--)
inStride[i] = inStride[i + 1] * (int64_t)inpShape[i + 1];
const int64_t inner = inpShape[ndims - 1];
std::vector<int> outOuterShape(ndims - 1);
for (int i = 0; i < ndims - 1; i++) outOuterShape[i] = outShape[i];
int64_t outerTotal = 1;
for (int i = 0; i < ndims - 1; i++) outerTotal *= outOuterShape[i];
if (outerTotal > 0 && inner > 0) {
const size_t innerBytes = (size_t)inner * esz;
const char* in_base = (const char*)inp.data;
char* out_base = (char*)out.data;
parallel_for_(Range(0, (int)outerTotal), [&](const Range& r) {
std::vector<int> outIdx(ndims - 1, 0);
// initialize outIdx to r.start in row-major
int64_t rem = r.start;
for (int k = ndims - 2; k >= 0; k--) {
outIdx[k] = (int)(rem % outOuterShape[k]);
rem /= outOuterShape[k];
}
for (int64_t idx = r.start; idx < r.end; idx++) {
// Compute input outer offset using perm.
int64_t inOff = 0;
for (int i = 0; i < ndims - 1; i++) {
// perm[i] is the input axis providing this output axis.
inOff += (int64_t)outIdx[i] * inStride[perm[i]];
}
// Output position: linear idx * inner.
std::memcpy(out_base + (size_t)idx * innerBytes,
in_base + (size_t)inOff * esz,
innerBytes);
// Advance outIdx (row-major over outOuterShape).
for (int k = ndims - 2; k >= 0; k--) {
if (++outIdx[k] < outOuterShape[k]) break;
outIdx[k] = 0;
}
}
}, std::max(1.0, (double)outerTotal * (double)innerBytes / 16384.0));
return;
}
}
int perm_[TRANSPOSE_MAX_DIMS];
int inpShape_[TRANSPOSE_MAX_DIMS];
int outShape_[TRANSPOSE_MAX_DIMS];
modules/dnn/src/layers/unsqueeze_layer.cpp
+2
View File
@@ -48,6 +48,8 @@ public:
return backendId == DNN_BACKEND_OPENCV;
}
virtual bool alwaysSupportInplace() const CV_OVERRIDE { return true; }
MatShape getOutShape(const MatShape& inpShape, const std::vector<int>& axes_) const
{
bool unsqueezeMask[MatShape::MAX_DIMS];
modules/dnn/src/net_impl2.cpp
+2 -1
View File
@@ -1245,8 +1245,9 @@ void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
mCond = outputs[0];
active = loopLayer->cond(mCond);
// Deep-copy: body buffers (and their UMats via Mat::fit reuse) are recycled across iterations.
for (int i = 0; i < n_state; i++)
state[i] = outputs[1 + i];
state[i] = outputs[1 + i].clone();
for (int i = 0; i < n_accum; i++)
history[i].push_back(outputs[1 + n_state + i].clone());
}
modules/dnn/test/test_onnx_conformance.cpp
+8
View File
@@ -1889,6 +1889,14 @@ TEST_P(Test_ONNX_conformance, Layer_Test)
default_lInf = 0.0005; // Expected: (normInf) <= (lInf), actual: 0.000455521 vs 0.0001
}
}
if (name == "test_reduce_prod_default_axes_keepdims_random") {
default_l1 = 0.002; // Expected: (normL1) <= (l1), actual: 0.00195312 vs 1e-05
default_lInf = 0.002; // Expected: (normInf) <= (lInf), actual: 0.00195312 vs 0.0001
}
if (name == "test_reduce_sum_square_default_axes_keepdims_random" ||
name == "test_reduce_sum_square_default_axes_keepdims_random_expanded") {
default_l1 = 2e-5; // Expected: (normL1) <= (l1), actual: 1.52588e-05 vs 1e-05
}
}
#ifdef HAVE_HALIDE
else if (backend == DNN_BACKEND_HALIDE)