1
0
mirror of https://github.com/opencv/opencv.git synced 2026-07-31 08:13:04 +04:00

Merge branch 4.x

This commit is contained in:
Alexander Smorkalov
2023-11-02 16:24:38 +03:00
110 changed files with 6532 additions and 2686 deletions
+11 -11
View File
@@ -2,32 +2,32 @@ function(download_ippicv root_var)
set(${root_var} "" PARENT_SCOPE)
# Commit SHA in the opencv_3rdparty repo
set(IPPICV_COMMIT "1224f78da6684df04397ac0f40c961ed37f79ccb")
set(IPPICV_COMMIT "0cc4aa06bf2bef4b05d237c69a5a96b9cd0cb85a")
# Define actual ICV versions
if(APPLE)
set(OPENCV_ICV_PLATFORM "macosx")
set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_mac")
set(OPENCV_ICV_NAME "ippicv_2021.8_mac_intel64_20230330_general.tgz")
set(OPENCV_ICV_HASH "d2b234a86af1b616958619a4560356d9")
set(OPENCV_ICV_NAME "ippicv_2021.9.1_mac_intel64_20230919_general.tgz")
set(OPENCV_ICV_HASH "14f01c5a4780bfae9dde9b0aaf5e56fc")
elseif((UNIX AND NOT ANDROID) OR (UNIX AND ANDROID_ABI MATCHES "x86"))
set(OPENCV_ICV_PLATFORM "linux")
set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_lnx")
if(X86_64)
set(OPENCV_ICV_NAME "ippicv_2021.8_lnx_intel64_20230330_general.tgz")
set(OPENCV_ICV_HASH "43219bdc7e3805adcbe3a1e2f1f3ef3b")
set(OPENCV_ICV_NAME "ippicv_2021.10.0_lnx_intel64_20230919_general.tgz")
set(OPENCV_ICV_HASH "606a19b207ebedfe42d59fd916cc4850")
else()
set(OPENCV_ICV_NAME "ippicv_2021.8_lnx_ia32_20230330_general.tgz")
set(OPENCV_ICV_HASH "165875443d72faa3fd2146869da90d07")
set(OPENCV_ICV_NAME "ippicv_2021.10.0_lnx_ia32_20230919_general.tgz")
set(OPENCV_ICV_HASH "ea08487b810baad2f68aca87b74a2db9")
endif()
elseif(WIN32 AND NOT ARM)
set(OPENCV_ICV_PLATFORM "windows")
set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_win")
if(X86_64)
set(OPENCV_ICV_NAME "ippicv_2021.8_win_intel64_20230330_general.zip")
set(OPENCV_ICV_HASH "71e4f58de939f0348ec7fb58ffb17dbf")
set(OPENCV_ICV_NAME "ippicv_2021.10.0_win_intel64_20230919_general.zip")
set(OPENCV_ICV_HASH "538a819ec84193a9c9f3c0f8df0be8b7")
else()
set(OPENCV_ICV_NAME "ippicv_2021.8_win_ia32_20230330_general.zip")
set(OPENCV_ICV_HASH "57fd4648cfe64eae9e2ad9d50173a553")
set(OPENCV_ICV_NAME "ippicv_2021.10.0_win_ia32_20230919_general.zip")
set(OPENCV_ICV_HASH "8ff93c69415ab0835cc1e94dc5660f5d")
endif()
else()
return()
+1 -1
View File
@@ -467,7 +467,7 @@ OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF
VISIBLE_IF TRUE
VERIFY HAVE_TIMVX)
OCV_OPTION(WITH_OBSENSOR "Include obsensor support (Orbbec RGB-D modules: Astra+/Femto)" ON
VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT) OR ( UNIX AND NOT APPLE AND NOT ANDROID)
VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT AND NOT MINGW) OR ( UNIX AND NOT APPLE AND NOT ANDROID)
VERIFY HAVE_OBSENSOR)
OCV_OPTION(WITH_CANN "Include CANN support" OFF
VISIBLE_IF TRUE
+18 -4
View File
@@ -53,6 +53,7 @@ list(APPEND CPU_ALL_OPTIMIZATIONS NEON VFPV3 FP16 NEON_DOTPROD)
list(APPEND CPU_ALL_OPTIMIZATIONS MSA)
list(APPEND CPU_ALL_OPTIMIZATIONS VSX VSX3)
list(APPEND CPU_ALL_OPTIMIZATIONS RVV)
list(APPEND CPU_ALL_OPTIMIZATIONS LSX)
list(APPEND CPU_ALL_OPTIMIZATIONS LASX)
list(REMOVE_DUPLICATES CPU_ALL_OPTIMIZATIONS)
@@ -331,7 +332,9 @@ if(X86 OR X86_64)
elseif(ARM OR AARCH64)
ocv_update(CPU_NEON_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_neon.cpp")
ocv_update(CPU_FP16_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_fp16.cpp")
ocv_update(CPU_NEON_DOTPROD_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_dotprod.cpp")
ocv_update(CPU_NEON_FP16_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_neon_fp16.cpp")
ocv_update(CPU_NEON_BF16_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_neon_bf16.cpp")
ocv_update(CPU_NEON_DOTPROD_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_neon_dotprod.cpp")
if(NOT AARCH64)
ocv_update(CPU_KNOWN_OPTIMIZATIONS "VFPV3;NEON;FP16")
if(NOT MSVC)
@@ -343,12 +346,17 @@ elseif(ARM OR AARCH64)
endif()
ocv_update(CPU_FP16_IMPLIES "NEON")
else()
ocv_update(CPU_KNOWN_OPTIMIZATIONS "NEON;FP16;NEON_DOTPROD")
ocv_update(CPU_KNOWN_OPTIMIZATIONS "NEON;FP16;NEON_DOTPROD;NEON_FP16;NEON_BF16")
ocv_update(CPU_NEON_FLAGS_ON "")
ocv_update(CPU_FP16_IMPLIES "NEON")
ocv_update(CPU_NEON_DOTPROD_FLAGS_ON "-march=armv8.2-a+dotprod")
ocv_update(CPU_NEON_DOTPROD_IMPLIES "NEON")
ocv_update(CPU_NEON_FP16_FLAGS_ON "-march=armv8.2-a+fp16")
ocv_update(CPU_NEON_FP16_IMPLIES "NEON")
ocv_update(CPU_NEON_BF16_FLAGS_ON "-march=armv8.2-a+fp16+bf16")
ocv_update(CPU_NEON_BF16_IMPLIES "NEON")
set(CPU_BASELINE "NEON;FP16" CACHE STRING "${HELP_CPU_BASELINE}")
set(CPU_DISPATCH "NEON_FP16;NEON_BF16;NEON_DOTPROD" CACHE STRING "${HELP_CPU_DISPATCH}")
endif()
elseif(MIPS)
ocv_update(CPU_MSA_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_msa.cpp")
@@ -390,10 +398,16 @@ elseif(RISCV)
set(CPU_BASELINE "DETECT" CACHE STRING "${HELP_CPU_BASELINE}")
elseif(LOONGARCH64)
ocv_update(CPU_LSX_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_lsx.cpp")
ocv_update(CPU_LASX_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_lasx.cpp")
ocv_update(CPU_KNOWN_OPTIMIZATIONS "LASX")
ocv_update(CPU_KNOWN_OPTIMIZATIONS "LSX;LASX")
ocv_update(CPU_LSX_FLAGS_ON "-mlsx")
ocv_update(CPU_LASX_FLAGS_ON "-mlasx")
set(CPU_BASELINE "LASX" CACHE STRING "${HELP_CPU_BASELINE}")
if("${CPU_BASELINE_DISABLE}" STREQUAL "LASX")
set(CPU_BASELINE "LSX" CACHE STRING "${HELP_CPU_BASELINE}")
else()
set(CPU_BASELINE "LASX" CACHE STRING "${HELP_CPU_BASELINE}")
endif()
endif()
+8
View File
@@ -355,6 +355,14 @@ if(CUDA_FOUND)
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xptxas;-dlcm=ca")
endif()
# Tell NVCC the maximum number of threads to be used to execute the compilation steps in parallel
# (option --threads was introduced in version 11.2)
if(NOT CUDA_VERSION VERSION_LESS "11.2")
if(CMAKE_GENERATOR MATCHES "Visual Studio" AND NOT $ENV{CMAKE_BUILD_PARALLEL_LEVEL} STREQUAL "")
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "--threads=$ENV{CMAKE_BUILD_PARALLEL_LEVEL}")
endif()
endif()
message(STATUS "CUDA NVCC target flags: ${CUDA_NVCC_FLAGS}")
OCV_OPTION(CUDA_FAST_MATH "Enable --use_fast_math for CUDA compiler " OFF)
+30 -5
View File
@@ -30,8 +30,14 @@ if(BUILD_PROTOBUF)
set(Protobuf_LIBRARIES "libprotobuf")
set(HAVE_PROTOBUF TRUE)
else()
# we still need this for command PROTOBUF_GENERATE_CPP.
set(protobuf_MODULE_COMPATIBLE ON)
unset(Protobuf_VERSION CACHE)
find_package(Protobuf QUIET)
find_package(Protobuf QUIET CONFIG)
if(NOT Protobuf_FOUND)
find_package(Protobuf QUIET)
endif()
# Backwards compatibility
# Define camel case versions of input variables
@@ -67,6 +73,20 @@ else()
endif()
endif()
# See https://github.com/opencv/opencv/issues/24369
# In Protocol Buffers v22.0 and later drops C++11 support and depends abseil-cpp.
# Details: https://protobuf.dev/news/2022-08-03/
# And if std::text_view is in abseil-cpp requests C++17 and later.
if(HAVE_PROTOBUF)
if(NOT (Protobuf_VERSION VERSION_LESS 22))
if((CMAKE_CXX_STANDARD EQUAL 98) OR (CMAKE_CXX_STANDARD LESS 17))
message(STATUS "CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} is too old to support protobuf(${Protobuf_VERSION}) and/or abseil-cpp. Use C++17 or later. Turning HAVE_PROTOBUF off")
set(HAVE_PROTOBUF FALSE)
endif()
endif()
endif()
if(HAVE_PROTOBUF AND PROTOBUF_UPDATE_FILES AND NOT COMMAND PROTOBUF_GENERATE_CPP)
message(FATAL_ERROR "Can't configure protobuf dependency (BUILD_PROTOBUF=${BUILD_PROTOBUF} PROTOBUF_UPDATE_FILES=${PROTOBUF_UPDATE_FILES})")
endif()
@@ -74,15 +94,20 @@ endif()
if(HAVE_PROTOBUF)
list(APPEND CUSTOM_STATUS protobuf)
if(NOT BUILD_PROTOBUF)
unset( __location)
if(TARGET "${Protobuf_LIBRARIES}")
get_target_property(__location "${Protobuf_LIBRARIES}" IMPORTED_LOCATION_RELEASE)
if(NOT __location)
get_target_property(__location "${Protobuf_LIBRARIES}" IMPORTED_LOCATION)
endif()
elseif(Protobuf_LIBRARY)
set(__location "${Protobuf_LIBRARY}")
else()
set(__location "${Protobuf_LIBRARIES}")
endif()
if(NOT __location)
if(Protobuf_LIBRARY)
set(__location "${Protobuf_LIBRARY}")
else()
set(__location "${Protobuf_LIBRARIES}")
endif()
endif()
endif()
list(APPEND CUSTOM_STATUS_protobuf " Protobuf:"
+15
View File
@@ -0,0 +1,15 @@
#include <stdio.h>
#include <lsxintrin.h>
int test()
{
const float src[] = { 0.0f, 1.0f, 2.0f, 3.0f};
v4f32 val = (v4f32)__lsx_vld((const float*)(src), 0);
return __lsx_vpickve2gr_w(__lsx_vftint_w_s(val), 3);
}
int main()
{
printf("%d\n", test());
return 0;
}
+46
View File
@@ -0,0 +1,46 @@
#if defined __GNUC__ && (defined __arm__ || defined __aarch64__)
#include <stdio.h>
#include "arm_neon.h"
/*#if defined __clang__
#pragma clang attribute push (__attribute__((target("bf16"))), apply_to=function)
#elif defined GCC
#pragma GCC push_options
#pragma GCC target("armv8.2-a", "bf16")
#endif*/
bfloat16x8_t vld1q_as_bf16(const float* src)
{
float32x4_t s0 = vld1q_f32(src), s1 = vld1q_f32(src + 4);
return vcombine_bf16(vcvt_bf16_f32(s0), vcvt_bf16_f32(s1));
}
void vprintreg(const char* name, const float32x4_t& r)
{
float data[4];
vst1q_f32(data, r);
printf("%s: (%.2f, %.2f, %.2f, %.2f)\n",
name, data[0], data[1], data[2], data[3]);
}
void test()
{
const float src1[] = { 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f };
const float src2[] = { 1.f, 3.f, 6.f, 10.f, 15.f, 21.f, 28.f, 36.f };
bfloat16x8_t s1 = vld1q_as_bf16(src1), s2 = vld1q_as_bf16(src2);
float32x4_t d = vbfdotq_f32(vdupq_n_f32(0.f), s1, s2);
vprintreg("(s1[0]*s2[0] + s1[1]*s2[1], ... s1[6]*s2[6] + s1[7]*s2[7])", d);
}
/*#if defined __clang__
#pragma clang attribute pop
#elif defined GCC
#pragma GCC pop_options
#endif*/
#else
#error "BF16 is not supported"
#endif
int main()
{
test();
return 0;
}
+46
View File
@@ -0,0 +1,46 @@
#include <stdio.h>
#if defined __GNUC__ && (defined __arm__ || defined __aarch64__)
#include "arm_neon.h"
float16x8_t vld1q_as_f16(const float* src)
{
float32x4_t s0 = vld1q_f32(src), s1 = vld1q_f32(src + 4);
return vcombine_f16(vcvt_f16_f32(s0), vcvt_f16_f32(s1));
}
void vprintreg(const char* name, const float16x8_t& r)
{
float data[8];
vst1q_f32(data, vcvt_f32_f16(vget_low_f16(r)));
vst1q_f32(data + 4, vcvt_f32_f16(vget_high_f16(r)));
printf("%s: (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f)\n",
name, data[0], data[1], data[2], data[3],
data[4], data[5], data[6], data[7]);
}
void test()
{
const float src1[] = { 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f };
const float src2[] = { 1.f, 3.f, 6.f, 10.f, 15.f, 21.f, 28.f, 36.f };
float16x8_t s1 = vld1q_as_f16(src1), s2 = vld1q_as_f16(src2);
float16x8_t d = vsubq_f16(s1, s1);
d = vfmaq_laneq_f16(d, s1, s2, 0);
d = vfmaq_laneq_f16(d, s1, s2, 1);
d = vfmaq_laneq_f16(d, s1, s2, 2);
d = vfmaq_laneq_f16(d, s1, s2, 3);
d = vfmaq_laneq_f16(d, s1, s2, 4);
d = vfmaq_laneq_f16(d, s1, s2, 5);
d = vfmaq_laneq_f16(d, s1, s2, 6);
d = vfmaq_laneq_f16(d, s1, s2, 7);
vprintreg("s1*s2[0]+s1*s2[1] + ... + s1*s2[7]", d);
}
#else
#error "FP16 is not supported"
#endif
int main()
{
test();
return 0;
}
@@ -80,7 +80,7 @@ using **np.ifft2()** function. The result, again, will be a complex number. You
absolute value.
@code{.py}
rows, cols = img.shape
crow,ccol = rows//2 , cols//2
crow, ccol = rows//2, cols//2
fshift[crow-30:crow+31, ccol-30:ccol+31] = 0
f_ishift = np.fft.ifftshift(fshift)
img_back = np.fft.ifft2(f_ishift)
@@ -146,7 +146,7 @@ content, and 0 at HF region.
@code{.py}
rows, cols = img.shape
crow,ccol = rows/2 , cols/2
crow, ccol = rows//2, cols//2
# create a mask first, center square is 1, remaining all zeros
mask = np.zeros((rows,cols,2),np.uint8)
+7 -7
View File
@@ -277,14 +277,14 @@ int farthestPointSampling(OutputArray sampled_point_flags, InputArray input_pts,
v_float32 v_last_p_y = vx_setall_f32(last_pt_y);
v_float32 v_last_p_z = vx_setall_f32(last_pt_z);
for (; i <= ori_pts_size - v_float32::nlanes; i += v_float32::nlanes)
for (; i <= ori_pts_size - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes())
{
v_float32 vx_diff = v_last_p_x - vx_load(ori_pts_ptr_x + i);
v_float32 vy_diff = v_last_p_y - vx_load(ori_pts_ptr_y + i);
v_float32 vz_diff = v_last_p_z - vx_load(ori_pts_ptr_z + i);
v_float32 vx_diff = v_sub(v_last_p_x, vx_load(ori_pts_ptr_x + i));
v_float32 vy_diff = v_sub(v_last_p_y, vx_load(ori_pts_ptr_y + i));
v_float32 vz_diff = v_sub(v_last_p_z, vx_load(ori_pts_ptr_z + i));
v_float32 v_next_dist_square =
vx_diff * vx_diff + vy_diff * vy_diff + vz_diff * vz_diff;
v_add(v_add(v_mul(vx_diff, vx_diff), v_mul(vy_diff, vy_diff)), v_mul(vz_diff, vz_diff));
// Update the distance from the points(in C) to S
float *dist_square_ptr = dist_square + i;
@@ -293,9 +293,9 @@ int farthestPointSampling(OutputArray sampled_point_flags, InputArray input_pts,
vx_store(dist_square_ptr, v_dist_square);
// Find a point in C that is the farthest away from S and take it from C to S
if (v_check_any(v_dist_square > vx_setall_f32(max_dist_square)))
if (v_check_any(v_gt(v_dist_square, vx_setall_f32(max_dist_square))))
{
for (int m = 0; m < v_float32::nlanes; ++m)
for (int m = 0; m < VTraits<v_float32>::vlanes(); ++m)
{
if (dist_square_ptr[m] > max_dist_square)
{
+79 -79
View File
@@ -112,35 +112,35 @@ void integrateColorTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f&
// optimization of the following:
//Point3f volPt = Point3f(x, y, z)*voxelSize;
//Point3f camSpacePt = vol2cam * volPt;
camSpacePt += zStep;
camSpacePt = v_add(camSpacePt, zStep);
float zCamSpace = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))).get0();
float zCamSpace = v_get0(v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))));
if (zCamSpace <= 0.f)
continue;
v_float32x4 camPixVec = camSpacePt / v_setall_f32(zCamSpace);
v_float32x4 camPixVec = v_div(camSpacePt, v_setall_f32(zCamSpace));
v_float32x4 projected = v_muladd(camPixVec, vfxy, vcxy);
// leave only first 2 lanes
projected = v_reinterpret_as_f32(v_reinterpret_as_u32(projected) &
v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0));
projected = v_reinterpret_as_f32(v_and(v_reinterpret_as_u32(projected),
v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0)));
depthType v;
// bilinearly interpolate depth at projected
{
const v_float32x4& pt = projected;
// check coords >= 0 and < imgSize
v_uint32x4 limits = v_reinterpret_as_u32(pt < v_setzero_f32()) |
v_reinterpret_as_u32(pt >= upLimits);
limits = limits | v_rotate_right<1>(limits);
if (limits.get0())
v_uint32x4 limits = v_or(v_reinterpret_as_u32(v_lt(pt, v_setzero_f32())),
v_reinterpret_as_u32(v_ge(pt, upLimits)));
limits = v_or(limits, v_rotate_right<1>(limits));
if (v_get0(limits))
continue;
// xi, yi = floor(pt)
v_int32x4 ip = v_floor(pt);
v_int32x4 ipshift = ip;
int xi = ipshift.get0();
int xi = v_get0(ipshift);
ipshift = v_rotate_right<1>(ipshift);
int yi = ipshift.get0();
int yi = v_get0(ipshift);
const depthType* row0 = depth[yi + 0];
const depthType* row1 = depth[yi + 1];
@@ -154,17 +154,17 @@ void integrateColorTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f&
// assume correct depth is positive
// don't fix missing data
if (v_check_all(vall > v_setzero_f32()))
if (v_check_all( v_gt(vall, v_setzero_f32())))
{
v_float32x4 t = pt - v_cvt_f32(ip);
float tx = t.get0();
v_float32x4 t = v_sub(pt, v_cvt_f32(ip));
float tx = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
v_float32x4 ty = v_setall_f32(t.get0());
v_float32x4 ty = v_setall_f32(v_get0(t));
// vx is y-interpolated between rows 0 and 1
v_float32x4 vx = v001 + ty * (v101 - v001);
float v0 = vx.get0();
v_float32x4 vx = v_add(v001, v_mul(ty, v_sub(v101, v001)));
float v0 = v_get0(vx);
vx = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vx)));
float v1 = vx.get0();
float v1 = v_get0(vx);
v = v0 + tx * (v1 - v0);
}
else
@@ -172,8 +172,8 @@ void integrateColorTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f&
}
// norm(camPixVec) produces double which is too slow
int _u = (int)projected.get0();
int _v = (int)v_rotate_right<1>(projected).get0();
int _u = (int)v_get0(projected);
int _v = (int)v_get0(v_rotate_right<1>(projected));
if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows))
continue;
@@ -330,21 +330,21 @@ inline float interpolateColorVoxel(const Mat& volume,
{
// tx, ty, tz = floor(p)
v_int32x4 ip = v_floor(p);
v_float32x4 t = p - v_cvt_f32(ip);
float tx = t.get0();
v_float32x4 t = v_sub(p, v_cvt_f32(ip));
float tx = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
float ty = t.get0();
float ty = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
float tz = t.get0();
float tz = v_get0(t);
int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2];
const RGBTsdfVoxel* volData = volume.ptr<RGBTsdfVoxel>();
int ix = ip.get0();
int ix = v_get0(ip);
ip = v_rotate_right<1>(ip);
int iy = ip.get0();
int iy = v_get0(ip);
ip = v_rotate_right<1>(ip);
int iz = ip.get0();
int iz = v_get0(ip);
int coordBase = ix * xdim + iy * ydim + iz * zdim;
@@ -354,15 +354,15 @@ inline float interpolateColorVoxel(const Mat& volume,
v_float32x4 v0246 = tsdfToFloat_INTR(v_int32x4(vx[0], vx[2], vx[4], vx[6]));
v_float32x4 v1357 = tsdfToFloat_INTR(v_int32x4(vx[1], vx[3], vx[5], vx[7]));
v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246);
v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246)));
v_float32x4 v00_10 = vxx;
v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx)));
v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10);
float v0 = v0_1.get0();
v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10)));
float v0 = v_get0(v0_1);
v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1)));
float v1 = v0_1.get0();
float v1 = v_get0(v0_1);
return v0 + tx * (v1 - v0);
}
@@ -419,27 +419,27 @@ inline v_float32x4 getNormalColorVoxel(const Mat& volume,
const Vec4i& volDims, const Vec8i& neighbourCoords, const Point3i volResolution,
const v_float32x4& p)
{
if (v_check_any(p < v_float32x4(1.f, 1.f, 1.f, 0.f)) ||
v_check_any(p >= v_float32x4((float)(volResolution.x - 2),
if (v_check_any(v_lt(p, v_float32x4(1.f, 1.f, 1.f, 0.f))) ||
v_check_any(v_ge(p, v_float32x4((float)(volResolution.x - 2),
(float)(volResolution.y - 2),
(float)(volResolution.z - 2), 1.f))
(float)(volResolution.z - 2), 1.f)))
)
return nanv;
v_int32x4 ip = v_floor(p);
v_float32x4 t = p - v_cvt_f32(ip);
float tx = t.get0();
v_float32x4 t = v_sub(p, v_cvt_f32(ip));
float tx = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
float ty = t.get0();
float ty = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
float tz = t.get0();
float tz = v_get0(t);
const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2];
const RGBTsdfVoxel* volData = volume.ptr<RGBTsdfVoxel>();
int ix = ip.get0(); ip = v_rotate_right<1>(ip);
int iy = ip.get0(); ip = v_rotate_right<1>(ip);
int iz = ip.get0();
int ix = v_get0(ip); ip = v_rotate_right<1>(ip);
int iy = v_get0(ip); ip = v_rotate_right<1>(ip);
int iz = v_get0(ip);
int coordBase = ix * xdim + iy * ydim + iz * zdim;
@@ -457,23 +457,23 @@ inline v_float32x4 getNormalColorVoxel(const Mat& volume,
v_float32x4 v0246(vx[0], vx[2], vx[4], vx[6]);
v_float32x4 v1357(vx[1], vx[3], vx[5], vx[7]);
v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246);
v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246)));
v_float32x4 v00_10 = vxx;
v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx)));
v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10);
float v0 = v0_1.get0();
v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10)));
float v0 = v_get0(v0_1);
v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1)));
float v1 = v0_1.get0();
float v1 = v_get0(v0_1);
nv = v0 + tx * (v1 - v0);
}
v_float32x4 n = v_load_aligned(an);
v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(n * n)));
v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(v_mul(n, n))));
return Norm.get0() < 0.0001f ? nanv : n / Norm;
return v_get0(Norm) < 0.0001f ? nanv : v_div(n, Norm);
}
inline Point3f getNormalColorVoxel(const Mat& volume,
@@ -544,15 +544,15 @@ inline float interpolateColor(float tx, float ty, float tz, float vx[8])
v_float32x4 v0246, v1357;
v_load_deinterleave(vx, v0246, v1357);
v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246);
v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246)));
v_float32x4 v00_10 = vxx;
v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx)));
v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10);
float v0 = v0_1.get0();
v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10)));
float v0 = v_get0(v0_1);
v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1)));
float v1 = v0_1.get0();
float v1 = v_get0(v0_1);
return v0 + tx * (v1 - v0);
}
@@ -579,10 +579,10 @@ inline v_float32x4 getColorVoxel(const Mat& volume,
const Vec4i& volDims, const Vec8i& neighbourCoords, const Point3i volResolution,
const float voxelSizeInv, const v_float32x4& p)
{
if (v_check_any(p < v_float32x4(1.f, 1.f, 1.f, 0.f)) ||
v_check_any(p >= v_float32x4((float)(volResolution.x - 2),
if (v_check_any(v_lt(p, v_float32x4(1.f, 1.f, 1.f, 0.f))) ||
v_check_any(v_ge(p, v_float32x4((float)(volResolution.x - 2),
(float)(volResolution.y - 2),
(float)(volResolution.z - 2), 1.f))
(float)(volResolution.z - 2), 1.f)))
)
return nanv;
@@ -591,9 +591,9 @@ inline v_float32x4 getColorVoxel(const Mat& volume,
const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2];
const RGBTsdfVoxel* volData = volume.ptr<RGBTsdfVoxel>();
int ix = ip.get0(); ip = v_rotate_right<1>(ip);
int iy = ip.get0(); ip = v_rotate_right<1>(ip);
int iz = ip.get0();
int ix = v_get0(ip); ip = v_rotate_right<1>(ip);
int iy = v_get0(ip); ip = v_rotate_right<1>(ip);
int iz = v_get0(ip);
int coordBase = ix * xdim + iy * ydim + iz * zdim;
float CV_DECL_ALIGNED(16) rgb[4];
@@ -608,12 +608,12 @@ inline v_float32x4 getColorVoxel(const Mat& volume,
}
v_float32x4 vsi(voxelSizeInv, voxelSizeInv, voxelSizeInv, voxelSizeInv);
v_float32x4 ptVox = p * vsi;
v_float32x4 ptVox = v_mul(p, vsi);
v_int32x4 iptVox = v_floor(ptVox);
v_float32x4 t = ptVox - v_cvt_f32(iptVox);
float tx = t.get0(); t = v_rotate_right<1>(t);
float ty = t.get0(); t = v_rotate_right<1>(t);
float tz = t.get0();
v_float32x4 t = v_sub(ptVox, v_cvt_f32(iptVox));
float tx = v_get0(t); t = v_rotate_right<1>(t);
float ty = v_get0(t); t = v_rotate_right<1>(t);
float tz = v_get0(t);
rgb[0] = interpolateColor(tx, ty, tz, r);
rgb[1] = interpolateColor(tx, ty, tz, g);
rgb[2] = interpolateColor(tx, ty, tz, b);
@@ -783,21 +783,21 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c
// get direction through pixel in volume space:
// 1. reproject (x, y) on projecting plane where z = 1.f
v_float32x4 planed = (v_float32x4((float)x, (float)y, 0.f, 0.f) - vcxy) * vfxy;
v_float32x4 planed = v_mul(v_sub(v_float32x4((float)x, (float)y, 0.f, 0.f), vcxy), vfxy);
planed = v_combine_low(planed, v_float32x4(1.f, 0.f, 0.f, 0.f));
// 2. rotate to volume space
planed = v_matmuladd(planed, camRot0, camRot1, camRot2, v_setzero_f32());
// 3. normalize
v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(planed * planed)));
v_float32x4 dir = planed * invNorm;
v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(v_mul(planed, planed))));
v_float32x4 dir = v_mul(planed, invNorm);
// compute intersection of ray with all six bbox planes
v_float32x4 rayinv = v_setall_f32(1.f) / dir;
v_float32x4 rayinv = v_div(v_setall_f32(1.f), dir);
// div by zero should be eliminated by these products
v_float32x4 tbottom = rayinv * (boxDown - orig);
v_float32x4 ttop = rayinv * (boxUp - orig);
v_float32x4 tbottom = v_mul(rayinv, v_sub(boxDown, orig));
v_float32x4 ttop = v_mul(rayinv, v_sub(boxUp, orig));
// re-order intersections to find smallest and largest on each axis
v_float32x4 minAx = v_min(ttop, tbottom);
@@ -818,14 +818,14 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c
if (tmin < tmax)
{
// interpolation optimized a little
orig *= invVoxelSize;
dir *= invVoxelSize;
orig = v_mul(orig, invVoxelSize);
dir = v_mul(dir, invVoxelSize);
int xdim = volDims[0];
int ydim = volDims[1];
int zdim = volDims[2];
v_float32x4 rayStep = dir * v_setall_f32(tstep);
v_float32x4 next = (orig + dir * v_setall_f32(tmin));
v_float32x4 rayStep = v_mul(dir, v_setall_f32(tstep));
v_float32x4 next = v_add(orig, v_mul(dir, v_setall_f32(tmin)));
float f = interpolateColorVoxel(volume, volDims, neighbourCoords, next);
float fnext = f;
@@ -834,11 +834,11 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c
int nSteps = cvFloor((tmax - tmin) / tstep);
for (; steps < nSteps; steps++)
{
next += rayStep;
next = v_add(next, rayStep);
v_int32x4 ip = v_round(next);
int ix = ip.get0(); ip = v_rotate_right<1>(ip);
int iy = ip.get0(); ip = v_rotate_right<1>(ip);
int iz = ip.get0();
int ix = v_get0(ip); ip = v_rotate_right<1>(ip);
int iy = v_get0(ip); ip = v_rotate_right<1>(ip);
int iz = v_get0(ip);
int coord = ix * xdim + iy * ydim + iz * zdim;
fnext = tsdfToFloat(volume.at<RGBTsdfVoxel>(coord).tsdf);
@@ -858,7 +858,7 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c
// linearly interpolate t between two f values
if (f > 0.f && fnext < 0.f)
{
v_float32x4 tp = next - rayStep;
v_float32x4 tp = v_sub(next, rayStep);
float ft = interpolateColorVoxel(volume, volDims, neighbourCoords, tp);
float ftdt = interpolateColorVoxel(volume, volDims, neighbourCoords, next);
float ts = tmin + tstep * (steps - ft / (ftdt - ft));
@@ -866,7 +866,7 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c
// avoid division by zero
if (!cvIsNaN(ts) && !cvIsInf(ts))
{
v_float32x4 pv = (orig + dir * v_setall_f32(ts));
v_float32x4 pv = v_add(orig, v_mul(dir, v_setall_f32(ts)));
v_float32x4 nv = getNormalColorVoxel(volume, volDims, neighbourCoords, volResolution, pv);
v_float32x4 cv = getColorVoxel(volume, volDims, neighbourCoords, volResolution, voxelSizeInv, pv);
@@ -876,7 +876,7 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c
//convert pv and nv to camera space
normal = v_matmuladd(nv, volRot0, volRot1, volRot2, v_setzero_f32());
// interpolation optimized a little
point = v_matmuladd(pv * v_float32x4(voxelSize, voxelSize, voxelSize, 1.f),
point = v_matmuladd(v_mul(pv, v_float32x4(voxelSize, voxelSize, voxelSize, 1.f)),
volRot0, volRot1, volRot2, volTrans);
}
}
+10 -10
View File
@@ -630,15 +630,15 @@ inline float interpolate(float tx, float ty, float tz, float vx[8])
v_float32x4 v0246, v1357;
v_load_deinterleave(vx, v0246, v1357);
v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246);
v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246)));
v_float32x4 v00_10 = vxx;
v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx)));
v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10);
float v0 = v0_1.get0();
v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10)));
float v0 = v_get0(v0_1);
v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1)));
float v1 = v0_1.get0();
float v1 = v_get0(v0_1);
return v0 + tx * (v1 - v0);
}
@@ -816,9 +816,9 @@ Point3f getNormalVoxel(
v_float32x4 czp0 = v_lut(vals, idxzp + 0); v_float32x4 czp1 = v_lut(vals, idxzp + 4);
v_float32x4 czn0 = v_lut(vals, idxzn + 0); v_float32x4 czn1 = v_lut(vals, idxzn + 4);
v_float32x4 cxv0 = cxn0 - cxp0; v_float32x4 cxv1 = cxn1 - cxp1;
v_float32x4 cyv0 = cyn0 - cyp0; v_float32x4 cyv1 = cyn1 - cyp1;
v_float32x4 czv0 = czn0 - czp0; v_float32x4 czv1 = czn1 - czp1;
v_float32x4 cxv0 = v_sub(cxn0, cxp0); v_float32x4 cxv1 = v_sub(cxn1, cxp1);
v_float32x4 cyv0 = v_sub(cyn0, cyp0); v_float32x4 cyv1 = v_sub(cyn1, cyp1);
v_float32x4 czv0 = v_sub(czn0, czp0); v_float32x4 czv1 = v_sub(czn1, czp1);
v_store(cxv + 0, cxv0); v_store(cxv + 4, cxv1);
v_store(cyv + 0, cyv0); v_store(cyv + 4, cyv1);
@@ -959,9 +959,9 @@ Point3f ocl_getNormalVoxel(
v_float32x4 czp0 = v_lut(vals, idxzp + 0); v_float32x4 czp1 = v_lut(vals, idxzp + 4);
v_float32x4 czn0 = v_lut(vals, idxzn + 0); v_float32x4 czn1 = v_lut(vals, idxzn + 4);
v_float32x4 cxv0 = cxn0 - cxp0; v_float32x4 cxv1 = cxn1 - cxp1;
v_float32x4 cyv0 = cyn0 - cyp0; v_float32x4 cyv1 = cyn1 - cyp1;
v_float32x4 czv0 = czn0 - czp0; v_float32x4 czv1 = czn1 - czp1;
v_float32x4 cxv0 = v_sub(cxn0, cxp0); v_float32x4 cxv1 = v_sub(cxn1, cxp1);
v_float32x4 cyv0 = v_sub(cyn0, cyp0); v_float32x4 cyv1 = v_sub(cyn1, cyp1);
v_float32x4 czv0 = v_sub(czn0, czp0); v_float32x4 czv1 = v_sub(czn1, czp1);
v_store(cxv + 0, cxv0); v_store(cxv + 4, cxv1);
v_store(cyv + 0, cyv0); v_store(cyv + 4, cyv1);
+23 -23
View File
@@ -1007,7 +1007,7 @@ static inline v_float32x4 crossProduct(const v_float32x4& a, const v_float32x4&
v_float32x4 ayzx, azxy, byzx, bzxy;
getCrossPerm(a, ayzx, azxy);
getCrossPerm(b, byzx, bzxy);
return ayzx * bzxy - azxy * byzx;
return v_sub(v_mul(ayzx, bzxy), v_mul(azxy, byzx));
}
#else
static inline bool fastCheck(const Point3f& p)
@@ -1082,11 +1082,11 @@ struct GetAbInvoker : ParallelLoopBody
//find correspondence by projecting the point
v_float32x4 oldCoords;
float pz = (v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(newP))).get0());
float pz = v_get0(v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(newP))));
// x, y, 0, 0
oldCoords = v_muladd(newP / v_setall_f32(pz), vfxy, vcxy);
oldCoords = v_muladd(v_div(newP, v_setall_f32(pz)), vfxy, vcxy);
if (!v_check_all((oldCoords >= v_setzero_f32()) & (oldCoords < vframe)))
if (!v_check_all(v_and(v_ge(oldCoords, v_setzero_f32()), v_lt(oldCoords, vframe))))
continue;
// bilinearly interpolate oldPts and oldNrm under oldCoords point
@@ -1094,12 +1094,12 @@ struct GetAbInvoker : ParallelLoopBody
v_float32x4 oldN;
{
v_int32x4 ixy = v_floor(oldCoords);
v_float32x4 txy = oldCoords - v_cvt_f32(ixy);
int xi = ixy.get0();
int yi = v_rotate_right<1>(ixy).get0();
v_float32x4 tx = v_setall_f32(txy.get0());
v_float32x4 txy = v_sub(oldCoords, v_cvt_f32(ixy));
int xi = v_get0(ixy);
int yi = v_get0(v_rotate_right<1>(ixy));
v_float32x4 tx = v_setall_f32(v_get0(txy));
txy = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(txy)));
v_float32x4 ty = v_setall_f32(txy.get0());
v_float32x4 ty = v_setall_f32(v_get0(txy));
const float* prow0 = (const float*)oldPts[yi + 0];
const float* prow1 = (const float*)oldPts[yi + 1];
@@ -1122,23 +1122,23 @@ struct GetAbInvoker : ParallelLoopBody
// NaN check is done later
v_float32x4 p0 = p00 + tx * (p01 - p00);
v_float32x4 p1 = p10 + tx * (p11 - p10);
oldP = p0 + ty * (p1 - p0);
v_float32x4 p0 = v_add(p00, v_mul(tx, v_sub(p01, p00)));
v_float32x4 p1 = v_add(p10, v_mul(tx, v_sub(p11, p10)));
oldP = v_add(p0, v_mul(ty, v_sub(p1, p0)));
v_float32x4 n0 = n00 + tx * (n01 - n00);
v_float32x4 n1 = n10 + tx * (n11 - n10);
oldN = n0 + ty * (n1 - n0);
v_float32x4 n0 = v_add(n00, v_mul(tx, v_sub(n01, n00)));
v_float32x4 n1 = v_add(n10, v_mul(tx, v_sub(n11, n10)));
oldN = v_add(n0, v_mul(ty, v_sub(n1, n0)));
}
bool oldPNcheck = fastCheck(oldP, oldN);
//filter by distance
v_float32x4 diff = newP - oldP;
bool distCheck = !(v_reduce_sum(diff * diff) > sqThresh);
v_float32x4 diff = v_sub(newP, oldP);
bool distCheck = !(v_reduce_sum(v_mul(diff, diff)) > sqThresh);
//filter by angle
bool angleCheck = !(abs(v_reduce_sum(newN * oldN)) < cosThresh);
bool angleCheck = !(abs(v_reduce_sum(v_mul(newN, oldN))) < cosThresh);
if (!(oldPNcheck && distCheck && angleCheck))
continue;
@@ -1146,17 +1146,17 @@ struct GetAbInvoker : ParallelLoopBody
// build point-wise vector ab = [ A | b ]
v_float32x4 VxNv = crossProduct(newP, oldN);
Point3f VxN;
VxN.x = VxNv.get0();
VxN.y = v_reinterpret_as_f32(v_extract<1>(v_reinterpret_as_u32(VxNv), v_setzero_u32())).get0();
VxN.z = v_reinterpret_as_f32(v_extract<2>(v_reinterpret_as_u32(VxNv), v_setzero_u32())).get0();
VxN.x = v_get0(VxNv);
VxN.y = v_get0(v_reinterpret_as_f32(v_extract<1>(v_reinterpret_as_u32(VxNv), v_setzero_u32())));
VxN.z = v_get0(v_reinterpret_as_f32(v_extract<2>(v_reinterpret_as_u32(VxNv), v_setzero_u32())));
float dotp = -v_reduce_sum(oldN * diff);
float dotp = -v_reduce_sum(v_mul(oldN, diff));
// build point-wise upper-triangle matrix [ab^T * ab] w/o last row
// which is [A^T*A | A^T*b]
// and gather sum
v_float32x4 vd = VxNv | v_float32x4(0, 0, 0, dotp);
v_float32x4 vd = v_or(VxNv, v_float32x4(0, 0, 0, dotp));
v_float32x4 n = oldN;
v_float32x4 nyzx;
{
+64 -64
View File
@@ -144,35 +144,35 @@ void integrateTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& volu
// optimization of the following:
//Point3f volPt = Point3f(x, y, z)*voxelSize;
//Point3f camSpacePt = vol2cam * volPt;
camSpacePt += zStep;
camSpacePt = v_add(camSpacePt, zStep);
float zCamSpace = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))).get0();
float zCamSpace = v_get0(v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))));
if (zCamSpace <= 0.f)
continue;
v_float32x4 camPixVec = camSpacePt / v_setall_f32(zCamSpace);
v_float32x4 camPixVec = v_div(camSpacePt, v_setall_f32(zCamSpace));
v_float32x4 projected = v_muladd(camPixVec, vfxy, vcxy);
// leave only first 2 lanes
projected = v_reinterpret_as_f32(v_reinterpret_as_u32(projected) &
v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0));
projected = v_reinterpret_as_f32(v_and(v_reinterpret_as_u32(projected),
v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0)));
depthType v;
// bilinearly interpolate depth at projected
{
const v_float32x4& pt = projected;
// check coords >= 0 and < imgSize
v_uint32x4 limits = v_reinterpret_as_u32(pt < v_setzero_f32()) |
v_reinterpret_as_u32(pt >= upLimits);
limits = limits | v_rotate_right<1>(limits);
if (limits.get0())
v_uint32x4 limits = v_or(v_reinterpret_as_u32(v_lt(pt, v_setzero_f32())),
v_reinterpret_as_u32(v_ge(pt, upLimits)));
limits = v_or(limits, v_rotate_right<1>(limits));
if (v_get0(limits))
continue;
// xi, yi = floor(pt)
v_int32x4 ip = v_floor(pt);
v_int32x4 ipshift = ip;
int xi = ipshift.get0();
int xi = v_get0(ipshift);
ipshift = v_rotate_right<1>(ipshift);
int yi = ipshift.get0();
int yi = v_get0(ipshift);
const depthType* row0 = depth[yi + 0];
const depthType* row1 = depth[yi + 1];
@@ -186,17 +186,17 @@ void integrateTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& volu
// assume correct depth is positive
// don't fix missing data
if (v_check_all(vall > v_setzero_f32()))
if (v_check_all(v_gt(vall, v_setzero_f32())))
{
v_float32x4 t = pt - v_cvt_f32(ip);
float tx = t.get0();
v_float32x4 t = v_sub(pt, v_cvt_f32(ip));
float tx = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
v_float32x4 ty = v_setall_f32(t.get0());
v_float32x4 ty = v_setall_f32(v_get0(t));
// vx is y-interpolated between rows 0 and 1
v_float32x4 vx = v001 + ty * (v101 - v001);
float v0 = vx.get0();
v_float32x4 vx = v_add(v001, v_mul(ty, v_sub(v101, v001)));
float v0 = v_get0(vx);
vx = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vx)));
float v1 = vx.get0();
float v1 = v_get0(vx);
v = v0 + tx * (v1 - v0);
}
else
@@ -204,8 +204,8 @@ void integrateTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& volu
}
// norm(camPixVec) produces double which is too slow
int _u = (int)projected.get0();
int _v = (int)v_rotate_right<1>(projected).get0();
int _u = (int)v_get0(projected);
int _v = (int)v_get0(v_rotate_right<1>(projected));
if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows))
continue;
float pixNorm = pixNorms.at<float>(_v, _u);
@@ -408,21 +408,21 @@ inline float interpolateTsdfVoxel(const Mat& volume,
{
// tx, ty, tz = floor(p)
v_int32x4 ip = v_floor(p);
v_float32x4 t = p - v_cvt_f32(ip);
float tx = t.get0();
v_float32x4 t = v_sub(p, v_cvt_f32(ip));
float tx = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
float ty = t.get0();
float ty = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
float tz = t.get0();
float tz = v_get0(t);
int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2];
const TsdfVoxel* volData = volume.ptr<TsdfVoxel>();
int ix = ip.get0();
int ix = v_get0(ip);
ip = v_rotate_right<1>(ip);
int iy = ip.get0();
int iy = v_get0(ip);
ip = v_rotate_right<1>(ip);
int iz = ip.get0();
int iz = v_get0(ip);
int coordBase = ix * xdim + iy * ydim + iz * zdim;
@@ -432,15 +432,15 @@ inline float interpolateTsdfVoxel(const Mat& volume,
v_float32x4 v0246 = tsdfToFloat_INTR(v_int32x4(vx[0], vx[2], vx[4], vx[6]));
v_float32x4 v1357 = tsdfToFloat_INTR(v_int32x4(vx[1], vx[3], vx[5], vx[7]));
v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246);
v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246)));
v_float32x4 v00_10 = vxx;
v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx)));
v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10);
float v0 = v0_1.get0();
v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10)));
float v0 = v_get0(v0_1);
v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1)));
float v1 = v0_1.get0();
float v1 = v_get0(v0_1);
return v0 + tx * (v1 - v0);
}
@@ -495,27 +495,27 @@ inline v_float32x4 getNormalVoxel( const Mat& volume,
const Vec4i& volDims, const Vec8i& neighbourCoords, const Point3i volResolution,
const v_float32x4& p)
{
if (v_check_any(p < v_float32x4(1.f, 1.f, 1.f, 0.f)) ||
v_check_any(p >= v_float32x4((float)(volResolution.x - 2),
if (v_check_any(v_lt(p, v_float32x4(1.f, 1.f, 1.f, 0.f))) ||
v_check_any(v_ge(p, v_float32x4((float)(volResolution.x - 2),
(float)(volResolution.y - 2),
(float)(volResolution.z - 2), 1.f))
(float)(volResolution.z - 2), 1.f)))
)
return nanv;
v_int32x4 ip = v_floor(p);
v_float32x4 t = p - v_cvt_f32(ip);
float tx = t.get0();
v_float32x4 t = v_sub(p, v_cvt_f32(ip));
float tx = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
float ty = t.get0();
float ty = v_get0(t);
t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t)));
float tz = t.get0();
float tz = v_get0(t);
const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2];
const TsdfVoxel* volData = volume.ptr<TsdfVoxel>();
int ix = ip.get0(); ip = v_rotate_right<1>(ip);
int iy = ip.get0(); ip = v_rotate_right<1>(ip);
int iz = ip.get0();
int ix = v_get0(ip); ip = v_rotate_right<1>(ip);
int iy = v_get0(ip); ip = v_rotate_right<1>(ip);
int iz = v_get0(ip);
int coordBase = ix * xdim + iy * ydim + iz * zdim;
@@ -533,23 +533,23 @@ inline v_float32x4 getNormalVoxel( const Mat& volume,
v_float32x4 v0246(vx[0], vx[2], vx[4], vx[6]);
v_float32x4 v1357(vx[1], vx[3], vx[5], vx[7]);
v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246);
v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246)));
v_float32x4 v00_10 = vxx;
v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx)));
v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10);
float v0 = v0_1.get0();
v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10)));
float v0 = v_get0(v0_1);
v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1)));
float v1 = v0_1.get0();
float v1 = v_get0(v0_1);
nv = v0 + tx * (v1 - v0);
}
v_float32x4 n = v_load_aligned(an);
v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(n * n)));
v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(v_mul(n, n))));
return Norm.get0() < 0.0001f ? nanv : n / Norm;
return v_get0(Norm) < 0.0001f ? nanv : v_div(n, Norm);
}
inline Point3f getNormalVoxel( const Mat& volume,
@@ -703,21 +703,21 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera
// get direction through pixel in volume space:
// 1. reproject (x, y) on projecting plane where z = 1.f
v_float32x4 planed = (v_float32x4((float)x, (float)y, 0.f, 0.f) - vcxy) * vfxy;
v_float32x4 planed = v_mul(v_sub(v_float32x4((float)x, (float)y, 0.f, 0.f), vcxy), vfxy);
planed = v_combine_low(planed, v_float32x4(1.f, 0.f, 0.f, 0.f));
// 2. rotate to volume space
planed = v_matmuladd(planed, camRot0, camRot1, camRot2, v_setzero_f32());
// 3. normalize
v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(planed * planed)));
v_float32x4 dir = planed * invNorm;
v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(v_mul(planed, planed))));
v_float32x4 dir = v_mul(planed, invNorm);
// compute intersection of ray with all six bbox planes
v_float32x4 rayinv = v_setall_f32(1.f) / dir;
v_float32x4 rayinv = v_div(v_setall_f32(1.f), dir);
// div by zero should be eliminated by these products
v_float32x4 tbottom = rayinv * (boxDown - orig);
v_float32x4 ttop = rayinv * (boxUp - orig);
v_float32x4 tbottom = v_mul(rayinv, v_sub(boxDown, orig));
v_float32x4 ttop = v_mul(rayinv, v_sub(boxUp, orig));
// re-order intersections to find smallest and largest on each axis
v_float32x4 minAx = v_min(ttop, tbottom);
@@ -738,14 +738,14 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera
if (tmin < tmax)
{
// interpolation optimized a little
orig *= invVoxelSize;
dir *= invVoxelSize;
orig = v_mul(orig, invVoxelSize);
dir = v_mul(dir, invVoxelSize);
int xdim = volDims[0];
int ydim = volDims[1];
int zdim = volDims[2];
v_float32x4 rayStep = dir * v_setall_f32(tstep);
v_float32x4 next = (orig + dir * v_setall_f32(tmin));
v_float32x4 rayStep = v_mul(dir, v_setall_f32(tstep));
v_float32x4 next = v_add(orig, v_mul(dir, v_setall_f32(tmin)));
float f = interpolateTsdfVoxel(volume, volDims, neighbourCoords, next);
float fnext = f;
@@ -754,11 +754,11 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera
int nSteps = cvFloor((tmax - tmin) / tstep);
for (; steps < nSteps; steps++)
{
next += rayStep;
next = v_add(next, rayStep);
v_int32x4 ip = v_round(next);
int ix = ip.get0(); ip = v_rotate_right<1>(ip);
int iy = ip.get0(); ip = v_rotate_right<1>(ip);
int iz = ip.get0();
int ix = v_get0(ip); ip = v_rotate_right<1>(ip);
int iy = v_get0(ip); ip = v_rotate_right<1>(ip);
int iz = v_get0(ip);
int coord = ix * xdim + iy * ydim + iz * zdim;
fnext = tsdfToFloat(volume.at<TsdfVoxel>(coord).tsdf);
@@ -778,7 +778,7 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera
// linearly interpolate t between two f values
if (f > 0.f && fnext < 0.f)
{
v_float32x4 tp = next - rayStep;
v_float32x4 tp = v_sub(next, rayStep);
float ft = interpolateTsdfVoxel(volume, volDims, neighbourCoords, tp);
float ftdt = interpolateTsdfVoxel(volume, volDims, neighbourCoords, next);
float ts = tmin + tstep * (steps - ft / (ftdt - ft));
@@ -786,7 +786,7 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera
// avoid division by zero
if (!cvIsNaN(ts) && !cvIsInf(ts))
{
v_float32x4 pv = (orig + dir * v_setall_f32(ts));
v_float32x4 pv = v_add(orig, v_mul(dir, v_setall_f32(ts)));
v_float32x4 nv = getNormalVoxel(volume, volDims, neighbourCoords, volResolution, pv);
if (!isNaN(nv))
@@ -794,7 +794,7 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera
//convert pv and nv to camera space
normal = v_matmuladd(nv, volRot0, volRot1, volRot2, v_setzero_f32());
// interpolation optimized a little
point = v_matmuladd(pv * v_float32x4(voxelSize, voxelSize, voxelSize, 1.f),
point = v_matmuladd(v_mul(pv, v_float32x4(voxelSize, voxelSize, voxelSize, 1.f)),
volRot0, volRot1, volRot2, volTrans);
}
}
+1 -1
View File
@@ -45,7 +45,7 @@ typedef Vec<uchar, sizeof(RGBTsdfVoxel)> VecRGBTsdfVoxel;
inline v_float32x4 tsdfToFloat_INTR(const v_int32x4& num)
{
v_float32x4 num128 = v_setall_f32(-1.f / 128.f);
return v_cvt_f32(num) * num128;
return v_mul(v_cvt_f32(num), num128);
}
#endif
+1 -1
View File
@@ -71,7 +71,7 @@ inline bool isNaN(cv::Point3f p)
#if USE_INTRINSICS
static inline bool isNaN(const cv::v_float32x4& p)
{
return cv::v_check_any(p != p);
return cv::v_check_any(v_ne(p, p));
}
#endif
+12 -12
View File
@@ -645,11 +645,11 @@ public:
v_float32 v_d = vx_setall_f32(d);
float* errors_cache_ptr = errors_cache.data();
for (; i <= pts_cnt - v_float32::nlanes; i += v_float32::nlanes)
for (; i <= pts_cnt - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes())
{
v_float32 v_error = v_a * vx_load(pts_ptr_x + i) + v_b * vx_load(pts_ptr_y + i)
+ v_c * vx_load(pts_ptr_z + i) + v_d;
v_store(errors_cache_ptr + i, v_error * v_error);
v_float32 v_error = v_add(v_add(v_mul(v_a, vx_load(pts_ptr_x + i)), v_mul(v_b, vx_load(pts_ptr_y + i))),
v_add(v_mul(v_c, vx_load(pts_ptr_z + i)), v_d));
v_store(errors_cache_ptr + i, v_mul(v_error, v_error));
}
#endif
for (; i < pts_cnt; ++i) {
@@ -725,16 +725,16 @@ public:
v_float32 v_radius = vx_setall_f32(radius);
float* errors_cache_ptr = errors_cache.data();
for (; i <= pts_cnt - v_float32::nlanes; i += v_float32::nlanes)
for (; i <= pts_cnt - VTraits<v_float32>::vlanes(); i += VTraits<v_float32>::vlanes())
{
v_float32 v_diff_x = v_center_x - vx_load(pts_ptr_x + i);
v_float32 v_diff_y = v_center_y - vx_load(pts_ptr_y + i);
v_float32 v_diff_z = v_center_z - vx_load(pts_ptr_z + i);
v_float32 v_diff_x = v_sub(v_center_x, vx_load(pts_ptr_x + i));
v_float32 v_diff_y = v_sub(v_center_y, vx_load(pts_ptr_y + i));
v_float32 v_diff_z = v_sub(v_center_z, vx_load(pts_ptr_z + i));
v_float32 v_distance_from_center = v_sqrt(v_diff_x * v_diff_x +
v_diff_y * v_diff_y + v_diff_z * v_diff_z);
v_float32 v_diff_dist = v_distance_from_center - v_radius;
v_store(errors_cache_ptr + i, v_diff_dist * v_diff_dist);
v_float32 v_distance_from_center = v_sqrt(v_add(v_add(v_mul(v_diff_x, v_diff_x),
v_mul(v_diff_y, v_diff_y)), v_mul(v_diff_z, v_diff_z)));
v_float32 v_diff_dist = v_sub(v_distance_from_center, v_radius);
v_store(errors_cache_ptr + i, v_mul(v_diff_dist, v_diff_dist));
}
#endif
for (; i < pts_cnt; ++i)
@@ -172,6 +172,11 @@
# define CV_MSA 1
#endif
#ifdef CV_CPU_COMPILE_LSX
# include <lsxintrin.h>
# define CV_LSX 1
#endif
#ifdef CV_CPU_COMPILE_LASX
# include <lasxintrin.h>
# define CV_LASX 1
@@ -376,6 +381,10 @@ struct VZeroUpperGuard {
# define CV_RVV 0
#endif
#ifndef CV_LSX
# define CV_LSX 0
#endif
#ifndef CV_LASX
# define CV_LASX 0
#endif
@@ -525,6 +525,27 @@
#endif
#define __CV_CPU_DISPATCH_CHAIN_RVV(fn, args, mode, ...) CV_CPU_CALL_RVV(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_LSX
# define CV_TRY_LSX 1
# define CV_CPU_FORCE_LSX 1
# define CV_CPU_HAS_SUPPORT_LSX 1
# define CV_CPU_CALL_LSX(fn, args) return (cpu_baseline::fn args)
# define CV_CPU_CALL_LSX_(fn, args) return (opt_LSX::fn args)
#elif !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_DISPATCH_COMPILE_LSX
# define CV_TRY_LSX 1
# define CV_CPU_FORCE_LSX 0
# define CV_CPU_HAS_SUPPORT_LSX (cv::checkHardwareSupport(CV_CPU_LSX))
# define CV_CPU_CALL_LSX(fn, args) if (CV_CPU_HAS_SUPPORT_LSX) return (opt_LSX::fn args)
# define CV_CPU_CALL_LSX_(fn, args) if (CV_CPU_HAS_SUPPORT_LSX) return (opt_LSX::fn args)
#else
# define CV_TRY_LSX 0
# define CV_CPU_FORCE_LSX 0
# define CV_CPU_HAS_SUPPORT_LSX 0
# define CV_CPU_CALL_LSX(fn, args)
# define CV_CPU_CALL_LSX_(fn, args)
#endif
#define __CV_CPU_DISPATCH_CHAIN_LSX(fn, args, mode, ...) CV_CPU_CALL_LSX(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_LASX
# define CV_TRY_LASX 1
# define CV_CPU_FORCE_LASX 1
+8 -2
View File
@@ -269,6 +269,8 @@ namespace cv {
#define CV_CPU_NEON 100
#define CV_CPU_NEON_DOTPROD 101
#define CV_CPU_NEON_FP16 102
#define CV_CPU_NEON_BF16 103
#define CV_CPU_MSA 150
@@ -279,7 +281,8 @@ namespace cv {
#define CV_CPU_RVV 210
#define CV_CPU_LASX 230
#define CV_CPU_LSX 230
#define CV_CPU_LASX 231
// CPU features groups
#define CV_CPU_AVX512_SKX 256
@@ -328,6 +331,8 @@ enum CpuFeatures {
CPU_NEON = 100,
CPU_NEON_DOTPROD = 101,
CPU_NEON_FP16 = 102,
CPU_NEON_BF16 = 103,
CPU_MSA = 150,
@@ -338,7 +343,8 @@ enum CpuFeatures {
CPU_RVV = 210,
CPU_LASX = 230,
CPU_LSX = 230,
CPU_LASX = 231,
CPU_AVX512_SKX = 256, //!< Skylake-X with AVX-512F/CD/BW/DQ/VL
CPU_AVX512_COMMON = 257, //!< Common instructions AVX-512F/CD for all CPUs that support AVX-512
@@ -206,7 +206,7 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE;
# undef CV_RVV
#endif
#if (CV_SSE2 || CV_NEON || CV_VSX || CV_MSA || CV_WASM_SIMD || CV_RVV071) && !defined(CV_FORCE_SIMD128_CPP)
#if (CV_SSE2 || CV_NEON || CV_VSX || CV_MSA || CV_WASM_SIMD || CV_RVV071 || CV_LSX) && !defined(CV_FORCE_SIMD128_CPP)
#define CV__SIMD_FORWARD 128
#include "opencv2/core/hal/intrin_forward.hpp"
#endif
@@ -242,6 +242,10 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE;
#include "opencv2/core/hal/intrin_rvv.hpp"
#endif
#elif CV_LSX && !defined(CV_FORCE_SIMD128_CPP)
#include "opencv2/core/hal/intrin_lsx.hpp"
#elif CV_LASX
#if !defined(CV_FORCE_SIMD128_CPP)
#define CV_FORCE_SIMD128_CPP 1
@@ -738,7 +742,7 @@ namespace CV__SIMD_NAMESPACE {
/** @brief SIMD processing state cleanup call */
inline void vx_cleanup() { VXPREFIX(_cleanup)(); }
#if !CV_SIMD_SCALABLE
#if !CV_SIMD_SCALABLE && !(CV_NEON && !defined(CV_FORCE_SIMD128_CPP))
// Compatibility layer
template<typename T> struct VTraits {
@@ -1166,6 +1170,74 @@ namespace CV__SIMD_NAMESPACE {
#endif //!CV_SIMD_SCALABLE
#if (CV_NEON /* || CV_others */) && !defined(CV_FORCE_SIMD128_CPP)
// Compatibility layer for the backend that cleaned up.
#define OPENCV_HAL_WRAP_BIN_OP_ADDSUB(_Tpvec) \
template<typename... Args> \
inline _Tpvec v_add(const _Tpvec& f1, const _Tpvec& f2, const Args&... vf) { \
return v_add(v_add(f1, f2), vf...); \
}
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint8)
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint16)
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint32)
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint64)
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int8)
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int16)
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int32)
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int64)
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float32)
#if CV_SIMD_64F
OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float64)
#endif
#define OPENCV_HAL_WRAP_BIN_OP_MUL(_Tpvec) \
template<typename... Args> \
inline _Tpvec v_mul(const _Tpvec& f1, const _Tpvec& f2, const Args&... vf) { \
return v_mul(v_mul(f1, f2), vf...); \
}
OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint8)
OPENCV_HAL_WRAP_BIN_OP_MUL(v_int8)
OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint16)
OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint32)
OPENCV_HAL_WRAP_BIN_OP_MUL(v_int16)
OPENCV_HAL_WRAP_BIN_OP_MUL(v_int32)
OPENCV_HAL_WRAP_BIN_OP_MUL(v_float32)
#if CV_SIMD_64F
OPENCV_HAL_WRAP_BIN_OP_MUL(v_float64)
#endif
#define OPENCV_HAL_WRAP_EXTRACT(_Tpvec) \
inline typename VTraits<_Tpvec>::lane_type v_extract_highest(const _Tpvec& v) \
{ \
return v_extract_n<VTraits<_Tpvec>::nlanes-1>(v); \
}
OPENCV_HAL_WRAP_EXTRACT(v_uint8)
OPENCV_HAL_WRAP_EXTRACT(v_int8)
OPENCV_HAL_WRAP_EXTRACT(v_uint16)
OPENCV_HAL_WRAP_EXTRACT(v_int16)
OPENCV_HAL_WRAP_EXTRACT(v_uint32)
OPENCV_HAL_WRAP_EXTRACT(v_int32)
OPENCV_HAL_WRAP_EXTRACT(v_uint64)
OPENCV_HAL_WRAP_EXTRACT(v_int64)
OPENCV_HAL_WRAP_EXTRACT(v_float32)
#if CV_SIMD_64F
OPENCV_HAL_WRAP_EXTRACT(v_float64)
#endif
#define OPENCV_HAL_WRAP_BROADCAST(_Tpvec) \
inline _Tpvec v_broadcast_highest(const _Tpvec& v) \
{ \
return v_broadcast_element<VTraits<_Tpvec>::nlanes-1>(v); \
}
OPENCV_HAL_WRAP_BROADCAST(v_uint32)
OPENCV_HAL_WRAP_BROADCAST(v_int32)
OPENCV_HAL_WRAP_BROADCAST(v_float32)
#endif //CV_NEON
//! @cond IGNORED
// backward compatibility
@@ -96,54 +96,22 @@ inline __m256d _v256_setall_pd(double f64)
inline __m256i _lasx_packus_h(const __m256i& a, const __m256i& b)
{
__m256i u8min = __lasx_xvreplgr2vr_h(0);
__m256i u8max = __lasx_xvreplgr2vr_h(255);
__m256i sat_a = __lasx_xvmax_h(a, u8min);
sat_a = __lasx_xvmin_h(sat_a, u8max);
__m256i sat_b = __lasx_xvmax_h(b, u8min);
sat_b = __lasx_xvmin_h(sat_b, u8max);
__m256i byteIndex = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30);
return __lasx_xvshuf_b(sat_b, sat_a, byteIndex);
return __lasx_xvssrarni_bu_h(b, a, 0);
}
inline __m256i _lasx_packs_h(const __m256i& a, const __m256i& b)
{
__m256i s8min = __lasx_xvreplgr2vr_h(-128);
__m256i s8max = __lasx_xvreplgr2vr_h(127);
__m256i sat_a = __lasx_xvmax_h(a, s8min);
sat_a = __lasx_xvmin_h(sat_a, s8max);
__m256i sat_b = __lasx_xvmax_h(b, s8min);
sat_b = __lasx_xvmin_h(sat_b, s8max);
__m256i byteIndex = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30);
return __lasx_xvshuf_b(sat_b, sat_a, byteIndex);
return __lasx_xvssrarni_b_h(b, a, 0);
}
inline __m256i _lasx_packus_w(const __m256i& a, const __m256i& b)
{
__m256i u16min = __lasx_xvreplgr2vr_w(0);
__m256i u16max = __lasx_xvreplgr2vr_w(0xffff);
__m256i sat_a = __lasx_xvmax_w(a, u16min);
sat_a = __lasx_xvmin_w(sat_a, u16max);
__m256i sat_b = __lasx_xvmax_w(b, u16min);
sat_b = __lasx_xvmin_w(sat_b, u16max);
__m256i hwordIndex = _v256_setr_h(0, 2, 4, 6, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14);
return __lasx_xvshuf_h(hwordIndex, sat_b, sat_a);
return __lasx_xvssrarni_hu_w(b, a, 0);
}
inline __m256i _lasx_packs_w(const __m256i& a, const __m256i& b)
{
__m256i s16min = __lasx_xvreplgr2vr_w(-0x8000);
__m256i s16max = __lasx_xvreplgr2vr_w(0x7fff);
__m256i sat_a = __lasx_xvmax_w(a, s16min);
sat_a = __lasx_xvmin_w(sat_a, s16max);
__m256i sat_b = __lasx_xvmax_w(b, s16min);
sat_b = __lasx_xvmin_w(sat_b, s16max);
__m256i hwordIndex = _v256_setr_h(0, 2, 4, 6, 8, 10, 12, 14,
0, 2, 4, 6, 8, 10, 12, 14);
return __lasx_xvshuf_h(hwordIndex, sat_b, sat_a);
return __lasx_xvssrarni_h_w(b, a, 0);
}
inline __m256i _v256_combine(const __m128i& lo, const __m128i& hi)
@@ -191,7 +159,7 @@ inline _Tpvec v256_permute4x64(const _Tpvec& a)
{ return _Tpvec(_v256_permute4x64<imm>(a.val)); }
inline __m128i _v256_extract_high(const __m256i& v)
{ __m256i temp256i = __lasx_xvpermi_q(v, v, 0x31);
{ __m256i temp256i = __lasx_xvpermi_d(v, 0x4E);
return *((__m128i*)&temp256i); }
inline __m128 _v256_extract_high(const __m256& v)
@@ -211,10 +179,7 @@ inline __m128d _v256_extract_low(const __m256d& v)
inline __m256i _v256_packs_epu32(const __m256i& a, const __m256i& b)
{
const __m256i maxv = __lasx_xvreplgr2vr_w(65535);
__m256i am = __lasx_xvmin_wu(a, maxv);
__m256i bm = __lasx_xvmin_wu(b, maxv);
return _lasx_packus_w(am, bm);
return __lasx_xvssrlrni_hu_w(b, a, 0);
}
template<int i>
@@ -869,14 +834,11 @@ OPENCV_HAL_IMPL_LASX_BIN_FUNC(v_mul_wrap, v_int16x16, __lasx_xvmul_h)
inline v_uint8x32 v_mul_wrap(const v_uint8x32& a, const v_uint8x32& b)
{
__m256i ad = __lasx_xvsrai_h(a.val, 8);
__m256i bd = __lasx_xvsrai_h(b.val, 8);
__m256i p0 = __lasx_xvmul_h(a.val, b.val);
__m256i p1 = __lasx_xvslli_h(__lasx_xvmul_h(ad, bd), 8);
const __m256i b01 = __lasx_xvreplgr2vr_w(0xFF00FF00);
return v_uint8x32(__lasx_xvbitsel_v(p0, p1, b01));
__m256i p0 = __lasx_xvmulwev_h_bu(a.val, b.val);
__m256i p1 = __lasx_xvmulwod_h_bu(a.val, b.val);
return v_uint8x32(__lasx_xvpackev_b(p1, p0));
}
inline v_int8x32 v_mul_wrap(const v_int8x32& a, const v_int8x32& b)
{
return v_reinterpret_as_s8(v_mul_wrap(v_reinterpret_as_u8(a), v_reinterpret_as_u8(b)));
@@ -963,14 +925,7 @@ inline v_uint16x16 v_mul_hi(const v_uint16x16& a, const v_uint16x16& b) { return
OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint16x16, v_int16x16, h, __lasx_xvsra_h)
OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint32x8, v_int32x8, w, __lasx_xvsra_w)
inline __m256i _v256_srai_dx(const __m256i a, const __m256i shift)
{
__m256i d = __lasx_xvreplgr2vr_d((int64)1 << 63);
__m256i r = __lasx_xvsrl_d(__lasx_xvadd_d(a, d), shift);
return __lasx_xvsub_d(r, __lasx_xvsrl_d(d, shift));
}
OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, _v256_srai_dx)
OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, __lasx_xvsra_d)
/** Bitwise logic **/
@@ -979,7 +934,7 @@ OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, _v256_srai_dx)
OPENCV_HAL_IMPL_LASX_BIN_OP(|, _Tpvec, __lasx_xvor_##suffix) \
OPENCV_HAL_IMPL_LASX_BIN_OP(^, _Tpvec, __lasx_xvxor_##suffix) \
inline _Tpvec operator ~ (const _Tpvec& a) \
{ return _Tpvec(__lasx_xvxor_##suffix(a.val, not_const)); }
{ return _Tpvec(__lasx_xvnori_b(a.val, 0)); }
OPENCV_HAL_IMPL_LASX_LOGIC_OP(v_uint8x32, v, __lasx_xvreplgr2vr_w(-1))
OPENCV_HAL_IMPL_LASX_LOGIC_OP(v_int8x32, v, __lasx_xvreplgr2vr_w(-1))
@@ -1224,11 +1179,9 @@ inline v_int8x32 v_reverse(const v_int8x32 &a)
inline v_uint16x16 v_reverse(const v_uint16x16 &a)
{
static const __m256i perm = _v256_setr_b(
14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1,
14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1);
__m256i vec = __lasx_xvshuf_b(a.val, a.val, perm);
return v_uint16x16(__lasx_xvpermi_q(vec, vec, 1));
__m256i vec = __lasx_xvshuf4i_h(a.val, 0x1B);
vec = __lasx_xvshuf4i_w(vec, 0x4E);
return v_uint16x16(__lasx_xvpermi_d(vec, 0x4E));
}
inline v_int16x16 v_reverse(const v_int16x16 &a)
@@ -1236,8 +1189,8 @@ inline v_int16x16 v_reverse(const v_int16x16 &a)
inline v_uint32x8 v_reverse(const v_uint32x8 &a)
{
static const __m256i perm = _v256_setr_w(7, 6, 5, 4, 3, 2, 1, 0);
return v_uint32x8(__lasx_xvperm_w(a.val, perm));
__m256i vec = __lasx_xvshuf4i_w(a.val, 0x1B);
return v_uint32x8(__lasx_xvpermi_d(vec, 0x4E));
}
inline v_int32x8 v_reverse(const v_int32x8 &a)
@@ -1266,17 +1219,19 @@ inline unsigned v_reduce_sum(const v_uint8x32& a)
__m256i t1 = __lasx_xvhaddw_hu_bu(a.val, a.val);
__m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1);
__m256i t3 = __lasx_xvhaddw_du_wu(t2, t2);
return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]);
__m256i t4 = __lasx_xvhaddw_qu_du(t3, t3);
return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]);
}
inline int v_reduce_sum(const v_int8x32& a)
{
__m256i t1 = __lasx_xvhaddw_h_b(a.val, a.val);
__m256i t2 = __lasx_xvhaddw_w_h(t1, t1);
__m256i t3 = __lasx_xvhaddw_d_w(t2, t2);
return (int)(((v4i64)t3)[0]+((v4i64)t3)[1]+((v4i64)t3)[2]+((v4i64)t3)[3]);
__m256i t4 = __lasx_xvhaddw_q_d(t3, t3);
return (int)(((v8i32)t4)[0]+((v8i32)t4)[4]);
}
#define OPENCV_HAL_IMPL_LASX_REDUCE_32(_Tpvec, sctype, func, intrin) \
inline sctype v_reduce_##func(const _Tpvec& a) \
{ \
@@ -1344,7 +1299,8 @@ OPENCV_HAL_IMPL_LASX_REDUCE_FLT(max, __lsx_vfmax_s)
inline int v_reduce_sum(const v_int32x8& a)
{
__m256i t1 = __lasx_xvhaddw_d_w(a.val, a.val);
return (int)(((v4i64)t1)[0]+((v4i64)t1)[1]+((v4i64)t1)[2]+((v4i64)t1)[3]);
__m256i t2 = __lasx_xvhaddw_q_d(t1, t1);
return (int)(((v8i32)t2)[0]+((v8i32)t2)[4]);
}
inline unsigned v_reduce_sum(const v_uint32x8& a)
@@ -1367,13 +1323,13 @@ inline float v_reduce_sum(const v_float32x8& a)
inline uint64 v_reduce_sum(const v_uint64x4& a)
{
uint64 *pa = (uint64*)&a;
return pa[0] + pa[1] + pa[2] + pa[3];
__m256i t0 = __lasx_xvhaddw_qu_du(a.val, a.val);
return (uint64)(((v4u64)t0)[0] + ((v4u64)t0)[2]);
}
inline int64 v_reduce_sum(const v_int64x4& a)
{
int64 *pa = (int64*)&a;
return pa[0] + pa[1] + pa[2] + pa[3];
__m256i t0 = __lasx_xvhaddw_q_d(a.val, a.val);
return (int64)(((v4i64)t0)[0] + ((v4i64)t0)[2]);
}
inline double v_reduce_sum(const v_float64x4& a)
{
@@ -1406,7 +1362,8 @@ inline unsigned v_reduce_sad(const v_uint8x32& a, const v_uint8x32& b)
__m256i t1 = __lasx_xvhaddw_hu_bu(t0, t0);
__m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1);
__m256i t3 = __lasx_xvhaddw_du_wu(t2, t2);
return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]);
__m256i t4 = __lasx_xvhaddw_qu_du(t3, t3);
return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]);
}
inline unsigned v_reduce_sad(const v_int8x32& a, const v_int8x32& b)
{
@@ -1414,7 +1371,8 @@ inline unsigned v_reduce_sad(const v_int8x32& a, const v_int8x32& b)
__m256i t1 = __lasx_xvhaddw_hu_bu(t0, t0);
__m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1);
__m256i t3 = __lasx_xvhaddw_du_wu(t2, t2);
return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]);
__m256i t4 = __lasx_xvhaddw_qu_du(t3, t3);
return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]);
}
inline unsigned v_reduce_sad(const v_uint16x16& a, const v_uint16x16& b)
{
@@ -1445,36 +1403,13 @@ inline float v_reduce_sad(const v_float32x8& a, const v_float32x8& b)
/** Popcount **/
inline v_uint8x32 v_popcount(const v_uint8x32& a)
{
__m256i _popcnt_table = _v256_setr_b(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);
__m256i _popcnt_mask = __lasx_xvreplgr2vr_b(0x0F);
return v_uint8x32(__lasx_xvadd_b(__lasx_xvshuf_b(_popcnt_table, _popcnt_table, __lasx_xvand_v(a.val, _popcnt_mask)),
__lasx_xvshuf_b(_popcnt_table, _popcnt_table, __lasx_xvand_v(__lasx_xvsrli_h(a.val, 4), _popcnt_mask))));
}
{ return v_uint8x32(__lasx_xvpcnt_b(a.val)); }
inline v_uint16x16 v_popcount(const v_uint16x16& a)
{
v_uint8x32 p = v_popcount(v_reinterpret_as_u8(a));
p += v_rotate_right<1>(p);
return v_reinterpret_as_u16(p) & v_uint16x16(__lasx_xvreplgr2vr_h(0x00ff));
}
{ return v_uint16x16(__lasx_xvpcnt_h(a.val)); }
inline v_uint32x8 v_popcount(const v_uint32x8& a)
{
v_uint8x32 p = v_popcount(v_reinterpret_as_u8(a));
p += v_rotate_right<1>(p);
p += v_rotate_right<2>(p);
return v_reinterpret_as_u32(p) & v_uint32x8(__lasx_xvreplgr2vr_w(0x000000ff));
}
{ return v_uint32x8(__lasx_xvpcnt_w(a.val)); }
inline v_uint64x4 v_popcount(const v_uint64x4& a)
{
v_uint8x32 atemp = v_popcount(v_reinterpret_as_u8(a));
uint8_t *pa = (uint8_t*)&atemp;
uint64 v[4];
for (int i = 0; i < 4; ++i) {
v[i] = pa[i*8] + pa[i*8+1] + pa[i*8+2] + pa[i*8+3] + pa[i*8+4] + pa[i*8+5] + pa[i*8+6] + pa[i*8+7];
}
return v_uint64x4(v[0], v[1], v[2], v[3]);
}
{ return v_uint64x4(__lasx_xvpcnt_d(a.val)); }
inline v_uint8x32 v_popcount(const v_int8x32& a)
{ return v_popcount(v_reinterpret_as_u8(a)); }
inline v_uint16x16 v_popcount(const v_int16x16& a)
@@ -1500,10 +1435,9 @@ OPENCV_HAL_IMPL_REINTERPRET_INT(double, int64)
inline int v_signmask(const v_int8x32& a)
{
int mask = 0;
int8_t *pa = (int8_t*)&a;
for( int i = 0; i < 32; i++ )
mask |= (reinterpret_int(pa[i]) < 0) << i;
__m256i result = __lasx_xvmskltz_b(a.val);
int mask = __lasx_xvpickve2gr_w(result, 0);
mask |= (__lasx_xvpickve2gr_w(result, 4) << 16);
return mask;
}
inline int v_signmask(const v_uint8x32& a)
@@ -1516,10 +1450,9 @@ inline int v_signmask(const v_uint16x16& a)
inline int v_signmask(const v_int32x8& a)
{
int mask = 0;
int *pa = (int*)&a;
for( int i = 0; i < 8; i++ )
mask |= (pa[i] < 0) << i;
__m256i result = __lasx_xvmskltz_w(a.val);
int mask = __lasx_xvpickve2gr_w(result, 0);
mask |= (__lasx_xvpickve2gr_w(result, 4) << 4);
return mask;
}
inline int v_signmask(const v_uint32x8& a)
@@ -1527,10 +1460,9 @@ inline int v_signmask(const v_uint32x8& a)
inline int v_signmask(const v_int64x4& a)
{
int mask = 0;
int64 *pa = (int64*)&a;
for( int i = 0; i < 4; i++ )
mask |= (pa[i] < 0) << i;
__m256i result = __lasx_xvmskltz_d(a.val);
int mask = __lasx_xvpickve2gr_d(result, 0);
mask |= (__lasx_xvpickve2gr_w(result, 4) << 2);
return mask;
}
inline int v_signmask(const v_uint64x4& a)
@@ -1592,7 +1524,7 @@ OPENCV_HAL_IMPL_LASX_MULADD(v_float64x4, d)
inline v_int32x8 v_fma(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c)
{
return a * b + c;
return v_int32x8(__lasx_xvmadd_w(c.val, a.val, b.val));
}
inline v_int32x8 v_muladd(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c)
@@ -1601,17 +1533,10 @@ inline v_int32x8 v_muladd(const v_int32x8& a, const v_int32x8& b, const v_int32x
}
inline v_float32x8 v_invsqrt(const v_float32x8& x)
{
v_float32x8 half = x * v_float32x8(0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5);
v_float32x8 t = v_float32x8(__lasx_xvfrsqrt_s(x.val));
t *= v_float32x8(1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5) - ((t * t) * half);
return t;
}
{ return v_float32x8(__lasx_xvfrsqrt_s(x.val)); }
inline v_float64x4 v_invsqrt(const v_float64x4& x)
{
return v_float64x4(1., 1., 1., 1.) / v_sqrt(x);
}
{ return v_float64x4(__lasx_xvfrsqrt_d(x.val)); }
/** Absolute values **/
#define OPENCV_HAL_IMPL_LASX_ABS(_Tpvec, suffix) \
@@ -1629,28 +1554,18 @@ inline v_float64x4 v_abs(const v_float64x4& x)
/** Absolute difference **/
inline v_uint8x32 v_absdiff(const v_uint8x32& a, const v_uint8x32& b)
{ return v_add_wrap(a - b, b - a); }
{ return (v_uint8x32)__lasx_xvabsd_bu(a.val, b.val); }
inline v_uint16x16 v_absdiff(const v_uint16x16& a, const v_uint16x16& b)
{ return v_add_wrap(a - b, b - a); }
{ return (v_uint16x16)__lasx_xvabsd_hu(a.val, b.val); }
inline v_uint32x8 v_absdiff(const v_uint32x8& a, const v_uint32x8& b)
{ return v_max(a, b) - v_min(a, b); }
{ return (v_uint32x8)__lasx_xvabsd_wu(a.val, b.val); }
inline v_uint8x32 v_absdiff(const v_int8x32& a, const v_int8x32& b)
{
v_int8x32 d = v_sub_wrap(a, b);
v_int8x32 m = a < b;
return v_reinterpret_as_u8(v_sub_wrap(d ^ m, m));
}
{ return (v_uint8x32)__lasx_xvabsd_b(a.val, b.val); }
inline v_uint16x16 v_absdiff(const v_int16x16& a, const v_int16x16& b)
{ return v_reinterpret_as_u16(v_sub_wrap(v_max(a, b), v_min(a, b))); }
{ return (v_uint16x16)__lasx_xvabsd_h(a.val, b.val); }
inline v_uint32x8 v_absdiff(const v_int32x8& a, const v_int32x8& b)
{
v_int32x8 d = a - b;
v_int32x8 m = a < b;
return v_reinterpret_as_u32((d ^ m) - m);
}
{ return (v_uint32x8)__lasx_xvabsd_w(a.val, b.val); }
inline v_float32x8 v_absdiff(const v_float32x8& a, const v_float32x8& b)
{ return v_abs(a - b); }
@@ -1740,28 +1655,8 @@ inline v_float64x4 v_cvt_f64_high(const v_float32x8& a)
return v_float64x4(__lasx_xvfcvtl_d_s((__m256)ahigh));
}
// from (Mysticial and wim) https://stackoverflow.com/q/41144668
inline v_float64x4 v_cvt_f64(const v_int64x4& v)
{
// constants encoded as floating-point
__m256i magic_i_lo = __lasx_xvreplgr2vr_d(0x4330000000000000);
__m256i magic_i_hi32 = __lasx_xvreplgr2vr_d(0x4530000080000000);
__m256i magic_i_all = __lasx_xvreplgr2vr_d(0x4530000080100000);
__m256d magic_d_all = _lasx_256_castsi256_pd(magic_i_all);
// Blend the 32 lowest significant bits of v with magic_int_lo
__m256i mask = _v256_set_w(0, -1, 0, -1, 0, -1, 0, -1);
__m256i v_lo = __lasx_xvbitsel_v(magic_i_lo, v.val, mask);
// Extract the 32 most significant bits of v
__m256i v_hi = __lasx_xvsrli_d(v.val, 32);
// Flip the msb of v_hi and blend with 0x45300000
v_hi = __lasx_xvxor_v(v_hi, magic_i_hi32);
// Compute in double precision
__m256d v_hi_dbl = __lasx_xvfsub_d(_lasx_256_castsi256_pd(v_hi), magic_d_all);
// (v_hi - magic_d_all) + v_lo Do not assume associativity of floating point addition
__m256d result = __lasx_xvfadd_d(v_hi_dbl, _lasx_256_castsi256_pd(v_lo));
return v_float64x4(result);
}
{ return v_float64x4(__lasx_xvffint_d_l(v.val)); }
////////////// Lookup table access ////////////////////
@@ -1967,11 +1862,9 @@ inline v_float32x8 v_interleave_pairs(const v_float32x8& vec)
inline v_int8x32 v_pack_triplets(const v_int8x32& vec)
{
__m256i vzero = __lasx_xvreplgr2vr_w(0);
__m256i t1 = __lasx_xvshuf_b(vec.val, vec.val,
_v256_set_d(0xffffff0f0e0d0c0a, 0x0908060504020100, 0xffffff0f0e0d0c0a, 0x0908060504020100));
__m256i t2 = __lasx_xvshuf_b(vzero, t1,
_v256_set_d(0x1211100c0b0a0908, 0x0706050403020100, 0x1211100c0b0a0908, 0x0706050403020100));
return v_int8x32(__lasx_xvperm_w(t2,
__m256i t1 = __lasx_xvshuf_b(vzero, vec.val,
_v256_set_d(0x1211100f0e0d0c0a, 0x0908060504020100, 0x1211100f0e0d0c0a, 0x0908060504020100));
return v_int8x32(__lasx_xvperm_w(t1,
_v256_set_d(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000)));
}
inline v_uint8x32 v_pack_triplets(const v_uint8x32& vec)
@@ -1980,11 +1873,9 @@ inline v_uint8x32 v_pack_triplets(const v_uint8x32& vec)
inline v_int16x16 v_pack_triplets(const v_int16x16& vec)
{
__m256i vzero = __lasx_xvreplgr2vr_w(0);
__m256i t1 = __lasx_xvshuf_b(vec.val, vec.val,
_v256_set_d(0xffff0f0e0d0c0b0a, 0x0908050403020100, 0xffff0f0e0d0c0b0a, 0x0908050403020100));
__m256i t2 = __lasx_xvshuf_b(vzero, t1,
_v256_set_d(0x11100d0c0b0a0908, 0x0706050403020100, 0x11100d0c0b0a0908, 0x0706050403020100));
return v_int16x16(__lasx_xvperm_w(t2,
__m256i t1 = __lasx_xvshuf_b(vzero, vec.val,
_v256_set_d(0x11100f0e0d0c0b0a, 0x0908050403020100, 0x11100f0e0d0c0b0a, 0x0908050403020100));
return v_int16x16(__lasx_xvperm_w(t1,
_v256_set_d(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000)));
}
inline v_uint16x16 v_pack_triplets(const v_uint16x16& vec)
@@ -2018,24 +1909,21 @@ inline v_int32x8 v_dotprod(const v_int16x16& a, const v_int16x16& b, const v_int
inline v_int64x4 v_dotprod(const v_int32x8& a, const v_int32x8& b)
{
__m256i even = __lasx_xvmulwev_d_w(a.val, b.val);
__m256i odd = __lasx_xvmulwod_d_w(a.val, b.val);
return v_int64x4(__lasx_xvadd_d(even, odd));
return v_int64x4(__lasx_xvmaddwod_d_w(even, a.val, b.val));
}
inline v_int64x4 v_dotprod(const v_int32x8& a, const v_int32x8& b, const v_int64x4& c)
{ return v_dotprod(a, b) + c; }
{
__m256i even = __lasx_xvmaddwev_d_w(c.val, a.val, b.val);
return v_int64x4(__lasx_xvmaddwod_d_w(even, a.val, b.val));
}
// 8 >> 32
inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b)
{
__m256i even_m = __lasx_xvreplgr2vr_w(0xFF00FF00);
__m256i even_a = __lasx_xvbitsel_v(a.val, __lasx_xvreplgr2vr_d(0), even_m);
__m256i odd_a = __lasx_xvsrli_h(a.val, 8);
__m256i even_b = __lasx_xvbitsel_v(b.val, __lasx_xvreplgr2vr_d(0), even_m);
__m256i odd_b = __lasx_xvsrli_h(b.val, 8);
__m256i prod0 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(even_a, even_b), __lasx_xvmulwod_w_h(even_a, even_b));
__m256i prod1 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(odd_a, odd_b),__lasx_xvmulwod_w_h(odd_a, odd_b));
__m256i even = __lasx_xvmulwev_h_bu(a.val, b.val);
__m256i odd = __lasx_xvmulwod_h_bu(a.val, b.val);
__m256i prod0 = __lasx_xvhaddw_wu_hu(even, even);
__m256i prod1 = __lasx_xvhaddw_wu_hu(odd, odd);
return v_uint32x8(__lasx_xvadd_w(prod0, prod1));
}
inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b, const v_uint32x8& c)
@@ -2043,14 +1931,10 @@ inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b, con
inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b)
{
__m256i even_a = __lasx_xvsrai_h(__lasx_xvbsll_v(a.val, 1), 8);
__m256i odd_a = __lasx_xvsrai_h(a.val, 8);
__m256i even_b = __lasx_xvsrai_h(__lasx_xvbsll_v(b.val, 1), 8);
__m256i odd_b = __lasx_xvsrai_h(b.val, 8);
__m256i prod0 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(even_a, even_b), __lasx_xvmulwod_w_h(even_a, even_b));
__m256i prod1 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(odd_a, odd_b),__lasx_xvmulwod_w_h(odd_a, odd_b));
__m256i even = __lasx_xvmulwev_h_b(a.val, b.val);
__m256i odd = __lasx_xvmulwod_h_b(a.val, b.val);
__m256i prod0 = __lasx_xvhaddw_w_h(even, even);
__m256i prod1 = __lasx_xvhaddw_w_h(odd, odd);
return v_int32x8(__lasx_xvadd_w(prod0, prod1));
}
inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b, const v_int32x8& c)
@@ -2059,36 +1943,24 @@ inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b, const
// 16 >> 64
inline v_uint64x4 v_dotprod_expand(const v_uint16x16& a, const v_uint16x16& b)
{
__m256i mullo = __lasx_xvmul_h(a.val, b.val);
__m256i mulhi = __lasx_xvmuh_hu(a.val, b.val);
__m256i mul0 = __lasx_xvilvl_h(mulhi, mullo);
__m256i mul1 = __lasx_xvilvh_h(mulhi, mullo);
__m256i p02 = __lasx_xvbitsel_v(mul0, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0));
__m256i p13 = __lasx_xvsrli_d(mul0, 32);
__m256i p46 = __lasx_xvbitsel_v(mul1, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0));
__m256i p57 = __lasx_xvsrli_d(mul1, 32);
__m256i p15_ = __lasx_xvadd_d(p02, p13);
__m256i p9d_ = __lasx_xvadd_d(p46, p57);
return v_uint64x4(__lasx_xvadd_d(
__lasx_xvilvl_d(p9d_, p15_),
__lasx_xvilvh_d(p9d_, p15_)));
__m256i even = __lasx_xvmulwev_w_hu(a.val, b.val);
__m256i odd = __lasx_xvmulwod_w_hu(a.val, b.val);
__m256i prod0 = __lasx_xvhaddw_du_wu(even, even);
__m256i prod1 = __lasx_xvhaddw_du_wu(odd, odd);
return v_uint64x4(__lasx_xvadd_d(prod0, prod1));
}
inline v_uint64x4 v_dotprod_expand(const v_uint16x16& a, const v_uint16x16& b, const v_uint64x4& c)
{ return v_dotprod_expand(a, b) + c; }
inline v_int64x4 v_dotprod_expand(const v_int16x16& a, const v_int16x16& b)
{
__m256i prod = __lasx_xvadd_w(__lasx_xvmulwev_w_h(a.val, b.val), __lasx_xvmulwod_w_h(a.val, b.val));
__m256i sign = __lasx_xvsrai_w(prod, 31);
__m256i lo = __lasx_xvilvl_w(sign, prod);
__m256i hi = __lasx_xvilvh_w(sign, prod);
return v_int64x4(__lasx_xvadd_d(__lasx_xvilvl_d(hi, lo), __lasx_xvilvh_d(hi, lo)));
__m256i even = __lasx_xvmulwev_w_h(a.val, b.val);
__m256i odd = __lasx_xvmulwod_w_h(a.val, b.val);
__m256i prod0 = __lasx_xvhaddw_d_w(even, even);
__m256i prod1 = __lasx_xvhaddw_d_w(odd, odd);
return v_int64x4(__lasx_xvadd_d(prod0, prod1));
}
inline v_int64x4 v_dotprod_expand(const v_int16x16& a, const v_int16x16& b, const v_int64x4& c)
{ return v_dotprod_expand(a, b) + c; }
@@ -2126,20 +1998,11 @@ inline v_int32x8 v_dotprod_expand_fast(const v_int8x32& a, const v_int8x32& b, c
// 16 >> 64
inline v_uint64x4 v_dotprod_expand_fast(const v_uint16x16& a, const v_uint16x16& b)
{
__m256i mullo = __lasx_xvmul_h(a.val, b.val);
__m256i mulhi = __lasx_xvmuh_hu(a.val, b.val);
__m256i mul0 = __lasx_xvilvl_h(mulhi, mullo);
__m256i mul1 = __lasx_xvilvh_h(mulhi, mullo);
__m256i p02 = __lasx_xvbitsel_v(mul0, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0));
__m256i p13 = __lasx_xvsrli_d(mul0, 32);
__m256i p46 = __lasx_xvbitsel_v(mul1, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0));
__m256i p57 = __lasx_xvsrli_d(mul1, 32);
__m256i p15_ = __lasx_xvadd_d(p02, p13);
__m256i p9d_ = __lasx_xvadd_d(p46, p57);
return v_uint64x4(__lasx_xvadd_d(p15_, p9d_));
__m256i even = __lasx_xvmulwev_w_hu(a.val, b.val);
__m256i odd = __lasx_xvmulwod_w_hu(a.val, b.val);
__m256i prod0 = __lasx_xvhaddw_du_wu(even, even);
__m256i prod1 = __lasx_xvhaddw_du_wu(odd, odd);
return v_uint64x4(__lasx_xvadd_d(__lasx_xvilvl_d(prod1, prod0), __lasx_xvilvh_d(prod1, prod0)));
}
inline v_uint64x4 v_dotprod_expand_fast(const v_uint16x16& a, const v_uint16x16& b, const v_uint64x4& c)
{ return v_dotprod_expand_fast(a, b) + c; }
@@ -2261,12 +2124,7 @@ inline v_int8x32 v_pack(const v_int16x16& a, const v_int16x16& b)
{ return v_int8x32(_v256_shuffle_odd_64(_lasx_packs_h(a.val, b.val))); }
inline v_uint8x32 v_pack(const v_uint16x16& a, const v_uint16x16& b)
{
__m256i t = __lasx_xvreplgr2vr_h(255);
__m256i a1 = __lasx_xvmin_hu(a.val, t);
__m256i b1 = __lasx_xvmin_hu(b.val, t);
return v_uint8x32(_v256_shuffle_odd_64(_lasx_packus_h(a1, b1)));
}
{ return v_uint8x32(_v256_shuffle_odd_64(__lasx_xvssrlrni_bu_h(b.val, a.val, 0))); }
inline v_uint8x32 v_pack_u(const v_int16x16& a, const v_int16x16& b)
{
@@ -2276,13 +2134,8 @@ inline v_uint8x32 v_pack_u(const v_int16x16& a, const v_int16x16& b)
inline void v_pack_store(schar* ptr, const v_int16x16& a)
{ v_store_low(ptr, v_pack(a, a)); }
inline void v_pack_store(uchar* ptr, const v_uint16x16& a)
{
const __m256i m = __lasx_xvreplgr2vr_h(255);
__m256i am = __lasx_xvmin_hu(a.val, m);
am = _v256_shuffle_odd_64(_lasx_packus_h(am, am));
v_store_low(ptr, v_uint8x32(am));
}
inline void v_pack_store(uchar *ptr, const v_uint16x16& a)
{ v_store_low(ptr, v_pack(a, a)); }
inline void v_pack_u_store(uchar* ptr, const v_int16x16& a)
{ v_store_low(ptr, v_pack_u(a, a)); }
@@ -2290,45 +2143,43 @@ inline void v_pack_u_store(uchar* ptr, const v_int16x16& a)
template<int n> inline
v_uint8x32 v_rshr_pack(const v_uint16x16& a, const v_uint16x16& b)
{
// we assume that n > 0, and so the shifted 16-bit values can be treated as signed numbers.
v_uint16x16 delta = v256_setall_u16((short)(1 << (n-1)));
return v_pack_u(v_reinterpret_as_s16((a + delta) >> n),
v_reinterpret_as_s16((b + delta) >> n));
__m256i res = __lasx_xvssrlrni_bu_h(b.val, a.val, n);
return v_uint8x32(_v256_shuffle_odd_64(res));
}
template<int n> inline
void v_rshr_pack_store(uchar* ptr, const v_uint16x16& a)
{
v_uint16x16 delta = v256_setall_u16((short)(1 << (n-1)));
v_pack_u_store(ptr, v_reinterpret_as_s16((a + delta) >> n));
__m256i res = __lasx_xvssrlrni_bu_h(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
}
template<int n> inline
v_uint8x32 v_rshr_pack_u(const v_int16x16& a, const v_int16x16& b)
{
v_int16x16 delta = v256_setall_s16((short)(1 << (n-1)));
return v_pack_u((a + delta) >> n, (b + delta) >> n);
__m256i res = __lasx_xvssrarni_bu_h(b.val, a.val, n);
return v_uint8x32(_v256_shuffle_odd_64(res));
}
template<int n> inline
void v_rshr_pack_u_store(uchar* ptr, const v_int16x16& a)
{
v_int16x16 delta = v256_setall_s16((short)(1 << (n-1)));
v_pack_u_store(ptr, (a + delta) >> n);
__m256i res = __lasx_xvssrarni_bu_h(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
}
template<int n> inline
v_int8x32 v_rshr_pack(const v_int16x16& a, const v_int16x16& b)
{
v_int16x16 delta = v256_setall_s16((short)(1 << (n-1)));
return v_pack((a + delta) >> n, (b + delta) >> n);
__m256i res = __lasx_xvssrarni_b_h(b.val, a.val, n);
return v_int8x32(_v256_shuffle_odd_64(res));
}
template<int n> inline
void v_rshr_pack_store(schar* ptr, const v_int16x16& a)
{
v_int16x16 delta = v256_setall_s16((short)(1 << (n-1)));
v_pack_store(ptr, (a + delta) >> n);
__m256i res = __lasx_xvssrarni_b_h(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
}
// 32
@@ -2346,67 +2197,51 @@ inline void v_pack_store(short* ptr, const v_int32x8& a)
inline void v_pack_store(ushort* ptr, const v_uint32x8& a)
{
const __m256i m = __lasx_xvreplgr2vr_w(65535);
__m256i am = __lasx_xvmin_wu(a.val, m);
am = _v256_shuffle_odd_64(_lasx_packus_w(am, am));
v_store_low(ptr, v_uint16x16(am));
__m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, 0);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
}
inline void v_pack_u_store(ushort* ptr, const v_int32x8& a)
{ v_store_low(ptr, v_pack_u(a, a)); }
template<int n> inline
v_uint16x16 v_rshr_pack(const v_uint32x8& a, const v_uint32x8& b)
{
// we assume that n > 0, and so the shifted 32-bit values can be treated as signed numbers.
v_uint32x8 delta = v256_setall_u32(1 << (n-1));
return v_pack_u(v_reinterpret_as_s32((a + delta) >> n),
v_reinterpret_as_s32((b + delta) >> n));
}
{ return v_uint16x16(_v256_shuffle_odd_64(__lasx_xvssrlrni_hu_w(b.val, a.val, n))); }
template<int n> inline
void v_rshr_pack_store(ushort* ptr, const v_uint32x8& a)
{
v_uint32x8 delta = v256_setall_u32(1 << (n-1));
v_pack_u_store(ptr, v_reinterpret_as_s32((a + delta) >> n));
__m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
}
template<int n> inline
v_uint16x16 v_rshr_pack_u(const v_int32x8& a, const v_int32x8& b)
{
v_int32x8 delta = v256_setall_s32(1 << (n-1));
return v_pack_u((a + delta) >> n, (b + delta) >> n);
}
{ return v_uint16x16(_v256_shuffle_odd_64(__lasx_xvssrarni_hu_w(b.val, a.val, n))); }
template<int n> inline
void v_rshr_pack_u_store(ushort* ptr, const v_int32x8& a)
{
v_int32x8 delta = v256_setall_s32(1 << (n-1));
v_pack_u_store(ptr, (a + delta) >> n);
__m256i res = __lasx_xvssrarni_hu_w(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
}
template<int n> inline
v_int16x16 v_rshr_pack(const v_int32x8& a, const v_int32x8& b)
{
v_int32x8 delta = v256_setall_s32(1 << (n-1));
return v_pack((a + delta) >> n, (b + delta) >> n);
}
{ return v_int16x16(_v256_shuffle_odd_64(__lasx_xvssrarni_h_w(b.val, a.val, n))); }
template<int n> inline
void v_rshr_pack_store(short* ptr, const v_int32x8& a)
{
v_int32x8 delta = v256_setall_s32(1 << (n-1));
v_pack_store(ptr, (a + delta) >> n);
__m256i res = __lasx_xvssrarni_h_w(a.val, a.val, n);
__lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0);
}
// 64
// Non-saturating pack
inline v_uint32x8 v_pack(const v_uint64x4& a, const v_uint64x4& b)
{
__m256i a0 = __lasx_xvshuf4i_w(a.val, 0x08);
__m256i b0 = __lasx_xvshuf4i_w(b.val, 0x08);
__m256i ab = __lasx_xvilvl_d(b0, a0);
__m256i ab = __lasx_xvpickev_w(b.val, a.val);
return v_uint32x8(_v256_shuffle_odd_64(ab));
}
@@ -2424,31 +2259,19 @@ inline void v_pack_store(int* ptr, const v_int64x4& b)
template<int n> inline
v_uint32x8 v_rshr_pack(const v_uint64x4& a, const v_uint64x4& b)
{
v_uint64x4 delta = v256_setall_u64((uint64)1 << (n-1));
return v_pack((a + delta) >> n, (b + delta) >> n);
}
{ return v_uint32x8(_v256_shuffle_odd_64(__lasx_xvsrlrni_w_d(b.val, a.val, n))); }
template<int n> inline
void v_rshr_pack_store(unsigned* ptr, const v_uint64x4& a)
{
v_uint64x4 delta = v256_setall_u64((uint64)1 << (n-1));
v_pack_store(ptr, (a + delta) >> n);
}
{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrlrni_w_d(a.val, a.val, n)), ptr, 0); }
template<int n> inline
v_int32x8 v_rshr_pack(const v_int64x4& a, const v_int64x4& b)
{
v_int64x4 delta = v256_setall_s64((int64)1 << (n-1));
return v_pack((a + delta) >> n, (b + delta) >> n);
}
{ return v_int32x8(_v256_shuffle_odd_64(__lasx_xvsrarni_w_d(b.val, a.val, n))); }
template<int n> inline
void v_rshr_pack_store(int* ptr, const v_int64x4& a)
{
v_int64x4 delta = v256_setall_s64((int64)1 << (n-1));
v_pack_store(ptr, (a + delta) >> n);
}
{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrarni_w_d(a.val, a.val, n)), ptr, 0); }
// pack boolean
inline v_uint8x32 v_pack_b(const v_uint16x16& a, const v_uint16x16& b)
@@ -2583,63 +2406,48 @@ template<int i>
inline v_float32x8 v_broadcast_element(const v_float32x8 &a)
{ return v_reinterpret_as_f32(v_broadcast_element<i>(v_reinterpret_as_u32(a))); }
///////////////////// load deinterleave /////////////////////////////
inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b )
inline void v_load_deinterleave(const uchar* ptr, v_uint8x32& a, v_uint8x32& b)
{
__m256i ab0 = __lasx_xvld(ptr, 0);
__m256i ab1 = __lasx_xvld(ptr + 32, 0);
__m256i t0 = __lasx_xvld(ptr, 0);
__m256i t1 = __lasx_xvld(ptr, 32);
const __m256i sh = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15,
0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
__m256i p0 = __lasx_xvshuf_b(ab0, ab0, sh);
__m256i p1 = __lasx_xvshuf_b(ab1, ab1, sh);
__m256i pl = __lasx_xvpermi_q(p0, p1, 0x02);
__m256i ph = __lasx_xvpermi_q(p0, p1, 0x13);
__m256i a0 = __lasx_xvilvl_d(ph, pl);
__m256i b0 = __lasx_xvilvh_d(ph, pl);
a = v_uint8x32(a0);
b = v_uint8x32(b0);
__m256i p0 = __lasx_xvpickev_b(t1, t0);
__m256i p1 = __lasx_xvpickod_b(t1, t0);
a.val = __lasx_xvpermi_d(p0, 0xd8);
b.val = __lasx_xvpermi_d(p1, 0xd8);
}
inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b )
{
__m256i ab0 = __lasx_xvld(ptr, 0);
__m256i ab1 = __lasx_xvld(ptr + 16, 0);
__m256i t0 = __lasx_xvld(ptr, 0);
__m256i t1 = __lasx_xvld(ptr, 32);
const __m256i sh = _v256_setr_b(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15,
0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15);
__m256i p0 = __lasx_xvshuf_b(ab0, ab0, sh);
__m256i p1 = __lasx_xvshuf_b(ab1, ab1, sh);
__m256i pl = __lasx_xvpermi_q(p0, p1, 0x02);
__m256i ph = __lasx_xvpermi_q(p0, p1, 0x13);
__m256i a0 = __lasx_xvilvl_d(ph, pl);
__m256i b0 = __lasx_xvilvh_d(ph, pl);
a = v_uint16x16(a0);
b = v_uint16x16(b0);
__m256i p0 = __lasx_xvpickev_h(t1, t0);
__m256i p1 = __lasx_xvpickod_h(t1, t0);
a.val = __lasx_xvpermi_d(p0, 0xd8);
b.val = __lasx_xvpermi_d(p1, 0xd8);
}
inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b )
{
__m256i ab0 = __lasx_xvld(ptr, 0);
__m256i ab1 = __lasx_xvld(ptr + 8, 0);
__m256i t0 = __lasx_xvld(ptr, 0);
__m256i t1 = __lasx_xvld(ptr, 32);
//const int sh = 0+2*4+1*16+3*64;
__m256i p0 = __lasx_xvshuf4i_w(ab0, 0xD8);
__m256i p1 = __lasx_xvshuf4i_w(ab1, 0xD8);
__m256i pl = __lasx_xvpermi_q(p0, p1, 0x02);
__m256i ph = __lasx_xvpermi_q(p0, p1, 0x13);
__m256i a0 = __lasx_xvilvl_d(ph, pl);
__m256i b0 = __lasx_xvilvh_d(ph, pl);
a = v_uint32x8(a0);
b = v_uint32x8(b0);
__m256i p0 = __lasx_xvpickev_w(t1, t0);
__m256i p1 = __lasx_xvpickod_w(t1, t0);
a.val = __lasx_xvpermi_d(p0, 0xd8);
b.val = __lasx_xvpermi_d(p1, 0xd8);
}
inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b )
{
__m256i ab0 = __lasx_xvld(ptr, 0);
__m256i ab1 = __lasx_xvld(ptr + 4, 0);
__m256i ab1 = __lasx_xvld(ptr, 32);
__m256i pl = __lasx_xvpermi_q(ab0, ab1, 0x02);
__m256i ph = __lasx_xvpermi_q(ab0, ab1, 0x13);
@@ -2652,8 +2460,8 @@ inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b
inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c )
{
__m256i bgr0 = __lasx_xvld(ptr, 0);
__m256i bgr1 = __lasx_xvld(ptr + 32, 0);
__m256i bgr2 = __lasx_xvld(ptr + 64, 0);
__m256i bgr1 = __lasx_xvld(ptr, 32);
__m256i bgr2 = __lasx_xvld(ptr, 64);
__m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02);
__m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13);
@@ -2686,8 +2494,8 @@ inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b,
inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c )
{
__m256i bgr0 = __lasx_xvld(ptr, 0);
__m256i bgr1 = __lasx_xvld(ptr + 16, 0);
__m256i bgr2 = __lasx_xvld(ptr + 32, 0);
__m256i bgr1 = __lasx_xvld(ptr, 32);
__m256i bgr2 = __lasx_xvld(ptr, 64);
__m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02);
__m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13);
@@ -2717,8 +2525,8 @@ inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16&
inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b, v_uint32x8& c )
{
__m256i bgr0 = __lasx_xvld(ptr, 0);
__m256i bgr1 = __lasx_xvld(ptr + 8, 0);
__m256i bgr2 = __lasx_xvld(ptr + 16, 0);
__m256i bgr1 = __lasx_xvld(ptr, 32);
__m256i bgr2 = __lasx_xvld(ptr, 64);
__m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02);
__m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13);
@@ -2741,8 +2549,8 @@ inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8&
inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b, v_uint64x4& c )
{
__m256i bgr0 = __lasx_xvld(ptr, 0);
__m256i bgr1 = __lasx_xvld(ptr + 4, 0);
__m256i bgr2 = __lasx_xvld(ptr + 8, 0);
__m256i bgr1 = __lasx_xvld(ptr, 32);
__m256i bgr2 = __lasx_xvld(ptr, 64);
__m256i s01 = __lasx_xvpermi_q(bgr0, bgr1, 0x12); // get bgr0 low 128 and bgr1 high 128
__m256i s12 = __lasx_xvpermi_q(bgr1, bgr2, 0x12);
@@ -2756,81 +2564,60 @@ inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b
c = v_uint64x4(r0);
}
inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c, v_uint8x32& d )
inline void v_load_deinterleave(const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c, v_uint8x32& d)
{
__m256i bgr0 = __lasx_xvld(ptr, 0);
__m256i bgr1 = __lasx_xvld(ptr + 32, 0);
__m256i bgr2 = __lasx_xvld(ptr + 64, 0);
__m256i bgr3 = __lasx_xvld(ptr + 96, 0);
const __m256i sh = _v256_setr_b(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15,
0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
__m256i t0 = __lasx_xvld(ptr, 0);
__m256i t1 = __lasx_xvld(ptr, 32);
__m256i t2 = __lasx_xvld(ptr, 64);
__m256i t3 = __lasx_xvld(ptr, 96);
__m256i p0 = __lasx_xvshuf_b(bgr0, bgr0, sh);
__m256i p1 = __lasx_xvshuf_b(bgr1, bgr1, sh);
__m256i p2 = __lasx_xvshuf_b(bgr2, bgr2, sh);
__m256i p3 = __lasx_xvshuf_b(bgr3, bgr3, sh);
const __m256i sh = _v256_setr_w(0, 4, 1, 5, 2, 6, 3, 7);
__m256i ac_lo = __lasx_xvpickev_b(t1, t0);
__m256i bd_lo = __lasx_xvpickod_b(t1, t0);
__m256i ac_hi = __lasx_xvpickev_b(t3, t2);
__m256i bd_hi = __lasx_xvpickod_b(t3, t2);
__m256i p01l = __lasx_xvilvl_w(p1, p0);
__m256i p01h = __lasx_xvilvh_w(p1, p0);
__m256i p23l = __lasx_xvilvl_w(p3, p2);
__m256i p23h = __lasx_xvilvh_w(p3, p2);
__m256i a_pre = __lasx_xvpickev_b(ac_hi, ac_lo);
__m256i c_pre = __lasx_xvpickod_b(ac_hi, ac_lo);
__m256i b_pre = __lasx_xvpickev_b(bd_hi, bd_lo);
__m256i d_pre = __lasx_xvpickod_b(bd_hi, bd_lo);
__m256i pll = __lasx_xvpermi_q(p01l, p23l, 0x02);
__m256i plh = __lasx_xvpermi_q(p01l, p23l, 0x13);
__m256i phl = __lasx_xvpermi_q(p01h, p23h, 0x02);
__m256i phh = __lasx_xvpermi_q(p01h, p23h, 0x13);
__m256i b0 = __lasx_xvilvl_w(plh, pll);
__m256i g0 = __lasx_xvilvh_w(plh, pll);
__m256i r0 = __lasx_xvilvl_w(phh, phl);
__m256i a0 = __lasx_xvilvh_w(phh, phl);
a = v_uint8x32(b0);
b = v_uint8x32(g0);
c = v_uint8x32(r0);
d = v_uint8x32(a0);
a.val = __lasx_xvperm_w(a_pre, sh);
b.val = __lasx_xvperm_w(b_pre, sh);
c.val = __lasx_xvperm_w(c_pre, sh);
d.val = __lasx_xvperm_w(d_pre, sh);
}
inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c, v_uint16x16& d )
inline void v_load_deinterleave(const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c, v_uint16x16& d)
{
__m256i bgr0 = __lasx_xvld(ptr, 0);
__m256i bgr1 = __lasx_xvld(ptr + 16, 0);
__m256i bgr2 = __lasx_xvld(ptr + 32, 0);
__m256i bgr3 = __lasx_xvld(ptr + 48, 0);
const __m256i sh = _v256_setr_b(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15,
0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15);
__m256i p0 = __lasx_xvshuf_b(bgr0, bgr0, sh);
__m256i p1 = __lasx_xvshuf_b(bgr1, bgr1, sh);
__m256i p2 = __lasx_xvshuf_b(bgr2, bgr2, sh);
__m256i p3 = __lasx_xvshuf_b(bgr3, bgr3, sh);
__m256i t0 = __lasx_xvld(ptr, 0);
__m256i t1 = __lasx_xvld(ptr, 32);
__m256i t2 = __lasx_xvld(ptr, 64);
__m256i t3 = __lasx_xvld(ptr, 96);
__m256i p01l = __lasx_xvilvl_w(p1, p0);
__m256i p01h = __lasx_xvilvh_w(p1, p0);
__m256i p23l = __lasx_xvilvl_w(p3, p2);
__m256i p23h = __lasx_xvilvh_w(p3, p2);
const __m256i sh = _v256_setr_w(0, 4, 1, 5, 2, 6, 3, 7);
__m256i ac_lo = __lasx_xvpickev_h(t1, t0);
__m256i bd_lo = __lasx_xvpickod_h(t1, t0);
__m256i ac_hi = __lasx_xvpickev_h(t3, t2);
__m256i bd_hi = __lasx_xvpickod_h(t3, t2);
__m256i pll = __lasx_xvpermi_q(p01l, p23l, 0x02);
__m256i plh = __lasx_xvpermi_q(p01l, p23l, 0x13);
__m256i phl = __lasx_xvpermi_q(p01h, p23h, 0x02);
__m256i phh = __lasx_xvpermi_q(p01h, p23h, 0x13);
__m256i a_pre = __lasx_xvpickev_h(ac_hi, ac_lo);
__m256i c_pre = __lasx_xvpickod_h(ac_hi, ac_lo);
__m256i b_pre = __lasx_xvpickev_h(bd_hi, bd_lo);
__m256i d_pre = __lasx_xvpickod_h(bd_hi, bd_lo);
__m256i b0 = __lasx_xvilvl_w(plh, pll);
__m256i g0 = __lasx_xvilvh_w(plh, pll);
__m256i r0 = __lasx_xvilvl_w(phh, phl);
__m256i a0 = __lasx_xvilvh_w(phh, phl);
a = v_uint16x16(b0);
b = v_uint16x16(g0);
c = v_uint16x16(r0);
d = v_uint16x16(a0);
a.val = __lasx_xvperm_w(a_pre, sh);
b.val = __lasx_xvperm_w(b_pre, sh);
c.val = __lasx_xvperm_w(c_pre, sh);
d.val = __lasx_xvperm_w(d_pre, sh);
}
inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b, v_uint32x8& c, v_uint32x8& d )
{
__m256i p0 = __lasx_xvld(ptr, 0);
__m256i p1 = __lasx_xvld(ptr + 8, 0);
__m256i p2 = __lasx_xvld(ptr + 16, 0);
__m256i p3 = __lasx_xvld(ptr + 24, 0);
__m256i p1 = __lasx_xvld(ptr, 32);
__m256i p2 = __lasx_xvld(ptr, 64);
__m256i p3 = __lasx_xvld(ptr, 96);
__m256i p01l = __lasx_xvilvl_w(p1, p0);
__m256i p01h = __lasx_xvilvh_w(p1, p0);
@@ -2856,9 +2643,9 @@ inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8&
inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b, v_uint64x4& c, v_uint64x4& d )
{
__m256i bgra0 = __lasx_xvld(ptr, 0);
__m256i bgra1 = __lasx_xvld(ptr + 4, 0);
__m256i bgra2 = __lasx_xvld(ptr + 8, 0);
__m256i bgra3 = __lasx_xvld(ptr + 12, 0);
__m256i bgra1 = __lasx_xvld(ptr, 32);
__m256i bgra2 = __lasx_xvld(ptr, 64);
__m256i bgra3 = __lasx_xvld(ptr, 96);
__m256i l02 = __lasx_xvpermi_q(bgra0, bgra2, 0x02);
__m256i h02 = __lasx_xvpermi_q(bgra0, bgra2, 0x13);
File diff suppressed because it is too large Load Diff
@@ -131,13 +131,22 @@ OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_I64(int64x2, int64x1, s64)
OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_F64(float64x2, float64x1,f64)
#endif
//////////// Compatibility layer ////////////
template<typename T> struct VTraits {
static inline int vlanes() { return T::nlanes; }
enum { max_nlanes = T::nlanes, nlanes = T::nlanes };
using lane_type = typename T::lane_type;
};
template<typename T>
inline typename VTraits<T>::lane_type v_get0(const T& v) \
{ \
return v.get0(); \
}
//////////// Types ////////////
struct v_uint8x16
{
typedef uchar lane_type;
enum { nlanes = 16 };
v_uint8x16() {}
explicit v_uint8x16(uint8x16_t v) : val(v) {}
v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7,
@@ -146,19 +155,22 @@ struct v_uint8x16
uchar v[] = {v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15};
val = vld1q_u8(v);
}
uint8x16_t val;
private:
friend struct VTraits<v_uint8x16>;
enum { nlanes = 16 };
typedef uchar lane_type;
friend typename VTraits<v_uint8x16>::lane_type v_get0<v_uint8x16>(const v_uint8x16& v);
uchar get0() const
{
return vgetq_lane_u8(val, 0);
}
uint8x16_t val;
};
struct v_int8x16
{
typedef schar lane_type;
enum { nlanes = 16 };
v_int8x16() {}
explicit v_int8x16(int8x16_t v) : val(v) {}
v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7,
@@ -167,19 +179,22 @@ struct v_int8x16
schar v[] = {v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15};
val = vld1q_s8(v);
}
int8x16_t val;
private:
friend struct VTraits<v_int8x16>;
enum { nlanes = 16 };
typedef schar lane_type;
friend typename VTraits<v_int8x16>::lane_type v_get0<v_int8x16>(const v_int8x16& v);
schar get0() const
{
return vgetq_lane_s8(val, 0);
}
int8x16_t val;
};
struct v_uint16x8
{
typedef ushort lane_type;
enum { nlanes = 8 };
v_uint16x8() {}
explicit v_uint16x8(uint16x8_t v) : val(v) {}
v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7)
@@ -187,19 +202,22 @@ struct v_uint16x8
ushort v[] = {v0, v1, v2, v3, v4, v5, v6, v7};
val = vld1q_u16(v);
}
uint16x8_t val;
private:
friend struct VTraits<v_uint16x8>;
enum { nlanes = 8 };
typedef ushort lane_type;
friend typename VTraits<v_uint16x8>::lane_type v_get0<v_uint16x8>(const v_uint16x8& v);
ushort get0() const
{
return vgetq_lane_u16(val, 0);
}
uint16x8_t val;
};
struct v_int16x8
{
typedef short lane_type;
enum { nlanes = 8 };
v_int16x8() {}
explicit v_int16x8(int16x8_t v) : val(v) {}
v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7)
@@ -207,19 +225,22 @@ struct v_int16x8
short v[] = {v0, v1, v2, v3, v4, v5, v6, v7};
val = vld1q_s16(v);
}
int16x8_t val;
private:
friend struct VTraits<v_int16x8>;
enum { nlanes = 8 };
typedef short lane_type;
friend typename VTraits<v_int16x8>::lane_type v_get0<v_int16x8>(const v_int16x8& v);
short get0() const
{
return vgetq_lane_s16(val, 0);
}
int16x8_t val;
};
struct v_uint32x4
{
typedef unsigned lane_type;
enum { nlanes = 4 };
v_uint32x4() {}
explicit v_uint32x4(uint32x4_t v) : val(v) {}
v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3)
@@ -227,19 +248,22 @@ struct v_uint32x4
unsigned v[] = {v0, v1, v2, v3};
val = vld1q_u32(v);
}
uint32x4_t val;
private:
friend struct VTraits<v_uint32x4>;
enum { nlanes = 4 };
typedef unsigned lane_type;
friend typename VTraits<v_uint32x4>::lane_type v_get0<v_uint32x4>(const v_uint32x4& v);
unsigned get0() const
{
return vgetq_lane_u32(val, 0);
}
uint32x4_t val;
};
struct v_int32x4
{
typedef int lane_type;
enum { nlanes = 4 };
v_int32x4() {}
explicit v_int32x4(int32x4_t v) : val(v) {}
v_int32x4(int v0, int v1, int v2, int v3)
@@ -247,18 +271,22 @@ struct v_int32x4
int v[] = {v0, v1, v2, v3};
val = vld1q_s32(v);
}
int32x4_t val;
private:
friend struct VTraits<v_int32x4>;
enum { nlanes = 4 };
typedef int lane_type;
friend typename VTraits<v_int32x4>::lane_type v_get0<v_int32x4>(const v_int32x4& v);
int get0() const
{
return vgetq_lane_s32(val, 0);
}
int32x4_t val;
};
struct v_float32x4
{
typedef float lane_type;
enum { nlanes = 4 };
v_float32x4() {}
explicit v_float32x4(float32x4_t v) : val(v) {}
v_float32x4(float v0, float v1, float v2, float v3)
@@ -266,18 +294,22 @@ struct v_float32x4
float v[] = {v0, v1, v2, v3};
val = vld1q_f32(v);
}
float32x4_t val;
private:
friend struct VTraits<v_float32x4>;
enum { nlanes = 4 };
typedef float lane_type;
friend typename VTraits<v_float32x4>::lane_type v_get0<v_float32x4>(const v_float32x4& v);
float get0() const
{
return vgetq_lane_f32(val, 0);
}
float32x4_t val;
};
struct v_uint64x2
{
typedef uint64 lane_type;
enum { nlanes = 2 };
v_uint64x2() {}
explicit v_uint64x2(uint64x2_t v) : val(v) {}
v_uint64x2(uint64 v0, uint64 v1)
@@ -285,18 +317,21 @@ struct v_uint64x2
uint64 v[] = {v0, v1};
val = vld1q_u64(v);
}
uint64x2_t val;
private:
friend struct VTraits<v_uint64x2>;
enum { nlanes = 2 };
typedef uint64 lane_type;
friend typename VTraits<v_uint64x2>::lane_type v_get0<v_uint64x2>(const v_uint64x2& v);
uint64 get0() const
{
return vgetq_lane_u64(val, 0);
}
uint64x2_t val;
};
struct v_int64x2
{
typedef int64 lane_type;
enum { nlanes = 2 };
v_int64x2() {}
explicit v_int64x2(int64x2_t v) : val(v) {}
v_int64x2(int64 v0, int64 v1)
@@ -304,19 +339,23 @@ struct v_int64x2
int64 v[] = {v0, v1};
val = vld1q_s64(v);
}
int64x2_t val;
private:
friend struct VTraits<v_int64x2>;
enum { nlanes = 2 };
typedef int64 lane_type;
friend typename VTraits<v_int64x2>::lane_type v_get0<v_int64x2>(const v_int64x2& v);
int64 get0() const
{
return vgetq_lane_s64(val, 0);
}
int64x2_t val;
};
#if CV_SIMD128_64F
struct v_float64x2
{
typedef double lane_type;
enum { nlanes = 2 };
v_float64x2() {}
explicit v_float64x2(float64x2_t v) : val(v) {}
v_float64x2(double v0, double v1)
@@ -324,11 +363,18 @@ struct v_float64x2
double v[] = {v0, v1};
val = vld1q_f64(v);
}
float64x2_t val;
private:
friend struct VTraits<v_float64x2>;
enum { nlanes = 2 };
typedef double lane_type;
friend typename VTraits<v_float64x2>::lane_type v_get0<v_float64x2>(const v_float64x2& v);
double get0() const
{
return vgetq_lane_f64(val, 0);
}
float64x2_t val;
};
#endif
@@ -460,71 +506,56 @@ inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0,
}
#define OPENCV_HAL_IMPL_NEON_BIN_OP(bin_op, _Tpvec, intrin) \
inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \
inline _Tpvec bin_op (const _Tpvec& a, const _Tpvec& b) \
{ \
return _Tpvec(intrin(a.val, b.val)); \
} \
inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \
{ \
a.val = intrin(a.val, b.val); \
return a; \
}
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint8x16, vqaddq_u8)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint8x16, vqsubq_u8)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int8x16, vqaddq_s8)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int8x16, vqsubq_s8)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint16x8, vqaddq_u16)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint16x8, vqsubq_u16)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int16x8, vqaddq_s16)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int16x8, vqsubq_s16)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int32x4, vaddq_s32)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int32x4, vsubq_s32)
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_int32x4, vmulq_s32)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint32x4, vaddq_u32)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint32x4, vsubq_u32)
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_uint32x4, vmulq_u32)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float32x4, vaddq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float32x4, vsubq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float32x4, vmulq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int64x2, vaddq_s64)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int64x2, vsubq_s64)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint64x2, vaddq_u64)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint64x2, vsubq_u64)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint8x16, vqaddq_u8)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint8x16, vqsubq_u8)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int8x16, vqaddq_s8)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int8x16, vqsubq_s8)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint16x8, vqaddq_u16)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint16x8, vqsubq_u16)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int16x8, vqaddq_s16)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int16x8, vqsubq_s16)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int32x4, vaddq_s32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int32x4, vsubq_s32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_int32x4, vmulq_s32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint32x4, vaddq_u32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint32x4, vsubq_u32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_uint32x4, vmulq_u32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_float32x4, vaddq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_float32x4, vsubq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_float32x4, vmulq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int64x2, vaddq_s64)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int64x2, vsubq_s64)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint64x2, vaddq_u64)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint64x2, vsubq_u64)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float32x4, vdivq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float64x2, vaddq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float64x2, vsubq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float64x2, vmulq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float64x2, vdivq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_div, v_float32x4, vdivq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_float64x2, vaddq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_float64x2, vsubq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_float64x2, vmulq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(v_div, v_float64x2, vdivq_f64)
#else
inline v_float32x4 operator / (const v_float32x4& a, const v_float32x4& b)
inline v_float32x4 v_div (const v_float32x4& a, const v_float32x4& b)
{
float32x4_t reciprocal = vrecpeq_f32(b.val);
reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal);
reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal);
return v_float32x4(vmulq_f32(a.val, reciprocal));
}
inline v_float32x4& operator /= (v_float32x4& a, const v_float32x4& b)
{
float32x4_t reciprocal = vrecpeq_f32(b.val);
reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal);
reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal);
a.val = vmulq_f32(a.val, reciprocal);
return a;
}
#endif
// saturating multiply 8-bit, 16-bit
#define OPENCV_HAL_IMPL_NEON_MUL_SAT(_Tpvec, _Tpwvec) \
inline _Tpvec operator * (const _Tpvec& a, const _Tpvec& b) \
inline _Tpvec v_mul (const _Tpvec& a, const _Tpvec& b) \
{ \
_Tpwvec c, d; \
v_mul_expand(a, b, c, d); \
return v_pack(c, d); \
} \
inline _Tpvec& operator *= (_Tpvec& a, const _Tpvec& b) \
{ a = a * b; return a; }
}
OPENCV_HAL_IMPL_NEON_MUL_SAT(v_int8x16, v_int16x8)
OPENCV_HAL_IMPL_NEON_MUL_SAT(v_uint8x16, v_uint16x8)
@@ -698,7 +729,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b)
inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b,
const v_uint32x4& c)
{
return v_dotprod_expand(a, b) + c;
return v_add(v_dotprod_expand(a, b), c);
}
inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b)
@@ -715,7 +746,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b)
inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b,
const v_int32x4& c)
{
return v_dotprod_expand(a, b) + c;
return v_add(v_dotprod_expand(a, b), c);
}
#endif
// 16 >> 64
@@ -735,7 +766,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b)
return v_uint64x2(vaddq_u64(s0, s1));
}
inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c)
{ return v_dotprod_expand(a, b) + c; }
{ return v_add(v_dotprod_expand(a, b), c); }
inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b)
{
@@ -752,7 +783,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b)
}
inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b,
const v_int64x2& c)
{ return v_dotprod_expand(a, b) + c; }
{ return v_add(v_dotprod_expand(a, b), c); }
// 32 >> 64f
#if CV_SIMD128_64F
@@ -760,7 +791,7 @@ inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b)
{ return v_cvt_f64(v_dotprod(a, b)); }
inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b,
const v_float64x2& c)
{ return v_dotprod_expand(a, b) + c; }
{ return v_add(v_dotprod_expand(a, b), c); }
#endif
//////// Fast Dot Product ////////
@@ -850,7 +881,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b
}
inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c)
{
return v_dotprod_expand_fast(a, b) + c;
return v_add(v_dotprod_expand_fast(a, b), c);
}
inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b)
@@ -861,7 +892,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b)
}
inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c)
{
return v_dotprod_expand_fast(a, b) + c;
return v_add(v_dotprod_expand_fast(a, b), c);
}
#endif
@@ -875,7 +906,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b
return v_uint64x2(vaddq_u64(s0, s1));
}
inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c)
{ return v_dotprod_expand_fast(a, b) + c; }
{ return v_add(v_dotprod_expand_fast(a, b), c); }
inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b)
{
@@ -884,22 +915,22 @@ inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b)
return v_int64x2(vaddl_s32(vget_low_s32(prod), vget_high_s32(prod)));
}
inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c)
{ return v_dotprod_expand_fast(a, b) + c; }
{ return v_add(v_dotprod_expand_fast(a, b), c); }
// 32 >> 64f
#if CV_SIMD128_64F
inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b)
{ return v_cvt_f64(v_dotprod_fast(a, b)); }
inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b, const v_float64x2& c)
{ return v_dotprod_expand_fast(a, b) + c; }
{ return v_add(v_dotprod_expand_fast(a, b), c); }
#endif
#define OPENCV_HAL_IMPL_NEON_LOGIC_OP(_Tpvec, suffix) \
OPENCV_HAL_IMPL_NEON_BIN_OP(&, _Tpvec, vandq_##suffix) \
OPENCV_HAL_IMPL_NEON_BIN_OP(|, _Tpvec, vorrq_##suffix) \
OPENCV_HAL_IMPL_NEON_BIN_OP(^, _Tpvec, veorq_##suffix) \
inline _Tpvec operator ~ (const _Tpvec& a) \
OPENCV_HAL_IMPL_NEON_BIN_OP(v_and, _Tpvec, vandq_##suffix) \
OPENCV_HAL_IMPL_NEON_BIN_OP(v_or, _Tpvec, vorrq_##suffix) \
OPENCV_HAL_IMPL_NEON_BIN_OP(v_xor, _Tpvec, veorq_##suffix) \
inline _Tpvec v_not (const _Tpvec& a) \
{ \
return _Tpvec(vreinterpretq_##suffix##_u8(vmvnq_u8(vreinterpretq_u8_##suffix(a.val)))); \
}
@@ -914,21 +945,16 @@ OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_uint64x2, u64)
OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_int64x2, s64)
#define OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(bin_op, intrin) \
inline v_float32x4 operator bin_op (const v_float32x4& a, const v_float32x4& b) \
inline v_float32x4 bin_op (const v_float32x4& a, const v_float32x4& b) \
{ \
return v_float32x4(vreinterpretq_f32_s32(intrin(vreinterpretq_s32_f32(a.val), vreinterpretq_s32_f32(b.val)))); \
} \
inline v_float32x4& operator bin_op##= (v_float32x4& a, const v_float32x4& b) \
{ \
a.val = vreinterpretq_f32_s32(intrin(vreinterpretq_s32_f32(a.val), vreinterpretq_s32_f32(b.val))); \
return a; \
}
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(&, vandq_s32)
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(|, vorrq_s32)
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(^, veorq_s32)
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_and, vandq_s32)
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_or, vorrq_s32)
OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_xor, veorq_s32)
inline v_float32x4 operator ~ (const v_float32x4& a)
inline v_float32x4 v_not (const v_float32x4& a)
{
return v_float32x4(vreinterpretq_f32_s32(vmvnq_s32(vreinterpretq_s32_f32(a.val))));
}
@@ -942,7 +968,7 @@ inline v_float32x4 v_sqrt(const v_float32x4& x)
inline v_float32x4 v_invsqrt(const v_float32x4& x)
{
v_float32x4 one = v_setall_f32(1.0f);
return one / v_sqrt(x);
return v_div(one, v_sqrt(x));
}
#else
inline v_float32x4 v_sqrt(const v_float32x4& x)
@@ -975,21 +1001,16 @@ inline v_float32x4 v_abs(v_float32x4 x)
#if CV_SIMD128_64F
#define OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(bin_op, intrin) \
inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \
inline v_float64x2 bin_op (const v_float64x2& a, const v_float64x2& b) \
{ \
return v_float64x2(vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val)))); \
} \
inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \
{ \
a.val = vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val))); \
return a; \
}
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(&, vandq_s64)
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(|, vorrq_s64)
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(^, veorq_s64)
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_and, vandq_s64)
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_or, vorrq_s64)
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_xor, veorq_s64)
inline v_float64x2 operator ~ (const v_float64x2& a)
inline v_float64x2 v_not (const v_float64x2& a)
{
return v_float64x2(vreinterpretq_f64_s32(vmvnq_s32(vreinterpretq_s32_f64(a.val))));
}
@@ -1002,7 +1023,7 @@ inline v_float64x2 v_sqrt(const v_float64x2& x)
inline v_float64x2 v_invsqrt(const v_float64x2& x)
{
v_float64x2 one = v_setall_f64(1.0f);
return one / v_sqrt(x);
return v_div(one, v_sqrt(x));
}
inline v_float64x2 v_abs(v_float64x2 x)
@@ -1037,17 +1058,17 @@ OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_max, vmaxq_f64)
#endif
#define OPENCV_HAL_IMPL_NEON_INT_CMP_OP(_Tpvec, cast, suffix, not_suffix) \
inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \
inline _Tpvec v_eq (const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(cast(vceqq_##suffix(a.val, b.val))); } \
inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \
inline _Tpvec v_ne (const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(cast(vmvnq_##not_suffix(vceqq_##suffix(a.val, b.val)))); } \
inline _Tpvec operator < (const _Tpvec& a, const _Tpvec& b) \
inline _Tpvec v_lt (const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(cast(vcltq_##suffix(a.val, b.val))); } \
inline _Tpvec operator > (const _Tpvec& a, const _Tpvec& b) \
inline _Tpvec v_gt (const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(cast(vcgtq_##suffix(a.val, b.val))); } \
inline _Tpvec operator <= (const _Tpvec& a, const _Tpvec& b) \
inline _Tpvec v_le (const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(cast(vcleq_##suffix(a.val, b.val))); } \
inline _Tpvec operator >= (const _Tpvec& a, const _Tpvec& b) \
inline _Tpvec v_ge (const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(cast(vcgeq_##suffix(a.val, b.val))); }
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint8x16, OPENCV_HAL_NOP, u8, u8)
@@ -1067,48 +1088,48 @@ static inline uint64x2_t vmvnq_u64(uint64x2_t a)
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint64x2, OPENCV_HAL_NOP, u64, u64)
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int64x2, vreinterpretq_s64_u64, s64, u64)
#else
static inline v_uint64x2 operator == (const v_uint64x2& a, const v_uint64x2& b)
static inline v_uint64x2 v_eq(const v_uint64x2& a, const v_uint64x2& b)
{
uint32x4_t cmp = vceqq_u32(vreinterpretq_u32_u64(a.val),
vreinterpretq_u32_u64(b.val));
uint32x4_t v_eq = vandq_u32(cmp, vrev64q_u32(cmp));
return v_uint64x2(vreinterpretq_u64_u32(v_eq));
}
static inline v_uint64x2 operator != (const v_uint64x2& a, const v_uint64x2& b)
static inline v_uint64x2 v_ne(const v_uint64x2& a, const v_uint64x2& b)
{
uint64x2_t v_mask = vorrq_u64(vsubq_u64(a.val, b.val), vsubq_u64(b.val, a.val));
int64x2_t v_smask = vshrq_n_s64(vreinterpretq_s64_u64(v_mask), 63);
return v_uint64x2(vreinterpretq_u64_s64(v_smask));
}
static inline v_int64x2 operator == (const v_int64x2& a, const v_int64x2& b)
static inline v_int64x2 v_eq(const v_int64x2& a, const v_int64x2& b)
{
uint32x4_t cmp = vceqq_u32(vreinterpretq_u32_s64(a.val),
vreinterpretq_u32_s64(b.val));
uint32x4_t v_eq = vandq_u32(cmp, vrev64q_u32(cmp));
return v_int64x2(vreinterpretq_s64_u32(v_eq));
}
static inline v_int64x2 operator != (const v_int64x2& a, const v_int64x2& b)
static inline v_int64x2 v_ne(const v_int64x2& a, const v_int64x2& b)
{
int64x2_t v_mask = vorrq_s64(vsubq_s64(a.val, b.val), vsubq_s64(b.val, a.val));
int64x2_t v_smask = vshrq_n_s64(v_mask, 63);
return v_int64x2(v_smask);
}
static inline v_uint64x2 operator > (const v_uint64x2& a, const v_uint64x2& b)
static inline v_uint64x2 v_gt(const v_uint64x2& a, const v_uint64x2& b)
{
int64x2_t v_mask = vreinterpretq_s64_u64(vsubq_u64(b.val, a.val));
return v_uint64x2(vreinterpretq_u64_s64(vshrq_n_s64(v_mask, 63)));
}
static inline v_uint64x2 operator < (const v_uint64x2& a, const v_uint64x2& b)
static inline v_uint64x2 v_lt(const v_uint64x2& a, const v_uint64x2& b)
{
int64x2_t v_mask = vreinterpretq_s64_u64(vsubq_u64(a.val, b.val));
return v_uint64x2(vreinterpretq_u64_s64(vshrq_n_s64(v_mask, 63)));
}
static inline v_int64x2 operator > (const v_int64x2& a, const v_int64x2& b)
static inline v_int64x2 v_gt(const v_int64x2& a, const v_int64x2& b)
{
int64x2_t v_mask = vsubq_s64(b.val, a.val);
return v_int64x2(vshrq_n_s64(v_mask, 63));
}
static inline v_int64x2 operator < (const v_int64x2& a, const v_int64x2& b)
static inline v_int64x2 v_lt (const v_int64x2& a, const v_int64x2& b)
{
int64x2_t v_mask = vsubq_s64(a.val, b.val);
return v_int64x2(vshrq_n_s64(v_mask, 63));
@@ -1224,9 +1245,9 @@ inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_
// trade efficiency for convenience
#define OPENCV_HAL_IMPL_NEON_SHIFT_OP(_Tpvec, suffix, _Tps, ssuffix) \
inline _Tpvec operator << (const _Tpvec& a, int n) \
inline _Tpvec v_shl (const _Tpvec& a, int n) \
{ return _Tpvec(vshlq_##suffix(a.val, vdupq_n_##ssuffix((_Tps)n))); } \
inline _Tpvec operator >> (const _Tpvec& a, int n) \
inline _Tpvec v_shr (const _Tpvec& a, int n) \
{ return _Tpvec(vshlq_##suffix(a.val, vdupq_n_##ssuffix((_Tps)-n))); } \
template<int n> inline _Tpvec v_shl(const _Tpvec& a) \
{ return _Tpvec(vshlq_n_##suffix(a.val, n)); } \
@@ -1248,13 +1269,13 @@ OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_int64x2, s64, int64, s64)
template<int n> inline _Tpvec v_rotate_right(const _Tpvec& a) \
{ return _Tpvec(vextq_##suffix(a.val, vdupq_n_##suffix(0), n)); } \
template<int n> inline _Tpvec v_rotate_left(const _Tpvec& a) \
{ return _Tpvec(vextq_##suffix(vdupq_n_##suffix(0), a.val, _Tpvec::nlanes - n)); } \
{ return _Tpvec(vextq_##suffix(vdupq_n_##suffix(0), a.val, VTraits<_Tpvec>::nlanes - n)); } \
template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \
{ return a; } \
template<int n> inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(vextq_##suffix(a.val, b.val, n)); } \
template<int n> inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(vextq_##suffix(b.val, a.val, _Tpvec::nlanes - n)); } \
{ return _Tpvec(vextq_##suffix(b.val, a.val, VTraits<_Tpvec>::nlanes - n)); } \
template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \
{ CV_UNUSED(b); return a; }
+7 -8
View File
@@ -21,14 +21,13 @@ NS_ASSUME_NONNULL_BEGIN
*/
CV_EXPORTS @interface Range : NSObject
#ifdef __cplusplus
@property(readonly) cv::Range& nativeRef;
#endif
#pragma mark - Properties
@property int start;
@property int end;
#ifdef __cplusplus
@property(readonly) cv::Range& nativeRef;
#endif
#pragma mark - Constructors
@@ -36,6 +35,10 @@ CV_EXPORTS @interface Range : NSObject
- (instancetype)initWithStart:(int)start end:(int)end;
- (instancetype)initWithVals:(NSArray<NSNumber*>*)vals;
#ifdef __cplusplus
+ (instancetype)fromNative:(cv::Range&)range;
#endif
#pragma mark - Methods
/**
@@ -94,10 +97,6 @@ CV_EXPORTS @interface Range : NSObject
*/
- (NSString*)description;
#ifdef __cplusplus
+ (instancetype)fromNative:(cv::Range&)range;
#endif
@end
NS_ASSUME_NONNULL_END
-86
View File
@@ -1,86 +0,0 @@
//
// Range.m
//
// Created by Giles Payne on 2019/10/08.
//
#import "Range.h"
@implementation Range
- (instancetype)init {
return [self initWithStart:0 end: 0];
}
- (instancetype)initWithStart:(int)start end:(int)end {
self = [super init];
if (self != nil) {
self.start = start;
self.end = end;
}
return self;
}
- (instancetype)initWithVals:(NSArray<NSNumber*>*)vals {
self = [self init];
if (self != nil) {
[self set:vals];
}
return self;
}
- (void)set:(NSArray<NSNumber*>*)vals {
self.start = (vals != nil && vals.count > 0) ? vals[0].intValue : 0;
self.end = (vals != nil && vals.count > 1 ) ? vals[1].intValue : 0;
}
- (int)size {
return [self empty] ? 0 : self.end - self.start;
}
- (BOOL)empty {
return self.end <= self.start;
}
+ (Range*)all {
return [[Range alloc] initWithStart:INT_MIN end:INT_MAX];
}
- (Range*)intersection:(Range*)r1 {
Range* out = [[Range alloc] initWithStart:MAX(r1.start, self.start) end:MIN(r1.end, self.end)];
out.end = MAX(out.end, out.start);
return out;
}
- (Range*)shift:(int)delta {
return [[Range alloc] initWithStart:self.start + delta end:self.end + delta];
}
- (Range*)clone {
return [[Range alloc] initWithStart:self.start end:self.end];
}
- (BOOL)isEqual:(id)other {
if (other == self) {
return YES;
} else if (![other isKindOfClass:[Range class]]) {
return NO;
} else {
Range* it = (Range*)other;
return self.start == it.start && self.end == it.end;
}
}
- (NSUInteger)hash {
int prime = 31;
uint32_t result = 1;
result = prime * result + self.start;
result = prime * result + self.end;
return result;
}
- (NSString *)description {
return [NSString stringWithFormat:@"Range {%d, %d}", self.start, self.end];
}
@end
+21 -5
View File
@@ -1,5 +1,5 @@
//
// Range.m
// Range.mm
//
// Created by Giles Payne on 2019/10/08.
//
@@ -10,6 +10,22 @@
cv::Range native;
}
- (int)start {
return native.start;
}
- (void)setStart:(int)val {
native.start = val;
}
- (int)end {
return native.end;
}
- (void)setEnd:(int)val {
native.end = val;
}
- (cv::Range&)nativeRef {
return native;
}
@@ -35,6 +51,10 @@
return self;
}
+ (instancetype)fromNative:(cv::Range&)range {
return [[Range alloc] initWithStart:range.start end:range.end];
}
- (void)set:(NSArray<NSNumber*>*)vals {
self.start = (vals != nil && vals.count > 0) ? vals[0].intValue : 0;
self.end = (vals != nil && vals.count > 1 ) ? vals[1].intValue : 0;
@@ -85,10 +105,6 @@
return result;
}
+ (instancetype)fromNative:(cv::Range&)range {
return [[Range alloc] initWithStart:range.start end:range.end];
}
- (NSString *)description {
return [NSString stringWithFormat:@"Range {%d, %d}", self.start, self.end];
}
+1 -1
View File
@@ -169,7 +169,7 @@
"Range": {
"objc_type": "Range*",
"to_cpp": "%(n)s.nativeRef",
"from_cpp": "[Point3i fromNative:%(n)s]"
"from_cpp": "[Range fromNative:%(n)s]"
},
"Rect": {
"objc_type": "Rect2i*",
+13 -13
View File
@@ -684,8 +684,8 @@ static inline void v_store_pair_as(unsigned* ptr, const v_float64& a, const v_fl
static inline void vx_load_as(const double* ptr, v_float32& a)
{
const int VECSZ = v_float32::nlanes;
float buf[VECSZ*2];
const int VECSZ = VTraits<v_float32>::vlanes();
float buf[VTraits<v_float32>::max_nlanes*2];
for( int i = 0; i < VECSZ; i++ )
buf[i] = saturate_cast<float>(ptr[i]);
@@ -694,8 +694,8 @@ static inline void vx_load_as(const double* ptr, v_float32& a)
static inline void vx_load_as(const uint64_t* ptr, v_float32& a)
{
const int VECSZ = v_float32::nlanes;
float buf[VECSZ*2];
const int VECSZ = VTraits<v_float32>::vlanes();
float buf[VTraits<v_float32>::max_nlanes*2];
for( int i = 0; i < VECSZ; i++ )
buf[i] = saturate_cast<float>(ptr[i]);
@@ -704,8 +704,8 @@ static inline void vx_load_as(const uint64_t* ptr, v_float32& a)
static inline void vx_load_as(const int64_t* ptr, v_float32& a)
{
const int VECSZ = v_float32::nlanes;
float buf[VECSZ*2];
const int VECSZ = VTraits<v_float32>::vlanes();
float buf[VTraits<v_float32>::max_nlanes*2];
for( int i = 0; i < VECSZ; i++ )
buf[i] = saturate_cast<float>(ptr[i]);
@@ -715,19 +715,19 @@ static inline void vx_load_as(const int64_t* ptr, v_float32& a)
template<typename _Tdvec>
static inline void vx_load_pair_as(const double* ptr, _Tdvec& a, _Tdvec& b)
{
const int VECSZ = _Tdvec::nlanes;
typename _Tdvec::lane_type buf[VECSZ*2];
const int VECSZ = VTraits<_Tdvec>::vlanes();
typename VTraits<_Tdvec>::lane_type buf[VTraits<_Tdvec>::max_nlanes*2];
for( int i = 0; i < VECSZ*2; i++ )
buf[i] = saturate_cast<typename _Tdvec::lane_type>(ptr[i]);
buf[i] = saturate_cast<typename VTraits<_Tdvec>::lane_type>(ptr[i]);
a = vx_load(buf);
b = vx_load(buf + VECSZ);
}
static inline void v_store_as(double* ptr, const v_float32& a)
{
const int VECSZ = v_float32::nlanes;
float buf[VECSZ];
const int VECSZ = VTraits<v_float32>::vlanes();
float buf[VTraits<v_float32>::max_nlanes];
v_store(buf, a);
for( int i = 0; i < VECSZ; i++ )
@@ -737,8 +737,8 @@ static inline void v_store_as(double* ptr, const v_float32& a)
template<typename _Tsvec>
static inline void v_store_pair_as(double* ptr, const _Tsvec& a, const _Tsvec& b)
{
const int VECSZ = _Tsvec::nlanes;
typename _Tsvec::lane_type buf[VECSZ*2];
const int VECSZ = VTraits<_Tsvec>::vlanes();
typename VTraits<_Tsvec>::lane_type buf[VTraits<_Tsvec>::max_nlanes*2];
v_store(buf, a); v_store(buf + VECSZ, b);
for( int i = 0; i < VECSZ*2; i++ )
+61 -61
View File
@@ -93,13 +93,13 @@ struct v_atan_f32
{
v_float32 ax = v_abs(x);
v_float32 ay = v_abs(y);
v_float32 c = v_min(ax, ay) / (v_max(ax, ay) + eps);
v_float32 cc = c * c;
v_float32 a = v_fma(v_fma(v_fma(cc, p7, p5), cc, p3), cc, p1)*c;
a = v_select(ax >= ay, a, val90 - a);
a = v_select(x < z, val180 - a, a);
a = v_select(y < z, val360 - a, a);
return a * s;
v_float32 c = v_div(v_min(ax, ay), v_add(v_max(ax, ay), this->eps));
v_float32 cc = v_mul(c, c);
v_float32 a = v_mul(v_fma(v_fma(v_fma(cc, this->p7, this->p5), cc, this->p3), cc, this->p1), c);
a = v_select(v_ge(ax, ay), a, v_sub(this->val90, a));
a = v_select(v_lt(x, this->z), v_sub(this->val180, a), a);
a = v_select(v_lt(y, this->z), v_sub(this->val360, a), a);
return v_mul(a, this->s);
}
v_float32 eps;
@@ -125,7 +125,7 @@ static void fastAtan32f_(const float *Y, const float *X, float *angle, int len,
float scale = angleInDegrees ? 1.f : (float)(CV_PI/180);
int i = 0;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
const int VECSZ = VTraits<v_float32>::vlanes();
v_atan_f32 v(scale);
for( ; i < len; i += VECSZ*2 )
@@ -198,7 +198,7 @@ void magnitude32f(const float* x, const float* y, float* mag, int len)
int i = 0;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
const int VECSZ = VTraits<v_float32>::vlanes();
for( ; i < len; i += VECSZ*2 )
{
if( i + VECSZ*2 > len )
@@ -209,8 +209,8 @@ void magnitude32f(const float* x, const float* y, float* mag, int len)
}
v_float32 x0 = vx_load(x + i), x1 = vx_load(x + i + VECSZ);
v_float32 y0 = vx_load(y + i), y1 = vx_load(y + i + VECSZ);
x0 = v_sqrt(v_muladd(x0, x0, y0*y0));
x1 = v_sqrt(v_muladd(x1, x1, y1*y1));
x0 = v_sqrt(v_muladd(x0, x0, v_mul(y0, y0)));
x1 = v_sqrt(v_muladd(x1, x1, v_mul(y1, y1)));
v_store(mag + i, x0);
v_store(mag + i + VECSZ, x1);
}
@@ -231,7 +231,7 @@ void magnitude64f(const double* x, const double* y, double* mag, int len)
int i = 0;
#if CV_SIMD_64F
const int VECSZ = v_float64::nlanes;
const int VECSZ = VTraits<v_float64>::vlanes();
for( ; i < len; i += VECSZ*2 )
{
if( i + VECSZ*2 > len )
@@ -242,8 +242,8 @@ void magnitude64f(const double* x, const double* y, double* mag, int len)
}
v_float64 x0 = vx_load(x + i), x1 = vx_load(x + i + VECSZ);
v_float64 y0 = vx_load(y + i), y1 = vx_load(y + i + VECSZ);
x0 = v_sqrt(v_muladd(x0, x0, y0*y0));
x1 = v_sqrt(v_muladd(x1, x1, y1*y1));
x0 = v_sqrt(v_muladd(x0, x0, v_mul(y0, y0)));
x1 = v_sqrt(v_muladd(x1, x1, v_mul(y1, y1)));
v_store(mag + i, x0);
v_store(mag + i + VECSZ, x1);
}
@@ -265,7 +265,7 @@ void invSqrt32f(const float* src, float* dst, int len)
int i = 0;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
const int VECSZ = VTraits<v_float32>::vlanes();
for( ; i < len; i += VECSZ*2 )
{
if( i + VECSZ*2 > len )
@@ -293,7 +293,7 @@ void invSqrt64f(const double* src, double* dst, int len)
int i = 0;
#if CV_SIMD_64F
const int VECSZ = v_float64::nlanes;
const int VECSZ = VTraits<v_float64>::vlanes();
for ( ; i < len; i += VECSZ*2)
{
if( i + VECSZ*2 > len )
@@ -321,7 +321,7 @@ void sqrt32f(const float* src, float* dst, int len)
int i = 0;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
const int VECSZ = VTraits<v_float32>::vlanes();
for( ; i < len; i += VECSZ*2 )
{
if( i + VECSZ*2 > len )
@@ -350,7 +350,7 @@ void sqrt64f(const double* src, double* dst, int len)
int i = 0;
#if CV_SIMD_64F
const int VECSZ = v_float64::nlanes;
const int VECSZ = VTraits<v_float64>::vlanes();
for( ; i < len; i += VECSZ*2 )
{
if( i + VECSZ*2 > len )
@@ -452,7 +452,7 @@ void exp32f( const float *_x, float *y, int n )
float postscale = (float)exp_postscale;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
const int VECSZ = VTraits<v_float32>::vlanes();
const v_float32 vprescale = vx_setall_f32((float)exp_prescale);
const v_float32 vpostscale = vx_setall_f32((float)exp_postscale);
const v_float32 vminval = vx_setall_f32(minval);
@@ -481,26 +481,26 @@ void exp32f( const float *_x, float *y, int n )
xf0 = v_min(v_max(xf0, vminval), vmaxval);
xf1 = v_min(v_max(xf1, vminval), vmaxval);
xf0 *= vprescale;
xf1 *= vprescale;
xf0 = v_mul(xf0, vprescale);
xf1 = v_mul(xf1, vprescale);
v_int32 xi0 = v_round(xf0);
v_int32 xi1 = v_round(xf1);
xf0 = (xf0 - v_cvt_f32(xi0))*vpostscale;
xf1 = (xf1 - v_cvt_f32(xi1))*vpostscale;
xf0 = v_mul(v_sub(xf0, v_cvt_f32(xi0)), vpostscale);
xf1 = v_mul(v_sub(xf1, v_cvt_f32(xi1)), vpostscale);
v_float32 yf0 = v_lut(expTab_f, xi0 & vidxmask);
v_float32 yf1 = v_lut(expTab_f, xi1 & vidxmask);
v_float32 yf0 = v_lut(expTab_f, v_and(xi0, vidxmask));
v_float32 yf1 = v_lut(expTab_f, v_and(xi1, vidxmask));
v_int32 v0 = vx_setzero_s32(), v127 = vx_setall_s32(127), v255 = vx_setall_s32(255);
xi0 = v_min(v_max(v_shr<EXPTAB_SCALE>(xi0) + v127, v0), v255);
xi1 = v_min(v_max(v_shr<EXPTAB_SCALE>(xi1) + v127, v0), v255);
xi0 = v_min(v_max(v_add(v_shr<6>(xi0), v127), v0), v255);
xi1 = v_min(v_max(v_add(v_shr<6>(xi1), v127), v0), v255);
yf0 *= v_reinterpret_as_f32(v_shl<23>(xi0));
yf1 *= v_reinterpret_as_f32(v_shl<23>(xi1));
yf0 = v_mul(yf0, v_reinterpret_as_f32(v_shl<23>(xi0)));
yf1 = v_mul(yf1, v_reinterpret_as_f32(v_shl<23>(xi1)));
v_float32 zf0 = xf0 + vA1;
v_float32 zf1 = xf1 + vA1;
v_float32 zf0 = v_add(xf0, vA1);
v_float32 zf1 = v_add(xf1, vA1);
zf0 = v_fma(zf0, xf0, vA2);
zf1 = v_fma(zf1, xf1, vA2);
@@ -511,8 +511,8 @@ void exp32f( const float *_x, float *y, int n )
zf0 = v_fma(zf0, xf0, vA4);
zf1 = v_fma(zf1, xf1, vA4);
zf0 *= yf0;
zf1 *= yf1;
zf0 = v_mul(zf0, yf0);
zf1 = v_mul(zf1, yf1);
if( y_aligned )
{
@@ -566,7 +566,7 @@ void exp64f( const double *_x, double *y, int n )
double maxval = (exp_max_val/exp_prescale);
#if CV_SIMD_64F
const int VECSZ = v_float64::nlanes;
const int VECSZ = VTraits<v_float64>::vlanes();
const v_float64 vprescale = vx_setall_f64(exp_prescale);
const v_float64 vpostscale = vx_setall_f64(exp_postscale);
const v_float64 vminval = vx_setall_f64(minval);
@@ -596,30 +596,30 @@ void exp64f( const double *_x, double *y, int n )
xf0 = v_min(v_max(xf0, vminval), vmaxval);
xf1 = v_min(v_max(xf1, vminval), vmaxval);
xf0 *= vprescale;
xf1 *= vprescale;
xf0 = v_mul(xf0, vprescale);
xf1 = v_mul(xf1, vprescale);
v_int32 xi0 = v_round(xf0);
v_int32 xi1 = v_round(xf1);
xf0 = (xf0 - v_cvt_f64(xi0))*vpostscale;
xf1 = (xf1 - v_cvt_f64(xi1))*vpostscale;
xf0 = v_mul(v_sub(xf0, v_cvt_f64(xi0)), vpostscale);
xf1 = v_mul(v_sub(xf1, v_cvt_f64(xi1)), vpostscale);
v_float64 yf0 = v_lut(expTab, xi0 & vidxmask);
v_float64 yf1 = v_lut(expTab, xi1 & vidxmask);
v_float64 yf0 = v_lut(expTab, v_and(xi0, vidxmask));
v_float64 yf1 = v_lut(expTab, v_and(xi1, vidxmask));
v_int32 v0 = vx_setzero_s32(), v1023 = vx_setall_s32(1023), v2047 = vx_setall_s32(2047);
xi0 = v_min(v_max(v_shr<EXPTAB_SCALE>(xi0) + v1023, v0), v2047);
xi1 = v_min(v_max(v_shr<EXPTAB_SCALE>(xi1) + v1023, v0), v2047);
xi0 = v_min(v_max(v_add(v_shr<6>(xi0), v1023), v0), v2047);
xi1 = v_min(v_max(v_add(v_shr<6>(xi1), v1023), v0), v2047);
v_int64 xq0, xq1, dummy;
v_expand(xi0, xq0, dummy);
v_expand(xi1, xq1, dummy);
yf0 *= v_reinterpret_as_f64(v_shl<52>(xq0));
yf1 *= v_reinterpret_as_f64(v_shl<52>(xq1));
yf0 = v_mul(yf0, v_reinterpret_as_f64(v_shl<52>(xq0)));
yf1 = v_mul(yf1, v_reinterpret_as_f64(v_shl<52>(xq1)));
v_float64 zf0 = xf0 + vA1;
v_float64 zf1 = xf1 + vA1;
v_float64 zf0 = v_add(xf0, vA1);
v_float64 zf1 = v_add(xf1, vA1);
zf0 = v_fma(zf0, xf0, vA2);
zf1 = v_fma(zf1, xf1, vA2);
@@ -633,8 +633,8 @@ void exp64f( const double *_x, double *y, int n )
zf0 = v_fma(zf0, xf0, vA5);
zf1 = v_fma(zf1, xf1, vA5);
zf0 *= yf0;
zf1 *= yf1;
zf0 = v_mul(zf0, yf0);
zf1 = v_mul(zf1, yf1);
if( y_aligned )
{
@@ -696,7 +696,7 @@ void log32f( const float *_x, float *y, int n )
const int* x = (const int*)_x;
#if CV_SIMD
const int VECSZ = v_float32::nlanes;
const int VECSZ = VTraits<v_float32>::vlanes();
const v_float32 vln2 = vx_setall_f32((float)ln_2);
const v_float32 v1 = vx_setall_f32(1.f);
const v_float32 vshift = vx_setall_f32(-1.f/512);
@@ -715,18 +715,18 @@ void log32f( const float *_x, float *y, int n )
}
v_int32 h0 = vx_load(x + i);
v_int32 yi0 = (v_shr<23>(h0) & vx_setall_s32(255)) - vx_setall_s32(127);
v_int32 xi0 = (h0 & vx_setall_s32(LOGTAB_MASK2_32F)) | vx_setall_s32(127 << 23);
v_int32 yi0 = v_sub(v_and(v_shr<23>(h0), vx_setall_s32(255)), vx_setall_s32(127));
v_int32 xi0 = v_or(v_and(h0, vx_setall_s32(LOGTAB_MASK2_32F)), vx_setall_s32(127 << 23));
h0 = v_shr<23 - LOGTAB_SCALE - 1>(h0) & vx_setall_s32(LOGTAB_MASK*2);
h0 = v_and(v_shr<23 - 8 - 1>(h0), vx_setall_s32(((1 << 8) - 1) * 2));
v_float32 yf0, xf0;
v_lut_deinterleave(logTab_f, h0, yf0, xf0);
yf0 = v_fma(v_cvt_f32(yi0), vln2, yf0);
v_float32 delta = v_select(v_reinterpret_as_f32(h0 == vx_setall_s32(510)), vshift, vx_setall<float>(0));
xf0 = v_fma((v_reinterpret_as_f32(xi0) - v1), xf0, delta);
v_float32 delta = v_select(v_reinterpret_as_f32(v_eq(h0, vx_setall_s32(510))), vshift, vx_setall<float>(0));
xf0 = v_fma((v_sub(v_reinterpret_as_f32(xi0), v1)), xf0, delta);
v_float32 zf0 = v_fma(xf0, vA0, vA1);
zf0 = v_fma(zf0, xf0, vA2);
@@ -771,7 +771,7 @@ void log64f( const double *x, double *y, int n )
int i = 0;
#if CV_SIMD_64F
const int VECSZ = v_float64::nlanes;
const int VECSZ = VTraits<v_float64>::vlanes();
const v_float64 vln2 = vx_setall_f64(ln_2);
const v_float64
@@ -791,20 +791,20 @@ void log64f( const double *x, double *y, int n )
v_int64 h0 = vx_load((const int64*)x + i);
v_int32 yi0 = v_pack(v_shr<52>(h0), vx_setzero_s64());
yi0 = (yi0 & vx_setall_s32(0x7ff)) - vx_setall_s32(1023);
yi0 = v_sub(v_and(yi0, vx_setall_s32(2047)), vx_setall_s32(1023));
v_int64 xi0 = (h0 & vx_setall_s64(LOGTAB_MASK2_64F)) | vx_setall_s64((int64)1023 << 52);
v_int64 xi0 = v_or(v_and(h0, vx_setall_s64(LOGTAB_MASK2_64F)), vx_setall_s64((int64)1023 << 52));
h0 = v_shr<52 - LOGTAB_SCALE - 1>(h0);
v_int32 idx = v_pack(h0, h0) & vx_setall_s32(LOGTAB_MASK*2);
v_int32 idx = v_and(v_pack(h0, h0), vx_setall_s32(((1 << 8) - 1) * 2));
v_float64 xf0, yf0;
v_lut_deinterleave(logTab, idx, yf0, xf0);
yf0 = v_fma(v_cvt_f64(yi0), vln2, yf0);
v_float64 delta = v_cvt_f64(idx == vx_setall_s32(510))*vx_setall_f64(1./512);
xf0 = v_fma(v_reinterpret_as_f64(xi0) - vx_setall_f64(1.), xf0, delta);
v_float64 delta = v_mul(v_cvt_f64(v_eq(idx, vx_setall_s32(510))), vx_setall_f64(1. / 512));
xf0 = v_fma(v_sub(v_reinterpret_as_f64(xi0), vx_setall_f64(1.)), xf0, delta);
v_float64 xq = xf0*xf0;
v_float64 xq = v_mul(xf0, xf0);
v_float64 zf0 = v_fma(xq, vA0, vA2);
v_float64 zf1 = v_fma(xq, vA1, vA3);
zf0 = v_fma(zf0, xq, vA4);
+14 -16
View File
@@ -1584,7 +1584,7 @@ transform_16u( const ushort* src, ushort* dst, const float* m, int len, int scn,
v_float32x4 _m2h = v_rotate_left<1>(_m2l);
v_float32x4 _m3h = v_rotate_left<1>(_m3l);
v_int16x8 _delta(0, -32768, -32768, -32768, -32768, -32768, -32768, 0);
for( ; x <= len*3 - v_uint16x8::nlanes; x += 3*v_uint16x8::nlanes/4 )
for( ; x <= len*3 - VTraits<v_uint16x8>::vlanes(); x += 3*VTraits<v_uint16x8>::vlanes()/4 )
v_store(dst + x, v_rotate_right<1>(v_reinterpret_as_u16(v_add_wrap(v_pack(
v_round(v_matmuladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand(src + x ))), _m0h, _m1h, _m2h, _m3h)),
v_round(v_matmuladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand(src + x + 3))), _m0l, _m1l, _m2l, _m3l))), _delta))));
@@ -1664,10 +1664,10 @@ transform_32f( const float* src, float* dst, const float* m, int len, int scn, i
v_float32x4 _m2 = v_load(m + 10);
v_float32x4 _m3 = v_load(m + 15);
v_float32x4 _m4(m[4], m[9], m[14], m[19]);
for( ; x < len*4; x += v_float32x4::nlanes )
for( ; x < len*4; x += VTraits<v_float32x4>::vlanes() )
{
v_float32x4 v_src = v_load(src + x);
v_store(dst + x, v_reduce_sum4(v_src * _m0, v_src * _m1, v_src * _m2, v_src * _m3) + _m4);
v_store(dst + x, v_add(v_reduce_sum4(v_mul(v_src, _m0), v_mul(v_src, _m1), v_mul(v_src, _m2), v_mul(v_src, _m3)), _m4));
}
#else // CV_SIMD_WIDTH >= 16 && !CV_SIMD128
for( ; x < len*4; x += 4 )
@@ -2113,12 +2113,12 @@ MulTransposedR(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
for( k = 0; k < size.height; k++, tsrc += srcstep )
{
v_float64x2 a = v_setall_f64((double)col_buf[k]);
s0 += a * v_load(tsrc+0);
s1 += a * v_load(tsrc+2);
s0 = v_add(s0, v_mul(a, v_load(tsrc + 0)));
s1 = v_add(s1, v_mul(a, v_load(tsrc + 2)));
}
v_store((double*)(tdst+j), s0*v_scale);
v_store((double*)(tdst+j+2), s1*v_scale);
v_store((double*)(tdst+j), v_mul(s0, v_scale));
v_store((double*)(tdst+j+2), v_mul(s1, v_scale));
} else
#endif
{
@@ -2174,12 +2174,12 @@ MulTransposedR(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
for( k = 0; k < size.height; k++, tsrc+=srcstep, d+=deltastep )
{
v_float64x2 a = v_setall_f64((double)col_buf[k]);
s0 += a * (v_load(tsrc+0) - v_load(d+0));
s1 += a * (v_load(tsrc+2) - v_load(d+2));
s0 = v_add(s0, v_mul(a, v_sub(v_load(tsrc + 0), v_load(d + 0))));
s1 = v_add(s1, v_mul(a, v_sub(v_load(tsrc + 2), v_load(d + 2))));
}
v_store((double*)(tdst+j), s0*v_scale);
v_store((double*)(tdst+j+2), s1*v_scale);
v_store((double*)(tdst+j), v_mul(s0, v_scale));
v_store((double*)(tdst+j+2), v_mul(s1, v_scale));
}
else
#endif
@@ -2249,8 +2249,7 @@ MulTransposedL(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
v_float64x2 v_s = v_setzero_f64();
for( k = 0; k <= size.width - 4; k += 4 )
v_s += (v_load(v_tsrc1+k) * v_load(v_tsrc2+k)) +
(v_load(v_tsrc1+k+2) * v_load(v_tsrc2+k+2));
v_s = v_add(v_s, v_add(v_mul(v_load(v_tsrc1 + k), v_load(v_tsrc2 + k)), v_mul(v_load(v_tsrc1 + k + 2), v_load(v_tsrc2 + k + 2))));
s += v_reduce_sum(v_s);
}
else
@@ -2303,8 +2302,7 @@ MulTransposedL(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double
v_float64x2 v_s = v_setzero_f64();
for( k = 0; k <= size.width - 4; k += 4, v_tdelta2 += delta_shift )
v_s += ((v_load(v_tsrc2+k) - v_load(v_tdelta2)) * v_load(v_row_buf+k)) +
((v_load(v_tsrc2+k+2) - v_load(v_tdelta2+2)) * v_load(v_row_buf+k+2));
v_s = v_add(v_s, v_add(v_mul(v_sub(v_load(v_tsrc2 + k), v_load(v_tdelta2)), v_load(v_row_buf + k)), v_mul(v_sub(v_load(v_tsrc2 + k + 2), v_load(v_tdelta2 + 2)), v_load(v_row_buf + k + 2))));
s += v_reduce_sum(v_s);
tdelta2 = (const dT *)(v_tdelta2);
@@ -2566,7 +2564,7 @@ double dotProd_32s(const int* src1, const int* src2, int len)
v_sum0 = v_dotprod_expand_fast(v_src10, v_src20, v_sum0);
v_sum1 = v_dotprod_expand_fast(v_src11, v_src21, v_sum1);
}
v_sum0 += v_sum1;
v_sum0 = v_add(v_sum0, v_sum1);
#endif
for (; i < len - step; i += step, src1 += step, src2 += step)
{
+18 -18
View File
@@ -356,10 +356,10 @@ void transposeND(InputArray src_, const std::vector<int>& order, OutputArray dst
#if CV_SIMD128
template<typename V> CV_ALWAYS_INLINE void flipHoriz_single( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size, size_t esz )
{
typedef typename V::lane_type T;
typedef typename VTraits<V>::lane_type T;
int end = (int)(size.width*esz);
int width = (end + 1)/2;
int width_1 = width & -v_uint8x16::nlanes;
int width_1 = width & -VTraits<v_uint8x16>::vlanes();
int i, j;
#if CV_STRONG_ALIGNMENT
@@ -368,15 +368,15 @@ template<typename V> CV_ALWAYS_INLINE void flipHoriz_single( const uchar* src, s
for( ; size.height--; src += sstep, dst += dstep )
{
for( i = 0, j = end; i < width_1; i += v_uint8x16::nlanes, j -= v_uint8x16::nlanes )
for( i = 0, j = end; i < width_1; i += VTraits<v_uint8x16>::vlanes(), j -= VTraits<v_uint8x16>::vlanes() )
{
V t0, t1;
t0 = v_load((T*)((uchar*)src + i));
t1 = v_load((T*)((uchar*)src + j - v_uint8x16::nlanes));
t1 = v_load((T*)((uchar*)src + j - VTraits<v_uint8x16>::vlanes()));
t0 = v_reverse(t0);
t1 = v_reverse(t1);
v_store((T*)(dst + j - v_uint8x16::nlanes), t0);
v_store((T*)(dst + j - VTraits<v_uint8x16>::vlanes()), t0);
v_store((T*)(dst + i), t1);
}
if (isAligned<sizeof(T)>(src, dst))
@@ -446,14 +446,14 @@ flipHoriz( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size,
#if CV_STRONG_ALIGNMENT
size_t alignmentMark = ((size_t)src)|((size_t)dst)|sstep|dstep;
#endif
if (esz == 2 * v_uint8x16::nlanes)
if (esz == 2 * (size_t)VTraits<v_uint8x16>::vlanes())
{
int end = (int)(size.width*esz);
int width = end/2;
for( ; size.height--; src += sstep, dst += dstep )
{
for( int i = 0, j = end - 2 * v_uint8x16::nlanes; i < width; i += 2 * v_uint8x16::nlanes, j -= 2 * v_uint8x16::nlanes )
for( int i = 0, j = end - 2 * VTraits<v_uint8x16>::vlanes(); i < width; i += 2 * VTraits<v_uint8x16>::vlanes(), j -= 2 * VTraits<v_uint8x16>::vlanes() )
{
#if CV_SIMD256
v_uint8x32 t0, t1;
@@ -466,25 +466,25 @@ flipHoriz( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size,
v_uint8x16 t0, t1, t2, t3;
t0 = v_load((uchar*)src + i);
t1 = v_load((uchar*)src + i + v_uint8x16::nlanes);
t1 = v_load((uchar*)src + i + VTraits<v_uint8x16>::vlanes());
t2 = v_load((uchar*)src + j);
t3 = v_load((uchar*)src + j + v_uint8x16::nlanes);
t3 = v_load((uchar*)src + j + VTraits<v_uint8x16>::vlanes());
v_store(dst + j, t0);
v_store(dst + j + v_uint8x16::nlanes, t1);
v_store(dst + j + VTraits<v_uint8x16>::vlanes(), t1);
v_store(dst + i, t2);
v_store(dst + i + v_uint8x16::nlanes, t3);
v_store(dst + i + VTraits<v_uint8x16>::vlanes(), t3);
#endif
}
}
}
else if (esz == v_uint8x16::nlanes)
else if (esz == (size_t)VTraits<v_uint8x16>::vlanes())
{
int end = (int)(size.width*esz);
int width = end/2;
for( ; size.height--; src += sstep, dst += dstep )
{
for( int i = 0, j = end - v_uint8x16::nlanes; i < width; i += v_uint8x16::nlanes, j -= v_uint8x16::nlanes )
for( int i = 0, j = end - VTraits<v_uint8x16>::vlanes(); i < width; i += VTraits<v_uint8x16>::vlanes(), j -= VTraits<v_uint8x16>::vlanes() )
{
v_uint8x16 t0, t1;
@@ -534,19 +534,19 @@ flipHoriz( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size,
for( ; size.height--; src += sstep, dst += dstep )
{
for ( int i = 0, j = end; i < width; i += v_uint8x16::nlanes + sizeof(uint64_t), j -= v_uint8x16::nlanes + sizeof(uint64_t) )
for ( int i = 0, j = end; i < width; i += VTraits<v_uint8x16>::vlanes() + sizeof(uint64_t), j -= VTraits<v_uint8x16>::vlanes() + sizeof(uint64_t) )
{
v_uint8x16 t0, t1;
uint64_t t2, t3;
t0 = v_load((uchar*)src + i);
t2 = *((uint64_t*)((uchar*)src + i + v_uint8x16::nlanes));
t1 = v_load((uchar*)src + j - v_uint8x16::nlanes - sizeof(uint64_t));
t2 = *((uint64_t*)((uchar*)src + i + VTraits<v_uint8x16>::vlanes()));
t1 = v_load((uchar*)src + j - VTraits<v_uint8x16>::vlanes() - sizeof(uint64_t));
t3 = *((uint64_t*)((uchar*)src + j - sizeof(uint64_t)));
v_store(dst + j - v_uint8x16::nlanes - sizeof(uint64_t), t0);
v_store(dst + j - VTraits<v_uint8x16>::vlanes() - sizeof(uint64_t), t0);
*((uint64_t*)(dst + j - sizeof(uint64_t))) = t2;
v_store(dst + i, t1);
*((uint64_t*)(dst + i + v_uint8x16::nlanes)) = t3;
*((uint64_t*)(dst + i + VTraits<v_uint8x16>::vlanes())) = t3;
}
}
}
+135 -135
View File
@@ -141,7 +141,7 @@ CV_ALWAYS_INLINE uint64_t v_reduce_min(const v_uint64x2& a)
CV_ALWAYS_INLINE v_uint64x2 v_select(const v_uint64x2& mask, const v_uint64x2& a, const v_uint64x2& b)
{
return b ^ ((a ^ b) & mask);
return v_xor(b, v_and(v_xor(a, b), mask));
}
#endif
@@ -151,16 +151,16 @@ minMaxIdx_reduce_##suffix( VT &valMin, VT &valMax, IT &idxMin, IT &idxMax, IT &n
T &minVal, T &maxVal, size_t &minIdx, size_t &maxIdx, \
size_t delta ) \
{ \
if ( v_check_any(idxMin != none) ) \
if ( v_check_any(v_ne(idxMin, none)) ) \
{ \
minVal = v_reduce_min(valMin); \
minIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_setall_##suffix((IR)minVal) == valMin), \
minIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_eq(v_setall_##suffix((IR)minVal), valMin)), \
idxMin, v_setall_##suffix2(maxLimit))) + delta; \
} \
if ( v_check_any(idxMax != none) ) \
if ( v_check_any(v_ne(idxMax, none)) ) \
{ \
maxVal = v_reduce_max(valMax); \
maxIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_setall_##suffix((IR)maxVal) == valMax), \
maxIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_eq(v_setall_##suffix((IR)maxVal), valMax)), \
idxMax, v_setall_##suffix2(maxLimit))) + delta; \
} \
}
@@ -210,18 +210,18 @@ static void minMaxIdx_8u(const uchar* src, const uchar* mask, int* minval, int*
size_t* minidx, size_t* maxidx, int len, size_t startidx )
{
#if CV_SIMD128
if ( len >= v_uint8x16::nlanes )
if ( len >= VTraits<v_uint8x16>::vlanes() )
{
int j, len0;
int minVal, maxVal;
size_t minIdx, maxIdx;
minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx,
(int)0, (int)UCHAR_MAX, v_uint8x16::nlanes, len, startidx, j, len0 );
(int)0, (int)UCHAR_MAX, VTraits<v_uint8x16>::vlanes(), len, startidx, j, len0 );
if ( j <= len0 - v_uint8x16::nlanes )
if ( j <= len0 - VTraits<v_uint8x16>::vlanes() )
{
v_uint8x16 inc = v_setall_u8(v_uint8x16::nlanes);
v_uint8x16 inc = v_setall_u8((uchar)VTraits<v_uint8x16>::vlanes());
v_uint8x16 none = v_reinterpret_as_u8(v_setall_s8(-1));
v_uint8x16 idxStart(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
@@ -235,31 +235,31 @@ static void minMaxIdx_8u(const uchar* src, const uchar* mask, int* minval, int*
if ( !mask )
{
for( ; k < std::min(len0, j + 15 * v_uint8x16::nlanes); k += v_uint8x16::nlanes )
for( ; k < std::min(len0, j + 15 * VTraits<v_uint8x16>::vlanes()); k += VTraits<v_uint8x16>::vlanes() )
{
v_uint8x16 data = v_load(src + k);
v_uint8x16 cmpMin = (data < valMin);
v_uint8x16 cmpMax = (data > valMax);
v_uint8x16 cmpMin = (v_lt(data, valMin));
v_uint8x16 cmpMax = (v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
else
{
for( ; k < std::min(len0, j + 15 * v_uint8x16::nlanes); k += v_uint8x16::nlanes )
for( ; k < std::min(len0, j + 15 * VTraits<v_uint8x16>::vlanes()); k += VTraits<v_uint8x16>::vlanes() )
{
v_uint8x16 data = v_load(src + k);
v_uint8x16 maskVal = v_load(mask + k) != v_setzero_u8();
v_uint8x16 cmpMin = (data < valMin) & maskVal;
v_uint8x16 cmpMax = (data > valMax) & maskVal;
v_uint8x16 maskVal = v_ne(v_load(mask + k), v_setzero_u8());
v_uint8x16 cmpMin = v_and(v_lt(data, valMin), maskVal);
v_uint8x16 cmpMax = v_and(v_gt(data, valMax), maskVal);
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(cmpMin, data, valMin);
valMax = v_select(cmpMax, data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
@@ -287,18 +287,18 @@ static void minMaxIdx_8s(const schar* src, const uchar* mask, int* minval, int*
size_t* minidx, size_t* maxidx, int len, size_t startidx )
{
#if CV_SIMD128
if ( len >= v_int8x16::nlanes )
if ( len >= VTraits<v_int8x16>::vlanes() )
{
int j, len0;
int minVal, maxVal;
size_t minIdx, maxIdx;
minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx,
(int)SCHAR_MIN, (int)SCHAR_MAX, v_int8x16::nlanes, len, startidx, j, len0 );
(int)SCHAR_MIN, (int)SCHAR_MAX, VTraits<v_int8x16>::vlanes(), len, startidx, j, len0 );
if ( j <= len0 - v_int8x16::nlanes )
if ( j <= len0 - VTraits<v_int8x16>::vlanes() )
{
v_uint8x16 inc = v_setall_u8(v_int8x16::nlanes);
v_uint8x16 inc = v_setall_u8((uchar)VTraits<v_int8x16>::vlanes());
v_uint8x16 none = v_reinterpret_as_u8(v_setall_s8(-1));
v_uint8x16 idxStart(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
@@ -312,31 +312,31 @@ static void minMaxIdx_8s(const schar* src, const uchar* mask, int* minval, int*
if ( !mask )
{
for( ; k < std::min(len0, j + 15 * v_int8x16::nlanes); k += v_int8x16::nlanes )
for( ; k < std::min(len0, j + 15 * VTraits<v_int8x16>::vlanes()); k += VTraits<v_int8x16>::vlanes() )
{
v_int8x16 data = v_load(src + k);
v_uint8x16 cmpMin = v_reinterpret_as_u8(data < valMin);
v_uint8x16 cmpMax = v_reinterpret_as_u8(data > valMax);
v_uint8x16 cmpMin = v_reinterpret_as_u8(v_lt(data, valMin));
v_uint8x16 cmpMax = v_reinterpret_as_u8(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
else
{
for( ; k < std::min(len0, j + 15 * v_int8x16::nlanes); k += v_int8x16::nlanes )
for( ; k < std::min(len0, j + 15 * VTraits<v_int8x16>::vlanes()); k += VTraits<v_int8x16>::vlanes() )
{
v_int8x16 data = v_load(src + k);
v_uint8x16 maskVal = v_load(mask + k) != v_setzero_u8();
v_uint8x16 cmpMin = v_reinterpret_as_u8(data < valMin) & maskVal;
v_uint8x16 cmpMax = v_reinterpret_as_u8(data > valMax) & maskVal;
v_uint8x16 maskVal = v_ne(v_load(mask + k), v_setzero_u8());
v_uint8x16 cmpMin = v_and(v_reinterpret_as_u8(v_lt(data, valMin)), maskVal);
v_uint8x16 cmpMax = v_and(v_reinterpret_as_u8(v_gt(data, valMax)), maskVal);
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_s8(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_s8(cmpMax), data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
@@ -364,18 +364,18 @@ static void minMaxIdx_16u(const ushort* src, const uchar* mask, int* minval, int
size_t* minidx, size_t* maxidx, int len, size_t startidx )
{
#if CV_SIMD128
if ( len >= v_uint16x8::nlanes )
if ( len >= VTraits<v_uint16x8>::vlanes() )
{
int j, len0;
int minVal, maxVal;
size_t minIdx, maxIdx;
minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx,
(int)0, (int)USHRT_MAX, v_uint16x8::nlanes, len, startidx, j, len0 );
(int)0, (int)USHRT_MAX, VTraits<v_uint16x8>::vlanes(), len, startidx, j, len0 );
if ( j <= len0 - v_uint16x8::nlanes )
if ( j <= len0 - VTraits<v_uint16x8>::vlanes() )
{
v_uint16x8 inc = v_setall_u16(v_uint16x8::nlanes);
v_uint16x8 inc = v_setall_u16((uchar)VTraits<v_uint16x8>::vlanes());
v_uint16x8 none = v_reinterpret_as_u16(v_setall_s16(-1));
v_uint16x8 idxStart(0, 1, 2, 3, 4, 5, 6, 7);
@@ -389,31 +389,31 @@ static void minMaxIdx_16u(const ushort* src, const uchar* mask, int* minval, int
if ( !mask )
{
for( ; k < std::min(len0, j + 8191 * v_uint16x8::nlanes); k += v_uint16x8::nlanes )
for( ; k < std::min(len0, j + 8191 * VTraits<v_uint16x8>::vlanes()); k += VTraits<v_uint16x8>::vlanes() )
{
v_uint16x8 data = v_load(src + k);
v_uint16x8 cmpMin = (data < valMin);
v_uint16x8 cmpMax = (data > valMax);
v_uint16x8 cmpMin = (v_lt(data, valMin));
v_uint16x8 cmpMax = (v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
else
{
for( ; k < std::min(len0, j + 8191 * v_uint16x8::nlanes); k += v_uint16x8::nlanes )
for( ; k < std::min(len0, j + 8191 * VTraits<v_uint16x8>::vlanes()); k += VTraits<v_uint16x8>::vlanes() )
{
v_uint16x8 data = v_load(src + k);
v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16();
v_uint16x8 cmpMin = (data < valMin) & maskVal;
v_uint16x8 cmpMax = (data > valMax) & maskVal;
v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16());
v_uint16x8 cmpMin = v_and(v_lt(data, valMin), maskVal);
v_uint16x8 cmpMax = v_and(v_gt(data, valMax), maskVal);
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(cmpMin, data, valMin);
valMax = v_select(cmpMax, data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
@@ -441,18 +441,18 @@ static void minMaxIdx_16s(const short* src, const uchar* mask, int* minval, int*
size_t* minidx, size_t* maxidx, int len, size_t startidx )
{
#if CV_SIMD128
if ( len >= v_int16x8::nlanes )
if ( len >= VTraits<v_int16x8>::vlanes() )
{
int j, len0;
int minVal, maxVal;
size_t minIdx, maxIdx;
minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx,
(int)SHRT_MIN, (int)SHRT_MAX, v_int16x8::nlanes, len, startidx, j, len0 );
(int)SHRT_MIN, (int)SHRT_MAX, VTraits<v_int16x8>::vlanes(), len, startidx, j, len0 );
if ( j <= len0 - v_int16x8::nlanes )
if ( j <= len0 - VTraits<v_int16x8>::vlanes() )
{
v_uint16x8 inc = v_setall_u16(v_int16x8::nlanes);
v_uint16x8 inc = v_setall_u16((uchar)VTraits<v_int16x8>::vlanes());
v_uint16x8 none = v_reinterpret_as_u16(v_setall_s16(-1));
v_uint16x8 idxStart(0, 1, 2, 3, 4, 5, 6, 7);
@@ -466,31 +466,31 @@ static void minMaxIdx_16s(const short* src, const uchar* mask, int* minval, int*
if ( !mask )
{
for( ; k < std::min(len0, j + 8191 * v_int16x8::nlanes); k += v_int16x8::nlanes )
for( ; k < std::min(len0, j + 8191 * VTraits<v_int16x8>::vlanes()); k += VTraits<v_int16x8>::vlanes() )
{
v_int16x8 data = v_load(src + k);
v_uint16x8 cmpMin = v_reinterpret_as_u16(data < valMin);
v_uint16x8 cmpMax = v_reinterpret_as_u16(data > valMax);
v_uint16x8 cmpMin = v_reinterpret_as_u16(v_lt(data, valMin));
v_uint16x8 cmpMax = v_reinterpret_as_u16(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
else
{
for( ; k < std::min(len0, j + 8191 * v_int16x8::nlanes); k += v_int16x8::nlanes )
for( ; k < std::min(len0, j + 8191 * VTraits<v_int16x8>::vlanes()); k += VTraits<v_int16x8>::vlanes() )
{
v_int16x8 data = v_load(src + k);
v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16();
v_uint16x8 cmpMin = v_reinterpret_as_u16(data < valMin) & maskVal;
v_uint16x8 cmpMax = v_reinterpret_as_u16(data > valMax) & maskVal;
v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16());
v_uint16x8 cmpMin = v_and(v_reinterpret_as_u16(v_lt(data, valMin)), maskVal);
v_uint16x8 cmpMax = v_and(v_reinterpret_as_u16(v_gt(data, valMax)), maskVal);
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_s16(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_s16(cmpMax), data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
@@ -518,14 +518,14 @@ static void minMaxIdx_32s(const int* src, const uchar* mask, int* minval, int* m
size_t* minidx, size_t* maxidx, int len, size_t startidx )
{
#if CV_SIMD128
if ( len >= 2 * v_int32x4::nlanes )
if ( len >= 2 * VTraits<v_int32x4>::vlanes() )
{
int j = 0, len0 = len & -(2 * v_int32x4::nlanes);
int j = 0, len0 = len & -(2 * VTraits<v_int32x4>::vlanes());
int minVal = *minval, maxVal = *maxval;
size_t minIdx = *minidx, maxIdx = *maxidx;
{
v_uint32x4 inc = v_setall_u32(v_int32x4::nlanes);
v_uint32x4 inc = v_setall_u32(VTraits<v_int32x4>::vlanes());
v_uint32x4 none = v_reinterpret_as_u32(v_setall_s32(-1));
v_uint32x4 idxStart(0, 1, 2, 3);
@@ -539,49 +539,49 @@ static void minMaxIdx_32s(const int* src, const uchar* mask, int* minval, int* m
if ( !mask )
{
for( ; k < std::min(len0, j + 32766 * 2 * v_int32x4::nlanes); k += 2 * v_int32x4::nlanes )
for( ; k < std::min(len0, j + 32766 * 2 * VTraits<v_int32x4>::vlanes()); k += 2 * VTraits<v_int32x4>::vlanes() )
{
v_int32x4 data = v_load(src + k);
v_uint32x4 cmpMin = v_reinterpret_as_u32(data < valMin);
v_uint32x4 cmpMax = v_reinterpret_as_u32(data > valMax);
v_uint32x4 cmpMin = v_reinterpret_as_u32(v_lt(data, valMin));
v_uint32x4 cmpMax = v_reinterpret_as_u32(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
data = v_load(src + k + v_int32x4::nlanes);
cmpMin = v_reinterpret_as_u32(data < valMin);
cmpMax = v_reinterpret_as_u32(data > valMax);
idx = v_add(idx, inc);
data = v_load(src + k + VTraits<v_int32x4>::vlanes());
cmpMin = v_reinterpret_as_u32(v_lt(data, valMin));
cmpMax = v_reinterpret_as_u32(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
else
{
for( ; k < std::min(len0, j + 32766 * 2 * v_int32x4::nlanes); k += 2 * v_int32x4::nlanes )
for( ; k < std::min(len0, j + 32766 * 2 * VTraits<v_int32x4>::vlanes()); k += 2 * VTraits<v_int32x4>::vlanes() )
{
v_int32x4 data = v_load(src + k);
v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16();
v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16());
v_int32x4 maskVal1, maskVal2;
v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2);
v_uint32x4 cmpMin = v_reinterpret_as_u32((data < valMin) & maskVal1);
v_uint32x4 cmpMax = v_reinterpret_as_u32((data > valMax) & maskVal1);
v_uint32x4 cmpMin = v_reinterpret_as_u32(v_and(v_lt(data, valMin), maskVal1));
v_uint32x4 cmpMax = v_reinterpret_as_u32(v_and(v_gt(data, valMax), maskVal1));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_s32(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_s32(cmpMax), data, valMax);
idx += inc;
data = v_load(src + k + v_int32x4::nlanes);
cmpMin = v_reinterpret_as_u32((data < valMin) & maskVal2);
cmpMax = v_reinterpret_as_u32((data > valMax) & maskVal2);
idx = v_add(idx, inc);
data = v_load(src + k + VTraits<v_int32x4>::vlanes());
cmpMin = v_reinterpret_as_u32(v_and(v_lt(data, valMin), maskVal2));
cmpMax = v_reinterpret_as_u32(v_and(v_gt(data, valMax), maskVal2));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_s32(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_s32(cmpMax), data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
@@ -609,18 +609,18 @@ static void minMaxIdx_32f(const float* src, const uchar* mask, float* minval, fl
size_t* minidx, size_t* maxidx, int len, size_t startidx )
{
#if CV_SIMD128
if ( len >= 2 * v_float32x4::nlanes )
if ( len >= 2 * VTraits<v_float32x4>::vlanes() )
{
int j, len0;
float minVal, maxVal;
size_t minIdx, maxIdx;
minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx,
FLT_MIN, FLT_MAX, 2 * v_float32x4::nlanes, len, startidx, j, len0 );
FLT_MIN, FLT_MAX, 2 * VTraits<v_float32x4>::vlanes(), len, startidx, j, len0 );
if ( j <= len0 - 2 * v_float32x4::nlanes )
if ( j <= len0 - 2 * VTraits<v_float32x4>::vlanes() )
{
v_uint32x4 inc = v_setall_u32(v_float32x4::nlanes);
v_uint32x4 inc = v_setall_u32(VTraits<v_float32x4>::vlanes());
v_uint32x4 none = v_reinterpret_as_u32(v_setall_s32(-1));
v_uint32x4 idxStart(0, 1, 2, 3);
@@ -634,49 +634,49 @@ static void minMaxIdx_32f(const float* src, const uchar* mask, float* minval, fl
if ( !mask )
{
for( ; k < std::min(len0, j + 32766 * 2 * v_float32x4::nlanes); k += 2 * v_float32x4::nlanes )
for( ; k < std::min(len0, j + 32766 * 2 * VTraits<v_float32x4>::vlanes()); k += 2 * VTraits<v_float32x4>::vlanes() )
{
v_float32x4 data = v_load(src + k);
v_uint32x4 cmpMin = v_reinterpret_as_u32(data < valMin);
v_uint32x4 cmpMax = v_reinterpret_as_u32(data > valMax);
v_uint32x4 cmpMin = v_reinterpret_as_u32(v_lt(data, valMin));
v_uint32x4 cmpMax = v_reinterpret_as_u32(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
data = v_load(src + k + v_float32x4::nlanes);
cmpMin = v_reinterpret_as_u32(data < valMin);
cmpMax = v_reinterpret_as_u32(data > valMax);
idx = v_add(idx, inc);
data = v_load(src + k + VTraits<v_float32x4>::vlanes());
cmpMin = v_reinterpret_as_u32(v_lt(data, valMin));
cmpMax = v_reinterpret_as_u32(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
else
{
for( ; k < std::min(len0, j + 32766 * 2 * v_float32x4::nlanes); k += 2 * v_float32x4::nlanes )
for( ; k < std::min(len0, j + 32766 * 2 * VTraits<v_float32x4>::vlanes()); k += 2 * VTraits<v_float32x4>::vlanes() )
{
v_float32x4 data = v_load(src + k);
v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16();
v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16());
v_int32x4 maskVal1, maskVal2;
v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2);
v_uint32x4 cmpMin = v_reinterpret_as_u32(v_reinterpret_as_s32(data < valMin) & maskVal1);
v_uint32x4 cmpMax = v_reinterpret_as_u32(v_reinterpret_as_s32(data > valMax) & maskVal1);
v_uint32x4 cmpMin = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_lt(data, valMin)), maskVal1));
v_uint32x4 cmpMax = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_gt(data, valMax)), maskVal1));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_f32(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_f32(cmpMax), data, valMax);
idx += inc;
data = v_load(src + k + v_float32x4::nlanes);
cmpMin = v_reinterpret_as_u32(v_reinterpret_as_s32(data < valMin) & maskVal2);
cmpMax = v_reinterpret_as_u32(v_reinterpret_as_s32(data > valMax) & maskVal2);
idx = v_add(idx, inc);
data = v_load(src + k + VTraits<v_float32x4>::vlanes());
cmpMin = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_lt(data, valMin)), maskVal2));
cmpMax = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_gt(data, valMax)), maskVal2));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_f32(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_f32(cmpMax), data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
@@ -704,18 +704,18 @@ static void minMaxIdx_64f(const double* src, const uchar* mask, double* minval,
size_t* minidx, size_t* maxidx, int len, size_t startidx )
{
#if CV_SIMD128_64F
if ( len >= 4 * v_float64x2::nlanes )
if ( len >= 4 * VTraits<v_float64x2>::vlanes() )
{
int j, len0;
double minVal, maxVal;
size_t minIdx, maxIdx;
minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx,
DBL_MIN, DBL_MAX, 4 * v_float64x2::nlanes, len, startidx, j, len0 );
DBL_MIN, DBL_MAX, 4 * VTraits<v_float64x2>::vlanes(), len, startidx, j, len0 );
if ( j <= len0 - 4 * v_float64x2::nlanes )
if ( j <= len0 - 4 * VTraits<v_float64x2>::vlanes() )
{
v_uint64x2 inc = v_setall_u64(v_float64x2::nlanes);
v_uint64x2 inc = v_setall_u64(VTraits<v_float64x2>::vlanes());
v_uint64x2 none = v_reinterpret_as_u64(v_setall_s64(-1));
v_uint64x2 idxStart(0, 1);
@@ -729,84 +729,84 @@ static void minMaxIdx_64f(const double* src, const uchar* mask, double* minval,
if ( !mask )
{
for( ; k < std::min(len0, j + 32764 * 4 * v_float64x2::nlanes); k += 4 * v_float64x2::nlanes )
for( ; k < std::min(len0, j + 32764 * 4 * VTraits<v_float64x2>::vlanes()); k += 4 * VTraits<v_float64x2>::vlanes() )
{
v_float64x2 data = v_load(src + k);
v_uint64x2 cmpMin = v_reinterpret_as_u64(data < valMin);
v_uint64x2 cmpMax = v_reinterpret_as_u64(data > valMax);
v_uint64x2 cmpMin = v_reinterpret_as_u64(v_lt(data, valMin));
v_uint64x2 cmpMax = v_reinterpret_as_u64(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
data = v_load(src + k + v_float64x2::nlanes);
cmpMin = v_reinterpret_as_u64(data < valMin);
cmpMax = v_reinterpret_as_u64(data > valMax);
idx = v_add(idx, inc);
data = v_load(src + k + VTraits<v_float64x2>::vlanes());
cmpMin = v_reinterpret_as_u64(v_lt(data, valMin));
cmpMax = v_reinterpret_as_u64(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
data = v_load(src + k + 2 * v_float64x2::nlanes);
cmpMin = v_reinterpret_as_u64(data < valMin);
cmpMax = v_reinterpret_as_u64(data > valMax);
idx = v_add(idx, inc);
data = v_load(src + k + 2 * VTraits<v_float64x2>::vlanes());
cmpMin = v_reinterpret_as_u64(v_lt(data, valMin));
cmpMax = v_reinterpret_as_u64(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
data = v_load(src + k + 3 * v_float64x2::nlanes);
cmpMin = v_reinterpret_as_u64(data < valMin);
cmpMax = v_reinterpret_as_u64(data > valMax);
idx = v_add(idx, inc);
data = v_load(src + k + 3 * VTraits<v_float64x2>::vlanes());
cmpMin = v_reinterpret_as_u64(v_lt(data, valMin));
cmpMax = v_reinterpret_as_u64(v_gt(data, valMax));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_min(data, valMin);
valMax = v_max(data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
else
{
for( ; k < std::min(len0, j + 32764 * 4 * v_float64x2::nlanes); k += 4 * v_float64x2::nlanes )
for( ; k < std::min(len0, j + 32764 * 4 * VTraits<v_float64x2>::vlanes()); k += 4 * VTraits<v_float64x2>::vlanes() )
{
v_float64x2 data = v_load(src + k);
v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16();
v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16());
v_int32x4 maskVal1, maskVal2;
v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2);
v_int64x2 maskVal3, maskVal4;
v_expand(maskVal1, maskVal3, maskVal4);
v_uint64x2 cmpMin = v_reinterpret_as_u64(v_reinterpret_as_s64(data < valMin) & maskVal3);
v_uint64x2 cmpMax = v_reinterpret_as_u64(v_reinterpret_as_s64(data > valMax) & maskVal3);
v_uint64x2 cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal3));
v_uint64x2 cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal3));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax);
idx += inc;
data = v_load(src + k + v_float64x2::nlanes);
cmpMin = v_reinterpret_as_u64(v_reinterpret_as_s64(data < valMin) & maskVal4);
cmpMax = v_reinterpret_as_u64(v_reinterpret_as_s64(data > valMax) & maskVal4);
idx = v_add(idx, inc);
data = v_load(src + k + VTraits<v_float64x2>::vlanes());
cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal4));
cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal4));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax);
idx += inc;
data = v_load(src + k + 2 * v_float64x2::nlanes);
idx = v_add(idx, inc);
data = v_load(src + k + 2 * VTraits<v_float64x2>::vlanes());
v_expand(maskVal2, maskVal3, maskVal4);
cmpMin = v_reinterpret_as_u64(v_reinterpret_as_s64(data < valMin) & maskVal3);
cmpMax = v_reinterpret_as_u64(v_reinterpret_as_s64(data > valMax) & maskVal3);
cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal3));
cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal3));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax);
idx += inc;
data = v_load(src + k + 3 * v_float64x2::nlanes);
cmpMin = v_reinterpret_as_u64(v_reinterpret_as_s64(data < valMin) & maskVal4);
cmpMax = v_reinterpret_as_u64(v_reinterpret_as_s64(data > valMax) & maskVal4);
idx = v_add(idx, inc);
data = v_load(src + k + 3 * VTraits<v_float64x2>::vlanes());
cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal4));
cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal4));
idxMin = v_select(cmpMin, idx, idxMin);
idxMax = v_select(cmpMax, idx, idxMax);
valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin);
valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax);
idx += inc;
idx = v_add(idx, inc);
}
}
+30 -11
View File
@@ -408,6 +408,8 @@ struct HWFeatures
g_hwFeatureNames[CPU_NEON] = "NEON";
g_hwFeatureNames[CPU_NEON_DOTPROD] = "NEON_DOTPROD";
g_hwFeatureNames[CPU_NEON_FP16] = "NEON_FP16";
g_hwFeatureNames[CPU_NEON_BF16] = "NEON_BF16";
g_hwFeatureNames[CPU_VSX] = "VSX";
g_hwFeatureNames[CPU_VSX3] = "VSX3";
@@ -425,6 +427,7 @@ struct HWFeatures
g_hwFeatureNames[CPU_RVV] = "RVV";
g_hwFeatureNames[CPU_LSX] = "LSX";
g_hwFeatureNames[CPU_LASX] = "LASX";
}
@@ -566,10 +569,15 @@ struct HWFeatures
while ((size_t)read(cpufile, &auxv, size_auxv_t) == size_auxv_t)
{
// see https://elixir.bootlin.com/linux/latest/source/arch/arm64/include/uapi/asm/hwcap.h
if (auxv.a_type == AT_HWCAP)
{
have[CV_CPU_NEON_DOTPROD] = (auxv.a_un.a_val & (1 << 20)) != 0;
break;
have[CV_CPU_NEON_DOTPROD] = (auxv.a_un.a_val & (1 << 20)) != 0; // HWCAP_ASIMDDP
have[CV_CPU_NEON_FP16] = (auxv.a_un.a_val & (1 << 10)) != 0; // HWCAP_ASIMDHP
}
else if (auxv.a_type == AT_HWCAP2)
{
have[CV_CPU_NEON_BF16] = (auxv.a_un.a_val & (1 << 14)) != 0; // HWCAP2_BF16
}
}
@@ -623,16 +631,23 @@ struct HWFeatures
have[CV_CPU_NEON] = true;
#endif
#if (defined __ARM_FP && (((__ARM_FP & 0x2) != 0) && defined __ARM_NEON__))
have[CV_CPU_FP16] = true;
#endif
#if (defined __ARM_FEATURE_DOTPROD)
int has_feat_dotprod = 0;
size_t has_feat_dotprod_size = sizeof(has_feat_dotprod);
sysctlbyname("hw.optional.arm.FEAT_DotProd", &has_feat_dotprod, &has_feat_dotprod_size, NULL, 0);
if (has_feat_dotprod) {
have[CV_CPU_NEON_DOTPROD] = true;
}
have[CV_CPU_FP16] = have[CV_CPU_NEON_FP16] = true;
#endif
// system.cpp may be compiled w/o special -march=armv8...+dotprod, -march=armv8...+bf16 etc.,
// so we check for the features in any case, no mater what are the compile flags.
// We check the real hardware capabilities here.
int has_feat_dotprod = 0;
size_t has_feat_dotprod_size = sizeof(has_feat_dotprod);
sysctlbyname("hw.optional.arm.FEAT_DotProd", &has_feat_dotprod, &has_feat_dotprod_size, NULL, 0);
if (has_feat_dotprod) {
have[CV_CPU_NEON_DOTPROD] = true;
}
int has_feat_bf16 = 0;
size_t has_feat_bf16_size = sizeof(has_feat_bf16);
sysctlbyname("hw.optional.arm.FEAT_BF16", &has_feat_bf16, &has_feat_bf16_size, NULL, 0);
if (has_feat_bf16) {
have[CV_CPU_NEON_BF16] = true;
}
#elif (defined __clang__)
#if (defined __ARM_NEON__ || (defined __ARM_NEON && defined __aarch64__))
have[CV_CPU_NEON] = true;
@@ -689,6 +704,10 @@ struct HWFeatures
have[CV_CPU_RVV] = true;
#endif
#if defined __loongarch_sx
have[CV_CPU_LSX] = true;
#endif
#if defined __loongarch_asx
have[CV_CPU_LASX] = true;
#endif
-5
View File
@@ -1745,13 +1745,8 @@ template<typename R> struct TheTest
R a = dataA;
R b = dataB;
#if CV_SIMD_SCALABLE
Data<R> dataEQ = v_eq(a, b);
Data<R> dataNE = v_ne(a, b);
#else
Data<R> dataEQ = (a == b);
Data<R> dataNE = (a != b);
#endif
for (int i = 0; i < VTraits<R>::vlanes(); ++i)
{
+1 -1
View File
@@ -602,7 +602,7 @@ static void setValue(SparseMat& M, const int* idx, double value, RNG& rng)
CV_Error(CV_StsUnsupportedFormat, "");
}
#if defined(__GNUC__) && (__GNUC__ == 11 || __GNUC__ == 12)
#if defined(__GNUC__) && (__GNUC__ == 11 || __GNUC__ == 12 || __GNUC__ == 13)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Warray-bounds"
#endif
+13
View File
@@ -240,6 +240,12 @@ ocv_create_module(${libs} ${dnn_runtime_libs})
ocv_add_samples()
ocv_add_accuracy_tests(${dnn_runtime_libs})
if(NOT BUILD_PROTOBUF)
if(TARGET opencv_test_dnn)
ocv_target_compile_definitions(opencv_test_dnn PRIVATE "OPENCV_DNN_EXTERNAL_PROTOBUF=1")
endif()
endif()
set(perf_path "${CMAKE_CURRENT_LIST_DIR}/perf")
file(GLOB_RECURSE perf_srcs "${perf_path}/*.cpp")
file(GLOB_RECURSE perf_hdrs "${perf_path}/*.hpp" "${perf_path}/*.h")
@@ -295,6 +301,13 @@ if(TARGET ocv.3rdparty.cann AND OPENCV_TEST_DNN_CANN)
endif()
endif()
ocv_option(OPENCV_TEST_DNN_TIMVX "Build test with TIM-VX" (HAVE_TIMVX))
if(OPENCV_TEST_DNN_TIMVX)
if(TARGET opencv_test_dnn)
ocv_target_compile_definitions(opencv_test_dnn PRIVATE "HAVE_TIMVX=1")
endif()
endif()
ocv_option(OPENCV_TEST_DNN_TFLITE "Build test with TFLite" (OPENCV_DNN_TFLITE))
if(OPENCV_TEST_DNN_TFLITE)
if(TARGET opencv_test_dnn)
+17 -1
View File
@@ -343,6 +343,22 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<GatherLayer> create(const LayerParams& params);
};
/** @brief GatherElements layer
* GatherElements takes two inputs data and indices of the same rank r >= 1 and an optional attribute axis and works such that:
* output[i][j][k] = data[index[i][j][k]][j][k] if axis = 0 and r = 3
* output[i][j][k] = data[i][index[i][j][k]][k] if axis = 1 and r = 3
* output[i][j][k] = data[i][j][index[i][j][k]] if axis = 2 and r = 3
*
* Gather, on the other hand, takes a data tensor of rank r >= 1, and indices tensor of rank q, and works such that:
* it gathers the enteries along axis dimension of the input data indexed by indices and concatenates them in an output tensor of rank q + (r - 1)
* e.g. If axis = 0, let k = indices[i_{0}, ..., i_{q-1}] then output[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[k , j_{0}, ..., j_{r-2}]:
**/
class CV_EXPORTS GatherElementsLayer : public Layer
{
public:
static Ptr<GatherElementsLayer> create(const LayerParams& params);
};
class CV_EXPORTS PoolingLayer : public Layer
{
public:
@@ -1127,7 +1143,7 @@ CV__DNN_INLINE_NS_BEGIN
class CV_EXPORTS LayerNormLayer : public Layer
{
public:
bool hasBias;
CV_DEPRECATED_EXTERNAL bool hasBias; // Deprecated, preserve for compatibility
int axis;
float epsilon;
+3 -3
View File
@@ -941,14 +941,14 @@ CV__DNN_INLINE_NS_BEGIN
* * `*.caffemodel` (Caffe, http://caffe.berkeleyvision.org/)
* * `*.pb` (TensorFlow, https://www.tensorflow.org/)
* * `*.weights` (Darknet, https://pjreddie.com/darknet/)
* * `*.bin` (DLDT, https://software.intel.com/openvino-toolkit)
* * `*.bin` | `*.onnx` (OpenVINO, https://software.intel.com/openvino-toolkit)
* * `*.onnx` (ONNX, https://onnx.ai/)
* @param[in] config Text file contains network configuration. It could be a
* file with the following extensions:
* * `*.prototxt` (Caffe, http://caffe.berkeleyvision.org/)
* * `*.pbtxt` (TensorFlow, https://www.tensorflow.org/)
* * `*.cfg` (Darknet, https://pjreddie.com/darknet/)
* * `*.xml` (DLDT, https://software.intel.com/openvino-toolkit)
* * `*.xml` (OpenVINO, https://software.intel.com/openvino-toolkit)
* @param[in] framework Explicit framework name tag to determine a format.
* @returns Net object.
*
@@ -979,7 +979,7 @@ CV__DNN_INLINE_NS_BEGIN
* backend.
*/
CV_EXPORTS_W
Net readNetFromModelOptimizer(const String &xml, const String &bin);
Net readNetFromModelOptimizer(const String &xml, const String &bin = "");
/** @brief Load a network from Intel's Model Optimizer intermediate representation.
* @param[in] bufferModelConfig Buffer contains XML configuration with network's topology.
+51
View File
@@ -633,6 +633,56 @@ PERF_TEST_P_(Layer_LayerNormExpanded, DISABLED_LayerNormExpanded)
test_layer({N, H ,W});
}
struct Layer_GatherElements : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& data_shape, const std::vector<int>& indices_shape, int axis = 0)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());
Mat data(data_shape, CV_32FC1);
Mat indices(indices_shape, CV_32FC1);
randu(data, 0.f, 1.f);
randu(indices, 0, data_shape[axis]);
Net net;
LayerParams lp;
lp.type = "GatherElements";
lp.name = "testLayer";
lp.set("axis", axis);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
// warmup
{
std::vector<String> inpNames(3);
inpNames[0] = "data";
inpNames[1] = "indices";
net.setInputsNames(inpNames);
net.setInput(data, inpNames[0]);
net.setInput(indices, inpNames[1]);
net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}
TEST_CYCLE()
{
Mat res = net.forward();
}
SANITY_CHECK_NOTHING();
}
};
PERF_TEST_P_(Layer_GatherElements, GatherElements)
{
test_layer({2700, 1, 2914}, {2700, 1, 81}, 2);
}
INSTANTIATE_TEST_CASE_P(/**/, Layer_Slice, dnnBackendsAndTargets(/* withInferenceEngine = */false, /* obsolete_withHalide = */false));
INSTANTIATE_TEST_CASE_P(/**/, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
#ifdef HAVE_CUDA
@@ -642,6 +692,7 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_Scatter, testing::Values(std::make_tuple(DNN
INSTANTIATE_TEST_CASE_P(/**/, Layer_ScatterND, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNormExpanded, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_GatherElements, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
typedef TestBaseWithParam<tuple<Vec4i, int, bool, tuple<Backend, Target> > > Layer_FullyConnected;
+5 -3
View File
@@ -39,9 +39,11 @@ Net readNet(const String& _model, const String& _config, const String& _framewor
std::swap(model, config);
return readNetFromDarknet(config, model);
}
if (framework == "dldt" || modelExt == "bin" || configExt == "bin" || modelExt == "xml" || configExt == "xml")
if (framework == "dldt" || framework == "openvino" ||
modelExt == "bin" || configExt == "bin" ||
modelExt == "xml" || configExt == "xml")
{
if (modelExt == "xml" || configExt == "bin")
if (modelExt == "xml" || configExt == "bin" || modelExt == "onnx")
std::swap(model, config);
return readNetFromModelOptimizer(config, model);
}
@@ -62,7 +64,7 @@ Net readNet(const String& _framework, const std::vector<uchar>& bufferModel,
return readNetFromTensorflow(bufferModel, bufferConfig);
else if (framework == "darknet")
return readNetFromDarknet(bufferConfig, bufferModel);
else if (framework == "dldt")
else if (framework == "dldt" || framework == "openvino")
return readNetFromModelOptimizer(bufferConfig, bufferModel);
else if (framework == "tflite")
return readNetFromTFLite(bufferModel);
+140 -28
View File
@@ -22,6 +22,48 @@ Image2BlobParams::Image2BlobParams(const Scalar& scalefactor_, const Size& size_
datalayout(datalayout_), paddingmode(mode_)
{}
void getVector(InputArrayOfArrays images_, std::vector<Mat>& images) {
images_.getMatVector(images);
}
void getVector(InputArrayOfArrays images_, std::vector<UMat>& images) {
images_.getUMatVector(images);
}
void getMat(UMat& blob, InputArray blob_, AccessFlag flag) {
if(blob_.kind() == _InputArray::UMAT)
blob = blob_.getUMat();
else if(blob_.kind() == _InputArray::MAT) {
blob = blob_.getUMat();
}
}
void getMat(Mat& blob, InputArray blob_, AccessFlag flag) {
if(blob_.kind() == _InputArray::UMAT)
blob = blob_.getMat();
else if(blob_.kind() == _InputArray::MAT) {
blob = blob_.getMat();
}
}
void getChannelFromBlob(Mat& m, InputArray blob, int i, int j, int rows, int cols, int type) {
m = Mat(rows, cols, type, blob.getMat().ptr(i, j));
}
void getChannelFromBlob(UMat& m, InputArray blob, int i, int j, int rows, int cols, int type) {
UMat ublob = blob.getUMat();
int offset = (i * ublob.step.p[0] + j * ublob.step.p[1]) / ublob.elemSize();
int length = 1;
for(int i = 0; i < ublob.dims; ++i) {
length *= ublob.size[i];
}
const int newShape[1] { length };
UMat reshaped = ublob.reshape(1, 1, newShape);
UMat roi = reshaped(Rect(0, offset, 1, rows * cols));
m = roi.reshape(CV_MAT_CN(type), rows);
}
Mat blobFromImage(InputArray image, const double scalefactor, const Size& size,
const Scalar& mean, bool swapRB, bool crop, int ddepth)
{
@@ -35,8 +77,13 @@ void blobFromImage(InputArray image, OutputArray blob, double scalefactor,
const Size& size, const Scalar& mean, bool swapRB, bool crop, int ddepth)
{
CV_TRACE_FUNCTION();
std::vector<Mat> images(1, image.getMat());
blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth);
if (image.kind() == _InputArray::UMAT) {
std::vector<UMat> images(1, image.getUMat());
blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth);
} else {
std::vector<Mat> images(1, image.getMat());
blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth);
}
}
Mat blobFromImages(InputArrayOfArrays images, double scalefactor, Size size,
@@ -52,9 +99,9 @@ void blobFromImages(InputArrayOfArrays images_, OutputArray blob_, double scalef
Size size, const Scalar& mean_, bool swapRB, bool crop, int ddepth)
{
CV_TRACE_FUNCTION();
if (images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT &&
if (images_.kind() != _InputArray::STD_VECTOR_UMAT && images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT &&
images_.kind() != _InputArray::STD_VECTOR_VECTOR) {
String error_message = "The data is expected as vectors of vectors or vectors of matrices.";
String error_message = "The data is expected as vectors of vectors, vectors of Mats or vectors of UMats.";
CV_Error(Error::StsBadArg, error_message);
}
Image2BlobParams param(Scalar::all(scalefactor), size, mean_, swapRB, ddepth);
@@ -71,13 +118,6 @@ Mat blobFromImageWithParams(InputArray image, const Image2BlobParams& param)
return blob;
}
void blobFromImageWithParams(InputArray image, OutputArray blob, const Image2BlobParams& param)
{
CV_TRACE_FUNCTION();
std::vector<Mat> images(1, image.getMat());
blobFromImagesWithParams(images, blob, param);
}
Mat blobFromImagesWithParams(InputArrayOfArrays images, const Image2BlobParams& param)
{
CV_TRACE_FUNCTION();
@@ -86,19 +126,22 @@ Mat blobFromImagesWithParams(InputArrayOfArrays images, const Image2BlobParams&
return blob;
}
void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, const Image2BlobParams& param)
template<class Tmat>
void blobFromImagesWithParamsImpl(InputArrayOfArrays images_, Tmat& blob_, const Image2BlobParams& param)
{
CV_TRACE_FUNCTION();
if (images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT &&
images_.kind() != _InputArray::STD_VECTOR_VECTOR) {
String error_message = "The data is expected as vectors of vectors or vectors of matrices.";
if(!std::is_same<Tmat, UMat>::value && !std::is_same<Tmat, Mat>::value) {
String error_message = "The template parameter is expected to be either a cv::Mat or a cv::UMat";
CV_Error(Error::StsBadArg, error_message);
}
CV_CheckType(param.ddepth, param.ddepth == CV_32F || param.ddepth == CV_8U,
"Blob depth should be CV_32F or CV_8U");
Size size = param.size;
std::vector<Mat> images;
images_.getMatVector(images);
std::vector<Tmat> images;
getVector(images_, images);
CV_Assert(!images.empty());
if (param.ddepth == CV_8U)
@@ -164,12 +207,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con
if (images[i].depth() == CV_8U && param.ddepth == CV_32F)
images[i].convertTo(images[i], CV_32F);
images[i] -= mean;
subtract(images[i], mean, images[i]);
multiply(images[i], scalefactor, images[i]);
}
size_t nimages = images.size();
Mat image0 = images[0];
Tmat image0 = images[0];
CV_Assert(image0.dims == 2);
if (param.datalayout == DNN_LAYOUT_NCHW)
@@ -178,21 +221,22 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con
{
int sz[] = { (int)nimages, nch, image0.rows, image0.cols };
blob_.create(4, sz, param.ddepth);
Mat blob = blob_.getMat();
Mat ch[4];
std::vector<Tmat> ch(4);
for (size_t i = 0; i < nimages; i++)
{
const Mat& image = images[i];
const Tmat& image = images[i];
CV_Assert(image.depth() == blob_.depth());
nch = image.channels();
CV_Assert(image.dims == 2 && (nch == 3 || nch == 4));
CV_Assert(image.size() == image0.size());
for (int j = 0; j < nch; j++)
ch[j] = Mat(image.rows, image.cols, param.ddepth, blob.ptr((int)i, j));
for (int j = 0; j < nch; j++) {
getChannelFromBlob(ch[j], blob_, i, j ,image.rows, image.cols, param.ddepth);
}
if (param.swapRB)
std::swap(ch[0], ch[2]);
split(image, ch);
}
}
@@ -201,11 +245,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con
CV_Assert(nch == 1);
int sz[] = { (int)nimages, 1, image0.rows, image0.cols };
blob_.create(4, sz, param.ddepth);
Mat blob = blob_.getMat();
Mat blob;
getMat(blob, blob_, ACCESS_RW);
for (size_t i = 0; i < nimages; i++)
{
const Mat& image = images[i];
const Tmat& image = images[i];
CV_Assert(image.depth() == blob_.depth());
nch = image.channels();
CV_Assert(image.dims == 2 && (nch == 1));
@@ -219,11 +264,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con
{
int sz[] = { (int)nimages, image0.rows, image0.cols, nch};
blob_.create(4, sz, param.ddepth);
Mat blob = blob_.getMat();
Mat blob;
getMat(blob, blob_, ACCESS_RW);
int subMatType = CV_MAKETYPE(param.ddepth, nch);
for (size_t i = 0; i < nimages; i++)
{
const Mat& image = images[i];
const Tmat& image = images[i];
CV_Assert(image.depth() == blob_.depth());
CV_Assert(image.channels() == image0.channels());
CV_Assert(image.size() == image0.size());
@@ -243,6 +289,72 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con
{
CV_Error(Error::StsUnsupportedFormat, "Unsupported data layout in blobFromImagesWithParams function.");
}
CV_Assert(blob_.total());
}
void blobFromImagesWithParams(InputArrayOfArrays images, OutputArray blob, const Image2BlobParams& param) {
CV_TRACE_FUNCTION();
if (images.kind() == _InputArray::STD_VECTOR_UMAT) {
if(blob.kind() == _InputArray::UMAT) {
UMat& u = blob.getUMatRef();
blobFromImagesWithParamsImpl<cv::UMat>(images, u, param);
return;
} else if(blob.kind() == _InputArray::MAT) {
UMat u = blob.getMatRef().getUMat(ACCESS_WRITE);
blobFromImagesWithParamsImpl<cv::UMat>(images, u, param);
u.copyTo(blob);
return;
}
} else if (images.kind() == _InputArray::STD_VECTOR_MAT) {
if(blob.kind() == _InputArray::UMAT) {
Mat m = blob.getUMatRef().getMat(ACCESS_WRITE);
blobFromImagesWithParamsImpl<cv::Mat>(images, m, param);
m.copyTo(blob);
return;
} else if(blob.kind() == _InputArray::MAT) {
Mat& m = blob.getMatRef();
blobFromImagesWithParamsImpl<cv::Mat>(images, m, param);
return;
}
}
CV_Error(Error::StsBadArg, "Images are expected to be a vector of either a Mat or UMat and Blob is expected to be either a Mat or UMat");
}
void blobFromImageWithParams(InputArray image, OutputArray blob, const Image2BlobParams& param)
{
CV_TRACE_FUNCTION();
if (image.kind() == _InputArray::UMAT) {
if(blob.kind() == _InputArray::UMAT) {
UMat& u = blob.getUMatRef();
std::vector<UMat> images(1, image.getUMat());
blobFromImagesWithParamsImpl<cv::UMat>(images, u, param);
return;
} else if(blob.kind() == _InputArray::MAT) {
UMat u = blob.getMatRef().getUMat(ACCESS_RW);
std::vector<UMat> images(1, image.getUMat());
blobFromImagesWithParamsImpl<cv::UMat>(images, u, param);
u.copyTo(blob);
return;
}
} else if (image.kind() == _InputArray::MAT) {
if(blob.kind() == _InputArray::UMAT) {
Mat m = blob.getUMatRef().getMat(ACCESS_RW);
std::vector<Mat> images(1, image.getMat());
blobFromImagesWithParamsImpl<cv::Mat>(images, m, param);
m.copyTo(blob);
return;
} else if(blob.kind() == _InputArray::MAT) {
Mat& m = blob.getMatRef();
std::vector<Mat> images(1, image.getMat());
blobFromImagesWithParamsImpl<cv::Mat>(images, m, param);
return;
}
}
CV_Error(Error::StsBadArg, "Image an Blob are expected to be either a Mat or UMat");
}
void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_)
+1
View File
@@ -157,6 +157,7 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(Arg, ArgLayer);
CV_DNN_REGISTER_LAYER_CLASS(Reciprocal, ReciprocalLayer);
CV_DNN_REGISTER_LAYER_CLASS(Gather, GatherLayer);
CV_DNN_REGISTER_LAYER_CLASS(GatherElements, GatherElementsLayer);
CV_DNN_REGISTER_LAYER_CLASS(LayerNormalization, LayerNormLayer);
CV_DNN_REGISTER_LAYER_CLASS(Expand, ExpandLayer);
@@ -29,10 +29,10 @@ static inline void v_expand_mul_add(const v_int8x16& a, const v_int8x16& b,
v_int32x4 t0, t1;
v_mul_expand(a0, b0, t0, t1);
out0 += t0; out1 += t1;
out0 = v_add(out0, t0); out1 = v_add(out1, t1);
v_mul_expand(a1, b1, t0, t1);
out2 += t0; out3 += t1;
out2 = v_add(out2, t0); out3 = v_add(out3, t1);
}
#endif
@@ -1055,10 +1055,10 @@ public:
v_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3);
v_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3);
vout0 = voutzp + v_round(v_cvt_f32(vout0)*vmult);
vout1 = voutzp + v_round(v_cvt_f32(vout1)*vmult);
vout2 = voutzp + v_round(v_cvt_f32(vout2)*vmult);
vout3 = voutzp + v_round(v_cvt_f32(vout3)*vmult);
vout0 = v_add(voutzp, v_round(v_mul(v_cvt_f32(vout0), vmult)));
vout1 = v_add(voutzp, v_round(v_mul(v_cvt_f32(vout1), vmult)));
vout2 = v_add(voutzp, v_round(v_mul(v_cvt_f32(vout2), vmult)));
vout3 = v_add(voutzp, v_round(v_mul(v_cvt_f32(vout3), vmult)));
vout0 = v_min(v_max(vout0, outmin), outmax);
vout1 = v_min(v_max(vout1, outmin), outmax);
@@ -1408,12 +1408,12 @@ public:
vs12 = v_dotprod_expand_fast(w1, r2, vs12);
vs13 = v_dotprod_expand_fast(w1, r3, vs13);
}
s0 += v_int32x4(v_reduce_sum(vs00), v_reduce_sum(vs01), v_reduce_sum(vs02), v_reduce_sum(vs03));
s1 += v_int32x4(v_reduce_sum(vs10), v_reduce_sum(vs11), v_reduce_sum(vs12), v_reduce_sum(vs13));
s0 = v_add(s0, v_int32x4(v_reduce_sum(vs00), v_reduce_sum(vs01), v_reduce_sum(vs02), v_reduce_sum(vs03)));
s1 = v_add(s1, v_int32x4(v_reduce_sum(vs10), v_reduce_sum(vs11), v_reduce_sum(vs12), v_reduce_sum(vs13)));
if( cn1 == inpCn )
{
s0 = voutzp + v_round(v_cvt_f32(s0)*vmult0);
s1 = voutzp + v_round(v_cvt_f32(s1)*vmult1);
s0 = v_add(voutzp, v_round(v_mul(v_cvt_f32(s0), vmult0)));
s1 = v_add(voutzp, v_round(v_mul(v_cvt_f32(s1), vmult1)));
s0 = v_min(v_max(s0, outmin), outmax);
s1 = v_min(v_max(s1, outmin), outmax);
@@ -323,8 +323,8 @@ public:
vs3 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep*3 + k), vs3);
}
s += v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3));
v_int32x4 out = outzp + v_round(v_cvt_f32(s)*mult);
s = v_add(s, v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3)));
v_int32x4 out = v_add(outzp, v_round(v_mul(v_cvt_f32(s), mult)));
v_store(dptr + i, v_min(v_max(out, outmin), outmax));
}
#endif
+8 -8
View File
@@ -631,17 +631,17 @@ public:
(int)srcData[index + stride_w*10], (int)srcData[index + stride_w*11]);
v_int32x4 v3((int)srcData[index + stride_w*12], (int)srcData[index + stride_w*13],
(int)srcData[index + stride_w*14], (int)srcData[index + stride_w*15]);
sum_val0 += v0;
sum_val1 += v1;
sum_val2 += v2;
sum_val3 += v3;
sum_val0 = v_add(sum_val0, v0);
sum_val1 = v_add(sum_val1, v1);
sum_val2 = v_add(sum_val2, v2);
sum_val3 = v_add(sum_val3, v3);
}
}
sum_val0 = v_round(v_cvt_f32(sum_val0)*ikarea) + voutzp;
sum_val1 = v_round(v_cvt_f32(sum_val1)*ikarea) + voutzp;
sum_val2 = v_round(v_cvt_f32(sum_val2)*ikarea) + voutzp;
sum_val3 = v_round(v_cvt_f32(sum_val3)*ikarea) + voutzp;
sum_val0 = v_add(v_round(v_mul(v_cvt_f32(sum_val0), ikarea)), voutzp);
sum_val1 = v_add(v_round(v_mul(v_cvt_f32(sum_val1), ikarea)), voutzp);
sum_val2 = v_add(v_round(v_mul(v_cvt_f32(sum_val2), ikarea)), voutzp);
sum_val3 = v_add(v_round(v_mul(v_cvt_f32(sum_val3), ikarea)), voutzp);
v_store(dstData + x0, v_pack(v_pack(sum_val0, sum_val1), v_pack(sum_val2, sum_val3)));
x0 += 15;
@@ -236,13 +236,11 @@ void depthWiseBlockConv2D(const float* wptr,
v21 = v_load(imgptr2 + in_j + dilation_w),
v22 = v_load(imgptr2 + in_j + dilation_w*2);
v_float32x4 vout = v00*vw00 + v01*vw01 + v02*vw02 +
v10*vw10 + v11*vw11 + v12*vw12 +
v20*vw20 + v21*vw21 + v22*vw22 + vbias;
v_float32x4 vout = v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_mul(v00, vw00), v_mul(v01, vw01)), v_mul(v02, vw02)), v_mul(v10, vw10)), v_mul(v11, vw11)), v_mul(v12, vw12)), v_mul(v20, vw20)), v_mul(v21, vw21)), v_mul(v22, vw22)), vbias);
if (fusedAdd)
vout = v_load(outptr + out_j) + vout;
vout = v_add(v_load(outptr + out_j), vout);
if (relu)
vout = v_select(vout > z, vout, vout*vrc);
vout = v_select(v_gt(vout, z), vout, v_mul(vout, vrc));
v_store(outptr + out_j, vout);
}
}
@@ -268,14 +266,12 @@ void depthWiseBlockConv2D(const float* wptr,
v_load_deinterleave(imgptr2 + in_j, v20, v21);
v_load_deinterleave(imgptr2 + in_j + 2, v22, unused);
v_float32x4 vout = v00 * vw00 + v01 * vw01 + v02 * vw02 +
v10 * vw10 + v11 * vw11 + v12 * vw12 +
v20 * vw20 + v21 * vw21 + v22 * vw22 + vbias;
v_float32x4 vout = v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_mul(v00, vw00), v_mul(v01, vw01)), v_mul(v02, vw02)), v_mul(v10, vw10)), v_mul(v11, vw11)), v_mul(v12, vw12)), v_mul(v20, vw20)), v_mul(v21, vw21)), v_mul(v22, vw22)), vbias);
if (fusedAdd)
vout = v_load(outptr + out_j) + vout;
vout = v_add(v_load(outptr + out_j), vout);
if (relu)
vout = v_select(vout > z, vout, vout*vrc);
vout = v_select(v_gt(vout, z), vout, v_mul(vout, vrc));
v_store(outptr + out_j, vout);
}
}
@@ -381,11 +377,11 @@ void depthWiseBlockConv1D(const float* wptr,
v01 = v_load(imgptr0 + in_j + dilation_w),
v02 = v_load(imgptr0 + in_j + dilation_w*2);
v_float32x4 vout = v00*vw00 + v01*vw01 + v02*vw02 + vbias;
v_float32x4 vout = v_add(v_add(v_add(v_mul(v00, vw00), v_mul(v01, vw01)), v_mul(v02, vw02)), vbias);
if (fusedAdd)
vout = v_load(outptr + out_j) + vout;
vout = v_add(v_load(outptr + out_j), vout);
if (relu)
vout = v_select(vout > z, vout, vout*vrc);
vout = v_select(v_gt(vout, z), vout, v_mul(vout, vrc));
v_store(outptr + out_j, vout);
}
}
@@ -407,13 +403,13 @@ void depthWiseBlockConv1D(const float* wptr,
v_load_deinterleave(imgptr0 + in_j, v00, v01);
v_load_deinterleave(imgptr0 + in_j + 2, v02, unused);
v_float32x4 vout = v00 * vw00 + v01 * vw01 + v02 * vw02 + vbias;
v_float32x4 vout = v_add(v_add(v_add(v_mul(v00, vw00), v_mul(v01, vw01)), v_mul(v02, vw02)), vbias);
if (fusedAdd)
vout = v_load(outptr + out_j) + vout;
vout = v_add(v_load(outptr + out_j), vout);
if (relu)
vout = v_select(vout > z, vout, vout*vrc);
vout = v_select(v_gt(vout, z), vout, v_mul(vout, vrc));
v_store(outptr + out_j, vout);
}
}
@@ -430,32 +430,32 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
/* Y[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*X */
/* Y[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*X */
v_float32x4 q5_25 = v_setall_f32(5.25f), t00, t01, t10, t11;
t00 = x40 - x20;
t01 = x41 - x21;
t10 = x30 - x50;
t11 = x31 - x51;
v_float32x4 y00 = v_fma(t00, q5_25, x00 - x60);
v_float32x4 y01 = v_fma(t01, q5_25, x01 - x61);
v_float32x4 y70 = v_fma(t10, q5_25, x70 - x10);
v_float32x4 y71 = v_fma(t11, q5_25, x71 - x11);
t00 = v_sub(x40, x20);
t01 = v_sub(x41, x21);
t10 = v_sub(x30, x50);
t11 = v_sub(x31, x51);
v_float32x4 y00 = v_fma(t00, q5_25, v_sub(x00, x60));
v_float32x4 y01 = v_fma(t01, q5_25, v_sub(x01, x61));
v_float32x4 y70 = v_fma(t10, q5_25, v_sub(x70, x10));
v_float32x4 y71 = v_fma(t11, q5_25, v_sub(x71, x11));
/* Y[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*X */
/* Y[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*X */
v_float32x4 qm4_25 = v_setall_f32(-4.25f);
t00 = v_fma(x30, qm4_25, x10 + x50);
t01 = v_fma(x31, qm4_25, x11 + x51);
t10 = v_fma(x40, qm4_25, x20 + x60);
t11 = v_fma(x41, qm4_25, x21 + x61);
t00 = v_fma(x30, qm4_25, v_add(x10, x50));
t01 = v_fma(x31, qm4_25, v_add(x11, x51));
t10 = v_fma(x40, qm4_25, v_add(x20, x60));
t11 = v_fma(x41, qm4_25, v_add(x21, x61));
v_float32x4 y10 = t00 + t10, y11 = t01 + t11;
v_float32x4 y20 = t10 - t00, y21 = t11 - t01;
v_float32x4 y10 = v_add(t00, t10), y11 = v_add(t01, t11);
v_float32x4 y20 = v_sub(t10, t00), y21 = v_sub(t11, t01);
/* Y[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*X */
/* Y[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*X */
v_float32x4 q0_5 = v_setall_f32(0.5f), q0_25 = v_setall_f32(0.25f);
v_float32x4 qm2_5 = v_setall_f32(-2.5f), qm1_25 = v_setall_f32(-1.25f);
t00 = v_fma(x10, q0_5, x50 + x50);
t01 = v_fma(x11, q0_5, x51 + x51);
t00 = v_fma(x10, q0_5, v_add(x50, x50));
t01 = v_fma(x11, q0_5, v_add(x51, x51));
t10 = v_fma(x20, q0_25, x60);
t11 = v_fma(x21, q0_25, x61);
t00 = v_fma(x30, qm2_5, t00);
@@ -463,14 +463,14 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
t10 = v_fma(x40, qm1_25, t10);
t11 = v_fma(x41, qm1_25, t11);
v_float32x4 y30 = t00 + t10, y31 = t01 + t11;
v_float32x4 y40 = t10 - t00, y41 = t11 - t01;
v_float32x4 y30 = v_add(t00, t10), y31 = v_add(t01, t11);
v_float32x4 y40 = v_sub(t10, t00), y41 = v_sub(t11, t01);
/* Y[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*X */
/* Y[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*X */
v_float32x4 q4 = v_setall_f32(4.f), qm5 = v_setall_f32(-5.f);
t00 = v_fma(x50, q0_5, x10 + x10);
t01 = v_fma(x51, q0_5, x11 + x11);
t00 = v_fma(x50, q0_5, v_add(x10, x10));
t01 = v_fma(x51, q0_5, v_add(x11, x11));
t10 = v_fma(x20, q4 , x60);
t11 = v_fma(x21, q4 , x61);
t00 = v_fma(x30, qm2_5, t00);
@@ -478,8 +478,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
t10 = v_fma(x40, qm5 , t10);
t11 = v_fma(x41, qm5 , t11);
v_float32x4 y50 = t00 + t10, y51 = t01 + t11;
v_float32x4 y60 = t10 - t00, y61 = t11 - t01;
v_float32x4 y50 = v_add(t00, t10), y51 = v_add(t01, t11);
v_float32x4 y60 = v_sub(t10, t00), y61 = v_sub(t11, t01);
/* transpose 8x8 matrix with v_transpose4x4 */
@@ -491,29 +491,29 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
/* Z[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*Y */
/* Z[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*Y */
t00 = y010 - y200;
t01 = y410 - y600;
t10 = y300 - y110;
t11 = y700 - y510;
z00 = v_fma(t00, q5_25, y000 - y210);
z01 = v_fma(t01, q5_25, y400 - y610);
z70 = v_fma(t10, q5_25, y310 - y100);
z71 = v_fma(t11, q5_25, y710 - y500);
t00 = v_sub(y010, y200);
t01 = v_sub(y410, y600);
t10 = v_sub(y300, y110);
t11 = v_sub(y700, y510);
z00 = v_fma(t00, q5_25, v_sub(y000, y210));
z01 = v_fma(t01, q5_25, v_sub(y400, y610));
z70 = v_fma(t10, q5_25, v_sub(y310, y100));
z71 = v_fma(t11, q5_25, v_sub(y710, y500));
/* Z[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*Y */
/* Z[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*Y */
t00 = v_fma(y300, qm4_25, y100 + y110);
t01 = v_fma(y700, qm4_25, y500 + y510);
t10 = v_fma(y010, qm4_25, y200 + y210);
t11 = v_fma(y410, qm4_25, y600 + y610);
t00 = v_fma(y300, qm4_25, v_add(y100, y110));
t01 = v_fma(y700, qm4_25, v_add(y500, y510));
t10 = v_fma(y010, qm4_25, v_add(y200, y210));
t11 = v_fma(y410, qm4_25, v_add(y600, y610));
z10 = t00 + t10; z11 = t01 + t11;
z20 = t10 - t00; z21 = t11 - t01;
z10 = v_add(t00, t10); z11 = v_add(t01, t11);
z20 = v_sub(t10, t00); z21 = v_sub(t11, t01);
/* Z[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*Y */
/* Z[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*Y */
t00 = v_fma(y100, q0_5, y110 + y110);
t01 = v_fma(y500, q0_5, y510 + y510);
t00 = v_fma(y100, q0_5, v_add(y110, y110));
t01 = v_fma(y500, q0_5, v_add(y510, y510));
t10 = v_fma(y200, q0_25, y210);
t11 = v_fma(y600, q0_25, y610);
t00 = v_fma(y300, qm2_5, t00);
@@ -521,13 +521,13 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
t10 = v_fma(y010, qm1_25, t10);
t11 = v_fma(y410, qm1_25, t11);
z30 = t00 + t10; z31 = t01 + t11;
z40 = t10 - t00; z41 = t11 - t01;
z30 = v_add(t00, t10); z31 = v_add(t01, t11);
z40 = v_sub(t10, t00); z41 = v_sub(t11, t01);
/* Z[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*Y */
/* Z[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*Y */
t00 = v_fma(y110, q0_5, y100 + y100);
t01 = v_fma(y510, q0_5, y500 + y500);
t00 = v_fma(y110, q0_5, v_add(y100, y100));
t01 = v_fma(y510, q0_5, v_add(y500, y500));
t10 = v_fma(y200, q4, y210);
t11 = v_fma(y600, q4, y610);
t00 = v_fma(y300, qm2_5, t00);
@@ -535,8 +535,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep,
t10 = v_fma(y010, qm5, t10);
t11 = v_fma(y410, qm5, t11);
z50 = t00 + t10; z51 = t01 + t11;
z60 = t10 - t00; z61 = t11 - t01;
z50 = v_add(t00, t10); z51 = v_add(t01, t11);
z60 = v_sub(t10, t00); z61 = v_sub(t11, t01);
}
const int outstep = winoIblock*winoAtomF32*Cg;
@@ -601,12 +601,12 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
{
v_float32x4 s12_0, s12_1, s34_0, s34_1, s56_0, s56_1;
s12_0 = x10 + x20; s12_1 = x11 + x21;
s34_0 = x30 + x40; s34_1 = x31 + x41;
s56_0 = x50 + x60; s56_1 = x51 + x61;
s12_0 = v_add(x10, x20); s12_1 = v_add(x11, x21);
s34_0 = v_add(x30, x40); s34_1 = v_add(x31, x41);
s56_0 = v_add(x50, x60); s56_1 = v_add(x51, x61);
v_float32x4 y00 = x00 + s12_0 + s34_0 + s56_0;
v_float32x4 y01 = x01 + s12_1 + s34_1 + s56_1;
v_float32x4 y00 = v_add(v_add(v_add(x00, s12_0), s34_0), s56_0);
v_float32x4 y01 = v_add(v_add(v_add(x01, s12_1), s34_1), s56_1);
v_float32x4 a0 = v_setall_f32(0.25f), a1 = v_setall_f32(4.0f);
v_float32x4 y20 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0));
@@ -616,13 +616,13 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
v_float32x4 y40 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0));
v_float32x4 y41 = v_fma(s56_1, a0, v_fma(s34_1, a1, s12_1));
s12_0 = x10 - x20; s12_1 = x11 - x21;
s34_0 = x30 - x40; s34_1 = x31 - x41;
s56_0 = x50 - x60; s56_1 = x51 - x61;
s12_0 = v_sub(x10, x20); s12_1 = v_sub(x11, x21);
s34_0 = v_sub(x30, x40); s34_1 = v_sub(x31, x41);
s56_0 = v_sub(x50, x60); s56_1 = v_sub(x51, x61);
a0 = v_setall_f32(1.f/32), a1 = v_setall_f32(32.f);
v_float32x4 y50 = v_fma(s56_0, a0, v_fma(s34_0, a1, x70 + s12_0));
v_float32x4 y51 = v_fma(s56_1, a0, v_fma(s34_1, a1, x71 + s12_1));
v_float32x4 y50 = v_fma(s56_0, a0, v_fma(s34_0, a1, v_add(x70, s12_0)));
v_float32x4 y51 = v_fma(s56_1, a0, v_fma(s34_1, a1, v_add(x71, s12_1)));
a0 = v_setall_f32(0.5f), a1 = v_setall_f32(2.f);
v_float32x4 y10 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0));
@@ -642,12 +642,12 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
v_transpose4x4(y40, y50, y60, y70, y400, y500, y600, y700);
v_transpose4x4(y41, y51, y61, y71, y410, y510, y610, y710);
s12_0 = y100 + y200; s12_1 = y500 + y600;
s34_0 = y300 + y010; s34_1 = y700 + y410;
s56_0 = y110 + y210; s56_1 = y510 + y610;
s12_0 = v_add(y100, y200); s12_1 = v_add(y500, y600);
s34_0 = v_add(y300, y010); s34_1 = v_add(y700, y410);
s56_0 = v_add(y110, y210); s56_1 = v_add(y510, y610);
z00 = y000 + s12_0 + s34_0 + s56_0;
z01 = y400 + s12_1 + s34_1 + s56_1;
z00 = v_add(v_add(v_add(y000, s12_0), s34_0), s56_0);
z01 = v_add(v_add(v_add(y400, s12_1), s34_1), s56_1);
a0 = v_setall_f32(0.25f), a1 = v_setall_f32(4.0f);
z20 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0));
@@ -657,13 +657,13 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
z40 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0));
z41 = v_fma(s56_1, a0, v_fma(s34_1, a1, s12_1));
s12_0 = y100 - y200; s12_1 = y500 - y600;
s34_0 = y300 - y010; s34_1 = y700 - y410;
s56_0 = y110 - y210; s56_1 = y510 - y610;
s12_0 = v_sub(y100, y200); s12_1 = v_sub(y500, y600);
s34_0 = v_sub(y300, y010); s34_1 = v_sub(y700, y410);
s56_0 = v_sub(y110, y210); s56_1 = v_sub(y510, y610);
a0 = v_setall_f32(1.f/32), a1 = v_setall_f32(32.0f);
z50 = v_fma(s56_0, a0, v_fma(s34_0, a1, y310 + s12_0));
z51 = v_fma(s56_1, a0, v_fma(s34_1, a1, y710 + s12_1));
z50 = v_fma(s56_0, a0, v_fma(s34_0, a1, v_add(y310, s12_0)));
z51 = v_fma(s56_1, a0, v_fma(s34_1, a1, v_add(y710, s12_1)));
a0 = v_setall_f32(0.5f), a1 = v_setall_f32(2.0f);
z10 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0));
z11 = v_fma(s56_1, a0, v_fma(s34_1, a1, s12_1));
@@ -673,34 +673,34 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep,
z31 = v_fma(s56_1, a0, v_fma(s34_1, a1, s12_1));
v_float32x4 vbias = v_setall_f32(bias);
z00 += vbias;
z01 += vbias;
z10 += vbias;
z11 += vbias;
z20 += vbias;
z21 += vbias;
z30 += vbias;
z31 += vbias;
z40 += vbias;
z41 += vbias;
z50 += vbias;
z51 += vbias;
z00 = v_add(z00, vbias);
z01 = v_add(z01, vbias);
z10 = v_add(z10, vbias);
z11 = v_add(z11, vbias);
z20 = v_add(z20, vbias);
z21 = v_add(z21, vbias);
z30 = v_add(z30, vbias);
z31 = v_add(z31, vbias);
z40 = v_add(z40, vbias);
z41 = v_add(z41, vbias);
z50 = v_add(z50, vbias);
z51 = v_add(z51, vbias);
}
if (bpptr)
{
z00 += v_load(bpptr);
z01 += v_load_low(bpptr + 4);
z10 += v_load(bpptr + bpstep);
z11 += v_load_low(bpptr + bpstep + 4);
z20 += v_load(bpptr + bpstep*2);
z21 += v_load_low(bpptr + bpstep*2 + 4);
z30 += v_load(bpptr + bpstep*3);
z31 += v_load_low(bpptr + bpstep*3 + 4);
z40 += v_load(bpptr + bpstep*4);
z41 += v_load_low(bpptr + bpstep*4 + 4);
z50 += v_load(bpptr + bpstep*5);
z51 += v_load_low(bpptr + bpstep*5 + 4);
z00 = v_add(z00, v_load(bpptr));
z01 = v_add(z01, v_load_low(bpptr + 4));
z10 = v_add(z10, v_load(bpptr + bpstep));
z11 = v_add(z11, v_load_low(bpptr + bpstep + 4));
z20 = v_add(z20, v_load(bpptr + bpstep * 2));
z21 = v_add(z21, v_load_low(bpptr + bpstep * 2 + 4));
z30 = v_add(z30, v_load(bpptr + bpstep * 3));
z31 = v_add(z31, v_load_low(bpptr + bpstep * 3 + 4));
z40 = v_add(z40, v_load(bpptr + bpstep * 4));
z41 = v_add(z41, v_load_low(bpptr + bpstep * 4 + 4));
z50 = v_add(z50, v_load(bpptr + bpstep * 5));
z51 = v_add(z51, v_load_low(bpptr + bpstep * 5 + 4));
}
if (ifMinMaxAct)
@@ -0,0 +1,160 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../../precomp.hpp"
#include "fast_norm.hpp"
namespace cv { namespace dnn {
void fastNorm(const Mat &input, Mat &output, float epsilon, size_t normalized_axis, bool normalize_variance) {
const auto input_shape = shape(input);
CV_CheckLT(normalized_axis, input_shape.size(), "fastNorm: axis out of range");
size_t loops = static_cast<size_t>(total(input_shape, 0, static_cast<int>(normalized_axis))),
norm_size = static_cast<size_t>(total(input_shape, static_cast<int>(normalized_axis)));
float inv_norm_size = 1.0 / norm_size;
auto fn = [&](const Range &r) {
const auto *input_data = input.ptr<const float>();
auto *output_data = output.ptr<float>();
for (int i = r.start; i < r.end; i++) {
const auto *x = input_data + norm_size * i;
auto *y = output_data + norm_size * i;
float mean = 0.f, mean_square = 0.f;
for (int j = 0; j < norm_size; j++) {
float v = x[j];
mean += v;
mean_square += v * v;
}
mean *= inv_norm_size;
mean_square = std::sqrt(std::max(0.f, mean_square * inv_norm_size - mean * mean) + epsilon);
float inv_stdev = normalize_variance ? 1.f / mean_square : 1.f;
for (size_t j = 0; j < norm_size; j++) {
y[j] = (x[j] - mean) * inv_stdev;
}
}
};
double nstripes = loops * norm_size * (1 / 1024.0);
parallel_for_(Range(0, loops), fn, nstripes);
}
void fastNorm(const Mat &input, const Mat &scale, Mat &output, float epsilon, size_t normalized_axis) {
const auto input_shape = shape(input);
CV_CheckLT(normalized_axis, input_shape.size(), "fastNorm: axis out of range");
size_t loops = static_cast<size_t>(total(input_shape, 0, static_cast<int>(normalized_axis))),
norm_size = static_cast<size_t>(total(input_shape, static_cast<int>(normalized_axis)));
float inv_norm_size = 1.0 / norm_size;
auto fn = [&](const Range &r) {
const auto *input_data = input.ptr<const float>();
const auto *scale_data = scale.ptr<const float>();
auto *output_data = output.ptr<float>();
for (int i = r.start; i < r.end; i++) {
const auto *x = input_data + norm_size * i;
auto *y = output_data + norm_size * i;
float mean = 0.f, mean_square = 0.f;
for (int j = 0; j < norm_size; j++) {
float v = x[j];
mean += v;
mean_square += v * v;
}
mean *= inv_norm_size;
mean_square = std::sqrt(std::max(0.f, mean_square * inv_norm_size - mean * mean) + epsilon);
float inv_stdev = 1.f / mean_square;
for (size_t j = 0; j < norm_size; j++) {
y[j] = scale_data[j] * (x[j] - mean) * inv_stdev;
}
}
};
double nstripes = loops * norm_size * (1 / 1024.0);
parallel_for_(Range(0, loops), fn, nstripes);
}
void fastNorm(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon, size_t normalized_axis) {
const auto input_shape = shape(input);
CV_CheckLT(normalized_axis, input_shape.size(), "fastNorm: axis out of range");
CV_CheckEQ(scale.total(), bias.total(), "fastNorm: scale and bias should have the same shape");
size_t loops = static_cast<size_t>(total(input_shape, 0, static_cast<int>(normalized_axis))),
norm_size = static_cast<size_t>(total(input_shape, static_cast<int>(normalized_axis)));
float inv_norm_size = 1.0 / norm_size;
auto fn = [&](const Range &r) {
const auto *input_data = input.ptr<const float>();
const auto *scale_data = scale.ptr<const float>();
const auto *bias_data = bias.ptr<const float>();
auto *output_data = output.ptr<float>();
for (int i = r.start; i < r.end; i++) {
const auto *x = input_data + norm_size * i;
auto *y = output_data + norm_size * i;
float mean = 0.f, mean_square = 0.f;
for (int j = 0; j < norm_size; j++) {
float v = x[j];
mean += v;
mean_square += v * v;
}
mean *= inv_norm_size;
mean_square = std::sqrt(std::max(0.f, mean_square * inv_norm_size - mean * mean) + epsilon);
float inv_stdev = 1.f / mean_square;
for (size_t j = 0; j < norm_size; j++) {
y[j] = scale_data[j] * (x[j] - mean) * inv_stdev + bias_data[j];
}
}
};
double nstripes = loops * norm_size * (1 / 1024.0);
parallel_for_(Range(0, loops), fn, nstripes);
}
void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon) {
const auto input_shape = shape(input);
CV_CheckEQ(scale.total(), bias.total(), "fastNormChannel: scale and bias should have the same shape");
CV_CheckGE(input.dims, 3, "fastNormChannel: input dimension >= 3");
size_t N = input_shape[0], C = input_shape[1];
size_t loops = N * C,
norm_size = static_cast<size_t>(total(input_shape, 2));
float inv_norm_size = 1.0 / norm_size;
auto fn = [&](const Range &r) {
const auto *input_data = input.ptr<const float>();
const auto *scale_data = scale.ptr<const float>();
const auto *bias_data = bias.ptr<const float>();
auto *output_data = output.ptr<float>();
for (int i = r.start; i < r.end; i++) {
const auto *x = input_data + norm_size * i;
auto *y = output_data + norm_size * i;
float mean = 0.f, mean_square = 0.f;
for (int j = 0; j < norm_size; j++) {
float v = x[j];
mean += v;
mean_square += v * v;
}
mean *= inv_norm_size;
mean_square = std::sqrt(std::max(0.f, mean_square * inv_norm_size - mean * mean) + epsilon);
float inv_stdev = 1.f / mean_square;
size_t c = i % C;
float s = scale_data[c], b = bias_data[c];
for (size_t j = 0; j < norm_size; j++) {
y[j] = s * (x[j] - mean) * inv_stdev + b;
}
}
};
double nstripes = loops * norm_size * (1 / 1024.0);
parallel_for_(Range(0, loops), fn, nstripes);
}
}} // cv::dnn
@@ -0,0 +1,26 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#ifndef OPENCV_DNN_FAST_NORM_HPP
#define OPENCV_DNN_FAST_NORM_HPP
#include <opencv2/dnn/shape_utils.hpp>
namespace cv { namespace dnn {
// Normalization speedup by multi-threading, mainly for Caffe MVN layer which has normalize_variance parameter.
void fastNorm(const Mat &input, Mat &output, float epsilon, size_t normalized_axis = 0, bool normalize_variance = true);
// Normalization speedup by multi-threading with absent bias. Mainly for LayerNormalization.
void fastNorm(const Mat &input, const Mat &scale, Mat &output, float epsilon, size_t normalized_axis = 0);
// Normalization speedup by multi-threading with scale and bias. Mainly for LayerNormalization.
void fastNorm(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon, size_t normalized_axis = 0);
// Channel-wise Normalization speedup by multi-threading. Scale and bias should have the same shape (C). Input should have dimension >= 3.
void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon);
}} // cv::dnn
#endif // OPENCV_DNN_FAST_NORM_HPP
+1 -1
View File
@@ -29,7 +29,7 @@ public:
std::vector<MatShape> &internals) const CV_OVERRIDE
{
Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals);
return true;
return exclusive_raw == 0;
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
+334 -59
View File
@@ -32,15 +32,14 @@ static bool IsTransposeReshapeForEinsum(const std::vector<size_t>& perm,
return true;
}
Mat batchwiseMatMul(
static Mat batchwiseMatMul(
const Mat& input1,
const MatShape& input1ShapeOverride,
const Mat& input2,
const MatShape& input2ShapeOverride)
{
// Sanity checks before the actual MatMul
//input_1.DataType() == input_2.DataType(), "Data types of the inputs must match for MatMul");
CV_CheckType(input1.type(), input2.type(), "Data types of the inputs must match for MatMul");
CV_CheckEQ(input1ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul");
CV_CheckEQ(input2ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul");
CV_CheckEQ((size_t) input1ShapeOverride[0], (size_t) input2ShapeOverride[0], "Batch dimension should match for MatMul;");
@@ -51,8 +50,6 @@ Mat batchwiseMatMul(
size_t K = input1ShapeOverride[2];
size_t N = input2ShapeOverride[2];
//TODO: deal with dynamic shapes
//TODO: deal with reshaping operation (it might not always be needed)
std::vector<Mat> output;
if (batches > 1)
{
@@ -141,26 +138,19 @@ Mat batchwiseMatMul(
return output_buffer;
};
Mat Transpose(
const cv::Mat& input,
static Mat Transpose(
const Mat& input,
const MatShape& input_shape_override,
const std::vector<size_t> permutation)
{
int input_rank = input_shape_override.size();
CV_Assert(input_rank == permutation.size());
// TODO: ouptimize
bool reshape = false;
if (input.dims != input_shape_override.size())
{
reshape = true;
}
bool reshape = input.dims != input_rank;
Mat input_reshaped;
if(reshape)
{
if(reshape){
input_reshaped = input.reshape(1, input_shape_override.size(), input_shape_override.data());
}
@@ -170,13 +160,9 @@ Mat Transpose(
outputDims.emplace_back(input_shape_override[dim]);
Mat output;
// TODO: ouptimize
MatShape tmp_perm;
tmp_perm.reserve(permutation.size());
for (int i = 0; i < permutation.size(); i++)
tmp_perm.emplace_back(static_cast<int>(permutation[i]));
MatShape order(permutation.begin(), permutation.end());
cv::transposeND((reshape ? input_reshaped : input), tmp_perm, output);
cv::transposeND((reshape ? input_reshaped : input), order, output);
return output;
}
@@ -201,12 +187,183 @@ bool IsTransposeRequired(size_t input_rank, const std::vector<size_t>& permutati
return transpose_required;
}
Mat Diagonal(
const cv::Mat& input,
int subscriptIndicesToInputIndex,
int dimIndexInIreprocessedInput)
bool IsTransposeRequiredForDiagonal(int dim1, int dim2, int rank) {
// If the input is 2D, we don't need a transpose
if (rank == 2)
return false;
// If the two dims are the innermost dims, no transpose is required
if ((dim1 == rank - 1 && dim2 == rank - 2) ||
(dim1 == rank - 2 && dim2 == rank - 1))
return false;
// Transpose is required
return true;
}
template <typename T>
Mat DiagonalDataAssignment(Mat input) {
int rank = input.dims;
CV_Assert(rank >= 2);
CV_Assert(input.size[rank - 1] == input.size[rank - 2]);
MatShape original_dims = shape(input);
if (rank > 3){
//reshape to 3D mat
int collapsed_size = 1;
for (int i = 0; i < rank - 2; ++i) {
collapsed_size *= input.size[i];
}
std::vector<int> reshaped_dims = {collapsed_size, input.size[rank - 2], input.size[rank - 1]};
input = input.reshape(1, reshaped_dims);
}
// Compute total number of higher-dimensional slices
int total_slices = input.size[0];
original_dims[rank - 1] = 1; // Set the last dimension to 1, as we have extracted the diagonal
Mat output = Mat(original_dims, input.type());
int inner_stride = input.size[input.dims - 1];
auto inputPtr = input.ptr<T>();
auto outputPtr = output.ptr<T>();
for (int slice = 0; slice < total_slices; ++slice) {
for (int j = 0; j < inner_stride; ++j) {
// Direct memory access using raw pointers
outputPtr[slice * inner_stride + j] = inputPtr[slice * inner_stride * inner_stride + j * inner_stride + j];
}
}
return output;
}
/* Extract the diagonal elements from the last two dimensions of the tensor.
For instance, given an input_shape of [1, 2, 3, 3]:
The flexibility in this implementation allows one to choose which of the two
last dimensions retains its value, determined by the `preserve_innermost_dim_val` parameter.
When preserve_innermost_dim_val == true:
The resulting shape is [1, 2, 1, 3], indicating the diagonal has 3 elements,
and it keeps the dimension value of the innermost dimension.
When preserve_innermost_dim_val == false:
The resulting shape is [1, 2, 3, 1], indicating the diagonal also has 3 elements,
but it retains the dimension value of the penultimate dimension. */
Mat DiagonalInnermostDims(const Mat& input, bool preserve_innermost_dim_val) {
const MatShape input_dims = shape(input);
int rank = input_dims.size();
// This is an internal method and we already have finished all validations in the calling method.
// We proceed without duplicating all validations again here.
// We have a minimalistic check here to make sure the innermost dims have the same dim value
// as the calling method may have done a transpose before calling this method
CV_CheckEQ(input.size[rank - 1], input.size[rank - 2],
"innermost dims should have the same dim value to parse the diagonal elements");
MatShape output_dims = input_dims; // Copy the original dims
if (preserve_innermost_dim_val) {
output_dims[rank - 2] = 1;
} else {
output_dims[rank - 1] = 1;
}
// TODO: hande different types
Mat output = DiagonalDataAssignment<float>(input);
if (output_dims != shape(output)){
CV_Error(Error::StsError, "Output shape does not match with calculated shape");
}
return output;
}
Mat Diagonal(const Mat& input, int dim1, int dim2)
{
CV_Error(Error::StsNotImplemented, "Diagonal Not Implemented Yet");
const MatShape input_dims = shape(input);
int rank = input_dims.size();
if (!(rank >= 2 && dim1 != dim2 && input_dims[dim1] == input_dims[dim2])){
std::string input_dims_str = std::accumulate(std::next(input_dims.begin()), input_dims.end(), std::to_string(input_dims[0]),
[](const std::string& a, int b) {
return a + ' ' + std::to_string(b);
});
CV_Error(Error::StsError, cv::format("Cannot parse the diagonal elements along dims %d and %d for input shape %s",dim1, dim2, input_dims_str.c_str()));
}
int first_dim = std::min(dim1, dim2);
int second_dim = std::max(dim1, dim2);
Mat output;
bool preserve_innermost_dim_val = false;
bool is_transpose_required = IsTransposeRequiredForDiagonal(dim1, dim2, rank);
if (is_transpose_required)
{
std::vector<size_t> permutation(rank, 0);
int first_dim_axis = -1; // This is the axis eventually occupied by the first_dim
// If one of the diagonal dimensions is one of the 2 innermost dims, then leave it as such
// so as to avoid transpose overhead
if (first_dim == rank - 2) { // If rank - 2 is occupied by first_dim, keep it there
permutation[rank - 2] = first_dim;
first_dim_axis = rank - 2;
} else {
if (second_dim != rank - 2) { // If rank - 2 is not occupied by second_dim, then put first_dim there
permutation[rank - 2] = first_dim;
first_dim_axis = rank - 2;
} else { // If rank - 2 is occupied by second_dim, then put first_dim in rank - 1
permutation[rank - 1] = first_dim;
first_dim_axis = rank - 1;
preserve_innermost_dim_val = true; // We always want to preserve the dim value of the first_dim
}
}
// Put the second_dim in the dim not occupied by the first_dim
if (first_dim_axis != rank - 1) {
permutation[rank - 1] = second_dim;
} else {
permutation[rank - 2] = second_dim;
}
size_t iter = 0;
for (int i = 0; i < rank; ++i) {
if (i != first_dim && i != second_dim) {
permutation[iter++] = i;
}
}
// Permutate the input so that the dims from which we need the diagonal forms the innermost dims
Mat transposed = Transpose(input, input_dims, permutation);
// Parse the diagonal from the innermost dims
output = DiagonalInnermostDims(transposed, preserve_innermost_dim_val);
// Swap back the dimensions to the original axes ordering using a "reverse permutation"
// Find the "reverse" permutation
iter = 0;
std::vector<size_t> reverse_permutation(rank, 0);
for (const auto& perm : permutation) {
reverse_permutation[perm] = iter++;
}
// Permutate using the reverse permutation to get back the original axes ordering
// (Pass in CPU Transpose function here as this Diagonal method will only be used for CPU based diagonal parsing)
output = Transpose(output, shape(output), reverse_permutation);
} else {
// No transposing required
output = DiagonalInnermostDims(input, preserve_innermost_dim_val);
}
// Make copy of the output dims
MatShape output_dims = shape(output);
// Unsqueeze the reduced dim
auto iter = output_dims.begin() + second_dim;
output_dims.erase(iter);
output = output.reshape(1, output_dims);
return output;
}
/**
@@ -299,7 +456,7 @@ public:
void parseEquation(String equation);
void processEquation(const std::vector<MatShape>& inputs);
void processBroadcastedDims();
void createOutputSubsctipt();
void validateOutputSubscript();
void calculateOutputShape();
void preProcessInputs(InputArrayOfArrays& inputs);
Mat reduceSum(Mat& src, MatShape& reduceAxis);
@@ -358,7 +515,7 @@ public:
processBroadcastedDims();
// calculate output shape
createOutputSubsctipt();
validateOutputSubscript();
calculateOutputShape();
}
@@ -624,7 +781,7 @@ void LayerEinsumImpl::calculateOutputShape()
{
// Traverse through each of the subscript labels within the output subscript.
bool middleOfEllipsis = false;
// int64_t ellipsisCharCount = 0;
int ellipsisCharCount = 0;
subscriptIndicesToOutputIndices.resize(numLetterIndices, -1);
@@ -636,7 +793,21 @@ void LayerEinsumImpl::calculateOutputShape()
{
if(letter == '.')
{
CV_Error(Error::StsNotImplemented, "Ellipsis are not supported yet");
middleOfEllipsis = true;
// Make sure there aren't more than 3 '.'s in the current subscript
if (++ellipsisCharCount > 3) {
CV_Error(Error::StsError, "Found a '.' not part of an ellipsis in the output subscript provided");
}
if (ellipsisCharCount == 3) { // Ellipsis is complete. Process it.
middleOfEllipsis = false;
for (size_t i = 0; i < numOfEllipsisDims; ++i) {
einsumOutDims.emplace_back(subscriptIndicesToDimValue[i]);
// The ellipsis is seen in the output and hence the corresponding dims are to not be reduced
subscriptIndicesToLastInput[i] = -1;
subscriptIndicesToOutputIndices[i] = outputDimCounter++;
}
}
} else {
CV_CheckEQ(middleOfEllipsis, false,
"Encountered '.' character that is not part of output subscript");
@@ -666,7 +837,7 @@ void LayerEinsumImpl::calculateOutputShape()
}
}
void LayerEinsumImpl::createOutputSubsctipt()
void LayerEinsumImpl::validateOutputSubscript()
{
// The explicit form requires no operation, as the output
// would have already been parsed during the input parsing process.
@@ -679,8 +850,6 @@ void LayerEinsumImpl::createOutputSubsctipt()
{
CV_Error(Error::StsError,
"Provided output subscript does not include ellipsis while Inputs subscrits constain ellipsis");
} else {
CV_Error(Error::StsNotImplemented, "Ellipsis are not yet supported");
}
}
}
@@ -689,9 +858,84 @@ void LayerEinsumImpl::createOutputSubsctipt()
void LayerEinsumImpl::processBroadcastedDims()
{
// Only compute this function if ellipsis "..." was found in the equation
if (numOfEllipsisDims > 0){
// add assert inplace of return bool
CV_Error(Error::StsError, "Ellipsis are not supperted currenly");
if (numOfEllipsisDims > 0)
{
// extend the number of subscript labels to include each ellipsis dim as
// theoretically each ellipsis dim does correspond to a "virtual" subscript label
numLetterIndices += numOfEllipsisDims;
// We are going to assign the broadcasted dims outermost subscript indices (i.e.) 0 -> numOfEllipsisDims - 1
// as most likely bradcasted dims will be batch dimensions (i.e.) outermost dimensions and hence we don't have to pay
// transposing while "homogenizing" the input
// Hence offset all subscript indices by numOfEllipsisDims
for (size_t i = 0; i < numOfLetters; ++i){
if (letter2count[i] != -1){
letter2index[i] += numOfEllipsisDims;
}
}
std::vector<int> tempIndex2LastInput(numLetterIndices, -1);
for (int i = 0; i < subscriptIndicesToLastInput.size(); ++i){
tempIndex2LastInput[i + numOfEllipsisDims] = subscriptIndicesToLastInput[i];
}
subscriptIndicesToLastInput = std::move(tempIndex2LastInput);
std::vector<int> tempIndexToDimValue(numLetterIndices, -1);
for (int i = 0; i < subscriptIndicesToDimValue.size(); ++i){
tempIndexToDimValue[i + numOfEllipsisDims] = subscriptIndicesToDimValue[i];
}
subscriptIndicesToDimValue = std::move(tempIndexToDimValue);
for (size_t i = 0; i < inputSubscriptIndices.size(); ++i)
{
auto& currentInputDimIndicesToSubscriptIndices = inputSubscriptIndices[i];
std::vector<int> tempCurrentInputDimIndicesToSubscriptIndices;
tempCurrentInputDimIndicesToSubscriptIndices.reserve(currentInputDimIndicesToSubscriptIndices.size());
// make sure it is correct
const auto& dims = einsumInpShapes[i];
auto rank = dims.size();
size_t dimIter = 0;
size_t numBroadcastedIndices = 0;
while (dimIter < currentInputDimIndicesToSubscriptIndices.size())
{
auto value = currentInputDimIndicesToSubscriptIndices[dimIter];
if (value == numOfLetters)
{ // This is a broadcasted dim
// Shouldn't hit this error - just a sanity check
CV_Assert(numBroadcastedIndices < numOfEllipsisDims);
tempCurrentInputDimIndicesToSubscriptIndices.push_back(static_cast<int>(numBroadcastedIndices));
subscriptIndicesToLastInput[numBroadcastedIndices] = i;
// This is the first time we are seeing this broadcasted dim
if (subscriptIndicesToDimValue[numBroadcastedIndices] == -1)
{
subscriptIndicesToDimValue[numBroadcastedIndices] = dims[dimIter];
} else { // We have seen this broadcasted dim before
// Check if the previous value is equal to the current value
if (subscriptIndicesToDimValue[numBroadcastedIndices] != dims[dimIter])
{
// If they are not equal, one of them needs to be 1
if (subscriptIndicesToDimValue[numBroadcastedIndices] == 1)
{
subscriptIndicesToDimValue[numBroadcastedIndices] = dims[dimIter];
} else {
CV_CheckEQ(dims[dimIter], 1, "The broadcasted dimensions of the inputs are incompatible");
}
}
}
++numBroadcastedIndices;
} else { // This is a regular dim - offset it by number of broadcasted dims
tempCurrentInputDimIndicesToSubscriptIndices.push_back(value + static_cast<int>(numOfEllipsisDims));
}
++dimIter;
}
// Shouldn't hit this error - just a sanity check
CV_Assert(dimIter == rank);
currentInputDimIndicesToSubscriptIndices = std::move(tempCurrentInputDimIndicesToSubscriptIndices);
}
}
}
@@ -718,18 +962,58 @@ void LayerEinsumImpl::processEquation(const std::vector<MatShape>& inputs)
// Variable to deal with "ellipsis" - '...' in the input
bool middleOfellipsis = false;
int ellipsisCharCount = 0;
for (auto letter : token)
{
// Broadcasting based tokens are not implemented yet
if (letter == '.')
{
CV_Error(Error::StsNotImplemented,
"Broad casting based indices are not supported currently");
} else
{
middleOfellipsis = true;
if (middleOfellipsis)
// there should not be more than 3 '.'s in the current subscript
if (++ellipsisCharCount > 3)
{
CV_Error(Error::StsError, cv::format("Found a '.' not part of an ellipsis in input: %d", inputIdx));
}
// We have seen all 3 '.'s. We can safely process the ellipsis now.
if (ellipsisCharCount == 3)
{
middleOfellipsis = false;
// Example for the following line of code
// Subscript "...ij" for an input of rank 6
// numOfEllipsisDims = 6 - 5 + 3 = 4
int currentNumOfEllipsisDims = static_cast<int>(rank) - token.length() + 3;
CV_CheckGE(currentNumOfEllipsisDims, 0,
"Einsum subscripts string contains too many subscript labels when compared to the rank of the input");
// Theoretically, currentNumOfEllipsisDims could be 0
// Example: For an input of rank 2 paired with a subscript "...ij"
if (currentNumOfEllipsisDims != 0)
{
// We have seen a ellipsis before - make sure ranks align as per the ONNX spec -
// "Ellipsis must indicate a fixed number of dimensions."
if (numOfEllipsisDims != 0){
CV_CheckEQ(numOfEllipsisDims, static_cast<size_t>(currentNumOfEllipsisDims),
"Ellipsis must indicate a fixed number of dimensions across all inputs");
} else {
numOfEllipsisDims = static_cast<size_t>(currentNumOfEllipsisDims);
}
// We reserve 'numOfLetters' for broadcasted dims as we only allow 'a' - 'z'
// and 'A' - 'Z' (0 - 51) for non-broadcasted dims.
// We will assign appropriate indices (based on number of dimensions the ellipsis corresponds to)
// during broadcasting related post-processing.
for (size_t i = 0; i < numOfEllipsisDims; ++i){
currTokenIndices.push_back(numOfLetters);
}
// Offset 'dim_count' by number of dimensions the ellipsis corresponds to
dim_count += numOfEllipsisDims;
}
}
} else {
if (middleOfellipsis){
CV_Error(Error::StsAssert,
cv::format(
"Encountered '.' character that is not part of an ellipsis in the input: [%d]",
@@ -744,8 +1028,7 @@ void LayerEinsumImpl::processEquation(const std::vector<MatShape>& inputs)
// The subscript label was not found in the global subscript label array
// Therefore, it is added to both the local and global subscript arrays
if(letter2count[letterIdx] == 0)
{
if(letter2count[letterIdx] == 0){
letter2index[letterIdx] = numLetterIndices++;
subscriptIndicesToDimValue.push_back(dimValue);
subscriptIndicesToLastInput.push_back(inputIdx);
@@ -756,20 +1039,12 @@ void LayerEinsumImpl::processEquation(const std::vector<MatShape>& inputs)
auto mappedIndx = letter2index[letterIdx];
subscriptIndicesToLastInput[mappedIndx] = inputIdx;
if (subscriptIndicesToDimValue[mappedIndx] != dimValue)
{
if(subscriptIndicesToDimValue[mappedIndx] == 1){
//TODO: uncomment later on
// subscriptIndicesToDimValue[mappedIndx] == dimValue;
} else
{
if (dimValue != 1)
{
CV_Error(Error::StsError, cv::format("Einsum operands can not be broadcasted."
"Check input shapes/equation passed."
"Input shape of operand [%d]", inputIdx) +
cv::format(" is incompatible in the dimention [%zu].", static_cast<size_t>(dim_count)));
}
if (subscriptIndicesToDimValue[mappedIndx] != dimValue) {
if (dimValue != 1) {
CV_Error(Error::StsError, cv::format("Einsum operands can not be broadcasted."
"Check input shapes/equation passed."
"Input shape of operand [%d]", inputIdx) +
cv::format(" is incompatible in the dimention [%zu].", static_cast<size_t>(dim_count)));
}
}
}
+12 -12
View File
@@ -336,10 +336,10 @@ struct ReLUFunctor : public BaseFunctor
v_float32x4 x1 = v_load(srcptr + i + 4);
v_float32x4 x2 = v_load(srcptr + i + 8);
v_float32x4 x3 = v_load(srcptr + i + 12);
x0 = v_select(x0 >= z, x0, x0*s4);
x1 = v_select(x1 >= z, x1, x1*s4);
x2 = v_select(x2 >= z, x2, x2*s4);
x3 = v_select(x3 >= z, x3, x3*s4);
x0 = v_select(v_ge(x0, z), x0, v_mul(x0, s4));
x1 = v_select(v_ge(x1, z), x1, v_mul(x1, s4));
x2 = v_select(v_ge(x2, z), x2, v_mul(x2, s4));
x3 = v_select(v_ge(x3, z), x3, v_mul(x3, s4));
v_store(dstptr + i, x0);
v_store(dstptr + i + 4, x1);
v_store(dstptr + i + 8, x2);
@@ -2288,10 +2288,10 @@ struct ChannelsPReLUFunctor : public BaseFunctor
v_float32x4 x1 = v_load(srcptr + i + 4);
v_float32x4 x2 = v_load(srcptr + i + 8);
v_float32x4 x3 = v_load(srcptr + i + 12);
x0 = v_select(x0 >= z, x0, x0*s4);
x1 = v_select(x1 >= z, x1, x1*s4);
x2 = v_select(x2 >= z, x2, x2*s4);
x3 = v_select(x3 >= z, x3, x3*s4);
x0 = v_select(v_ge(x0, z), x0, v_mul(x0, s4));
x1 = v_select(v_ge(x1, z), x1, v_mul(x1, s4));
x2 = v_select(v_ge(x2, z), x2, v_mul(x2, s4));
x3 = v_select(v_ge(x3, z), x3, v_mul(x3, s4));
v_store(dstptr + i, x0);
v_store(dstptr + i + 4, x1);
v_store(dstptr + i + 8, x2);
@@ -2435,10 +2435,10 @@ struct PReLUFunctor : public ChannelsPReLUFunctor
v_float32x4 s1 = v_load(scaleptr + i + 4);
v_float32x4 s2 = v_load(scaleptr + i + 8);
v_float32x4 s3 = v_load(scaleptr + i + 12);
x0 = v_select(x0 >= z, x0, x0*s0);
x1 = v_select(x1 >= z, x1, x1*s1);
x2 = v_select(x2 >= z, x2, x2*s2);
x3 = v_select(x3 >= z, x3, x3*s3);
x0 = v_select(v_ge(x0, z), x0, v_mul(x0, s0));
x1 = v_select(v_ge(x1, z), x1, v_mul(x1, s1));
x2 = v_select(v_ge(x2, z), x2, v_mul(x2, s2));
x3 = v_select(v_ge(x3, z), x3, v_mul(x3, s3));
v_store(dstptr + i, x0);
v_store(dstptr + i + 4, x1);
v_store(dstptr + i + 8, x2);
@@ -306,7 +306,7 @@ public:
}
v_float32x4 s = v_reduce_sum4(vs0, vs1, vs2, vs3);
s += v_load(biasptr + i);
s = v_add(s, v_load(biasptr + i));
v_store(dptr + i, s);
}
#endif
@@ -0,0 +1,160 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace cv { namespace dnn {
static inline int calculateOffset(int outer_dim, const MatShape &shape_indices, int axis_skip, const MatStep &step_data) {
int offset = 0;
for (int axis = static_cast<int>(shape_indices.size()) - 2; axis >= 0; axis--) {
int dim = shape_indices[axis];
if (axis != axis_skip) {
offset += (outer_dim % dim) * step_data[axis];
}
outer_dim /= dim;
}
return offset;
}
class GatherElementsLayerImpl CV_FINAL : public GatherElementsLayer
{
public:
GatherElementsLayerImpl(const LayerParams& params)
{
setParamsFrom(params);
axis = params.get<int>("axis", 0);
}
virtual bool supportBackend(int backendId) CV_OVERRIDE
{
return backendId == DNN_BACKEND_OPENCV;
}
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const CV_OVERRIDE
{
CV_CheckEQ(inputs.size(), 2ull, "GatherElements: requires two inputs");
const auto &data = inputs[0];
const auto &indices = inputs[1];
CV_CheckEQ(data.size(), indices.size(), "GatherElements: data and indices should have the same dimension");
int normalized_axis = normalize_axis(axis, static_cast<int>(data.size()));
CV_CheckGE(normalized_axis, 0, "GatherElements: axis out of range");
CV_CheckLT(normalized_axis, static_cast<int>(data.size()), "GatherElements: axis out of range");
for (size_t i = 0; i < data.size(); i++) {
if (i != normalized_axis) {
CV_CheckEQ(data[i], indices[i], "GatherElements: shape mismatched");
}
}
outputs.assign(1, inputs[1]); // shape of output is same as indices
return false;
}
virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE {
std::vector<Mat> inputs;
inputs_arr.getMatVector(inputs);
const auto &data = inputs[0];
axis = normalize_axis(axis, data.dims);
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
{
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
if (inputs_arr.depth() == CV_16S)
{
forward_fallback(inputs_arr, outputs_arr, internals_arr);
return;
}
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
const Mat& data = inputs[0];
const Mat& indices = inputs[1];
Mat& out = outputs[0];
typeDispatch(outputs[0].type(), data, indices, out);
}
template <typename T>
void forward_impl(const Mat& data_, const Mat& indices_, Mat& out_)
{
const auto *ptr_data = data_.ptr<const T>();
const auto *ptr_indices = indices_.ptr<const T>();
auto *ptr_out = out_.ptr<T>();
const auto shape_data = shape(data_);
const auto &step_data = data_.step;
const auto shape_indices = shape(indices_);
int inner_most_dim = shape_indices.back();
int axis_dim = shape_data[axis];
size_t axis_step = static_cast<size_t>(step_data[axis] / sizeof(T));
bool innermost_axis = axis == static_cast<int>(shape_data.size() - 1);
auto fn = [&](const Range &r) {
for (int i = r.start; i < r.end; i++) {
auto *data = ptr_data + static_cast<size_t>(calculateOffset(i, shape_indices, axis, step_data) / sizeof(T));
auto *indices = ptr_indices + i * inner_most_dim;
auto *out = ptr_out + i * inner_most_dim;
if (innermost_axis) {
for (int j = 0; j < inner_most_dim; j++) {
int index = static_cast<int>((indices[j] + axis_dim)) % axis_dim; // TODO: Check out-of-range index
out[j] = data[index];
}
} else {
for (int j = 0; j < inner_most_dim; j++) {
int index = static_cast<int>(indices[j] + axis_dim) % axis_dim; // TODO: Check out-of-range index
out[j] = data[index * axis_step + j];
}
}
}
};
int outer_dims = total(shape_indices, 0, shape_indices.size() - 1);
double nstripes = static_cast<size_t>(outer_dims * inner_most_dim * (1 / 1024.0));
parallel_for_(Range(0, outer_dims), fn, nstripes);
}
template<typename... Args>
inline void typeDispatch(const int type, Args&&... args)
{
switch (type)
{
case CV_8U:
forward_impl<uint8_t>(std::forward<Args>(args)...);
break;
case CV_32S:
forward_impl<int32_t>(std::forward<Args>(args)...);
break;
case CV_32F:
forward_impl<float>(std::forward<Args>(args)...);
break;
default:
CV_Error(cv::Error::BadDepth, "DNN/GatherElements: Unsupported type.");
};
}
private:
int axis;
};
Ptr<GatherElementsLayer> GatherElementsLayer::create(const LayerParams& params)
{
return makePtr<GatherElementsLayerImpl>(params);
}
}} // namespace cv::dnn
+17 -93
View File
@@ -4,6 +4,7 @@
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "cpu_kernels/fast_norm.hpp"
namespace cv { namespace dnn {
@@ -15,11 +16,8 @@ public:
setParamsFrom(params);
// standard attr
axis = params.get<int>("axis", 0);
axis = params.get<int>("axis", -1);
epsilon = params.get<float>("epsilon", 1e-5);
// opencv attr
hasBias = params.get<bool>("hasBias", false);
}
virtual bool supportBackend(int backendId) CV_OVERRIDE
@@ -46,104 +44,25 @@ public:
CV_CheckEQ(x_ndims - axis, w_ndims, "LayerNorm: shape of weight does not match with given axis and shape of input");
for (int i = 0; i < w_ndims; ++i)
CV_CheckEQ(x_shape[axis+i], w_shape[i], "LayerNorm: weight dimensions does not match with input dimensions");
if (hasBias)
if (inputs.size() == static_cast<int>(3))
{
CV_CheckEQ(inputs.size(), (size_t)3, "");
auto b_shape = inputs[2];
CV_CheckEQ(w_shape.size(), b_shape.size(), "LayerNorm: shape of weight does not match with shape of bias");
for (size_t i = 0; i < w_shape.size(); ++i)
CV_CheckEQ(w_shape[i], b_shape[i], "LayerNorm: bias dimensions does not match with weight dimensions");
}
// only one output is needed; Mean & InvStdDev are not needed
// in inference and should beomitted in onnx importer
outputs.assign(1, inputs[0]);
return false;
}
template<bool hasBias>
class LayerNormInvoker : public ParallelLoopBody
{
public:
const Mat& src;
const float* scaleData;
const float* biasData;
Mat& dst;
virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE {
std::vector<Mat> inputs;
inputs_arr.getMatVector(inputs);
float epsilon;
int total;
int normSize;
float invNormSize;
LayerNormInvoker(const Mat& src_, const Mat& scale, const Mat* b, Mat& dst_, int axis, float epsilon_)
: src(src_), scaleData(scale.ptr<float>()), biasData(nullptr), dst(dst_), epsilon(epsilon_)
{
if (hasBias)
{
CV_Assert(b != nullptr);
CV_Assert(b->isContinuous());
biasData = (const float*)b->ptr<float>();
}
auto dstShape = shape(dst);
total = std::accumulate(dstShape.begin(), dstShape.begin() + axis, 1, std::multiplies<int>());
normSize = std::accumulate(dstShape.begin() + axis, dstShape.end(), 1, std::multiplies<int>());
invNormSize = 1.0f / normSize;
}
static void run(const Mat& src, const Mat& scale, const Mat* b, Mat& dst, int axis, float epsilon)
{
CV_Assert(src.isContinuous());
CV_Assert(dst.isContinuous());
CV_CheckTypeEQ(src.type(), CV_32F, "DNN/LayerNorm: only support float32");
CV_CheckTypeEQ(src.type(), dst.type(), "");
CV_Assert(scale.isContinuous());
CV_CheckGE(epsilon, 0.0f, "");
LayerNormInvoker p(src, scale, b, dst, axis, epsilon);
double nstripes = ((size_t)p.total * p.normSize) * (1 / 1024.0);
// double nstripes = ((size_t)p.total) * (1 / 1024.0);
parallel_for_(Range(0, p.total), p, nstripes);
}
void operator()(const Range& r) const CV_OVERRIDE
{
int stripeStart = r.start;
int stripeEnd = r.end;
const float* srcData = src.ptr<float>();
float* dstData = dst.ptr<float>();
for (int ofs = stripeStart; ofs < stripeEnd; ++ofs)
{
const float* first = srcData + ofs * normSize;
float* dstFirst = dstData + ofs * normSize;
float mean = 0;
float meanSquare = 0;
for (int h = 0; h < normSize; ++h)
{
float v = first[h];
mean += v;
meanSquare += v * v;
}
mean *= invNormSize;
meanSquare = std::sqrt(std::max(0.f, meanSquare * invNormSize - mean * mean) + epsilon);
float invMeanSquare = 1.0f / meanSquare;
for (int h = 0; h < normSize; ++h)
{
float v = (first[h] - mean) * invMeanSquare * scaleData[h];
if (hasBias) {
v = v + biasData[h];
}
dstFirst[h] = v;
}
}
}
};
const auto input_shape = shape(inputs[0]);
axis = normalize_axis(axis, static_cast<int>(input_shape.size()));
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
{
@@ -160,10 +79,15 @@ public:
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
if (hasBias) {
LayerNormInvoker<true>::run(inputs[0], inputs[1], &inputs[2], outputs[0], axis, epsilon);
const auto &input = inputs[0];
const auto &scale = inputs[1];
auto &output = outputs[0];
if (inputs.size() == 3) {
const auto &bias = inputs[2];
fastNorm(input, scale, bias, output, epsilon, static_cast<size_t>(axis));
} else {
LayerNormInvoker<false>::run(inputs[0], inputs[1], nullptr, outputs[0], axis, epsilon);
fastNorm(input, scale, output, epsilon, static_cast<size_t>(axis));
}
}
};
+5 -4
View File
@@ -149,10 +149,11 @@ void getPoolingKernelParams(const LayerParams &params, std::vector<size_t>& kern
std::vector<size_t>& strides, cv::String &padMode)
{
bool is_global = params.get<bool>("global_pooling", false);
globalPooling.resize(3);
globalPooling[0] = params.get<bool>("global_pooling_d", is_global);
globalPooling[1] = params.get<bool>("global_pooling_h", is_global);
globalPooling[2] = params.get<bool>("global_pooling_w", is_global);
globalPooling.assign({
params.get<bool>("global_pooling_d", is_global),
params.get<bool>("global_pooling_h", is_global),
params.get<bool>("global_pooling_w", is_global)
});
if (globalPooling[0] || globalPooling[1] || globalPooling[2])
{
+14 -64
View File
@@ -46,6 +46,8 @@
#include "../ie_ngraph.hpp"
#include "../op_cuda.hpp"
#include "./cpu_kernels/fast_norm.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#ifdef HAVE_OPENCL
@@ -69,9 +71,12 @@ public:
MVNLayerImpl(const LayerParams& params)
{
setParamsFrom(params);
// Caffe params
normVariance = params.get<bool>("normalize_variance", true);
acrossChannels = params.get<bool>("across_channels", false);
eps = params.get<double>("eps", 1e-9);
fuse_batch_norm = false;
fuse_relu = false;
relu_slope = 0.f;
@@ -310,73 +315,18 @@ public:
return;
}
std::vector<Mat> inputs, outputs, internals;
inputs_arr.getMatVector(inputs);
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs); // assume only one input
outputs_arr.getMatVector(outputs);
internals_arr.getMatVector(internals);
for (size_t inpIdx = 0; inpIdx < inputs.size(); inpIdx++)
{
Mat &inpBlob = inputs[inpIdx];
Mat &outBlob = outputs[inpIdx];
const auto &input = inputs[0];
int splitDim = (acrossChannels) ? 1 : 2;
int i, newRows = 1;
for( i = 0; i < splitDim; i++ )
newRows *= inpBlob.size[i];
Mat inpMat = inpBlob.reshape(1, newRows);
Mat outMat = outBlob.reshape(1, newRows);
if ( inpBlob.total() == newRows )
{
// MVN is applied to single values at an every row.
if (shift.empty())
{
outBlob.setTo(0);
}
else
{
for ( i = 0; i < newRows; i++ )
{
outMat.row(i).setTo(((float*)shift.data)[i]);
}
}
return;
}
Scalar mean, dev;
for ( i = 0; i < newRows; i++)
{
Mat inpRow = inpMat.row(i);
Mat outRow = outMat.row(i);
float weight = 1.f;
float bias = 0.f;
if (fuse_batch_norm)
{
weight = i < scale.cols ? ((float*)scale.data)[i] : weight;
bias = i < shift.cols ? ((float*)shift.data)[i] : bias;
}
cv::meanStdDev(inpRow, mean, (normVariance) ? dev : noArray());
double alpha = 1;
if (normVariance)
{
alpha = 1 / std::sqrt(eps + dev[0]*dev[0]);
}
double normalizationScale = 1.0;
double normalizationShift = 0.0;
if (fuse_batch_norm)
{
normalizationScale = alpha * weight;
normalizationShift = -mean[0] * normalizationScale + bias;
}
else
{
normalizationScale = alpha;
normalizationShift = -mean[0] * alpha;
}
inpRow.convertTo(outRow, outRow.type(), normalizationScale, normalizationShift);
}
if (fuse_batch_norm) { // channel-wise scale/bias of shape (C)
CV_CheckTrue(normVariance, "DNN/MVN: not supported");
fastNormChannel(input, scale, shift, outputs[0], eps);
} else {
size_t axis = acrossChannels ? 1 : 2;
fastNorm(input, outputs[0], eps, axis, normVariance);
}
}
+11 -11
View File
@@ -873,25 +873,25 @@ public:
v_float32x4 max_idx0 = v_setall_f32(-1.f);
v_float32x4 max_idx1 = max_idx0;
int index0 = ystart * inp_width + xstart;
v_float32x4 idx0 = idx00 + v_setall_f32((float)index0);
v_float32x4 idx1 = idx0 + v_setall_f32((float)(stride_w*4));
v_float32x4 idx0 = v_add(idx00, v_setall_f32((float)index0));
v_float32x4 idx1 = v_add(idx0, v_setall_f32((float)(stride_w * 4)));
for (int y = ystart; y < yend; ++y)
{
for (int x = xstart; x < xend; ++x, idx0 += ones, idx1 += ones)
for (int x = xstart; x < xend; ++x, idx0 = v_add(idx0, ones), idx1 = v_add(idx1, ones))
{
const int index = y * inp_width + x;
v_float32x4 v0(srcData[index], srcData[index + stride_w],
srcData[index + stride_w*2], srcData[index + stride_w*3]);
v_float32x4 v1(srcData[index + stride_w*4], srcData[index + stride_w*5],
srcData[index + stride_w*6], srcData[index + stride_w*7]);
max_idx0 = v_select(v0 > max_val0, idx0, max_idx0);
max_idx1 = v_select(v1 > max_val1, idx1, max_idx1);
max_idx0 = v_select(v_gt(v0, max_val0), idx0, max_idx0);
max_idx1 = v_select(v_gt(v1, max_val1), idx1, max_idx1);
max_val0 = v_max(max_val0, v0);
max_val1 = v_max(max_val1, v1);
}
idx0 += idx_delta;
idx1 += idx_delta;
idx0 = v_add(idx0, idx_delta);
idx1 = v_add(idx1, idx_delta);
}
v_store(dstData + x0, max_val0);
v_store(dstData + x0 + 4, max_val1);
@@ -1044,12 +1044,12 @@ public:
srcData[index + stride_w*2], srcData[index + stride_w*3]);
v_float32x4 v1(srcData[index + stride_w*4], srcData[index + stride_w*5],
srcData[index + stride_w*6], srcData[index + stride_w*7]);
sum_val0 += v0;
sum_val1 += v1;
sum_val0 = v_add(sum_val0, v0);
sum_val1 = v_add(sum_val1, v1);
}
}
v_store(dstData + x0, sum_val0*ikarea);
v_store(dstData + x0 + 4, sum_val1*ikarea);
v_store(dstData + x0, v_mul(sum_val0, ikarea));
v_store(dstData + x0 + 4, v_mul(sum_val1, ikarea));
x0 += 7;
}
else
+49 -6
View File
@@ -179,6 +179,7 @@ private:
void parseCast (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseConstantFill (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseGather (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseGatherElements (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
@@ -2555,6 +2556,53 @@ void ONNXImporter::parseGather(LayerParams& layerParams, const opencv_onnx::Node
addLayer(layerParams, node_proto);
}
void ONNXImporter::parseGatherElements(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
CV_CheckEQ(node_proto.input_size(), 2, "GatherElements: two inputs are required");
size_t num_const = 0;
for (size_t i = 0; i < node_proto.input_size(); ++i){
if (constBlobs.find(node_proto.input(i)) != constBlobs.end())
++num_const;
}
if (num_const == node_proto.input_size())
{
std::vector<Mat> inputs, output;
for (size_t i = 0; i < node_proto.input_size(); i++) {
Mat blob = getBlob(node_proto, i);
if (i == 1) { // indices, from int32/int64 to float32 for compatibility
blob.convertTo(blob, CV_32F);
}
inputs.push_back(blob);
}
runLayer(layerParams, inputs, output);
CV_Assert(output.size() == 1);
addConstant(node_proto.output(0), output[0]);
return;
} else if (num_const > 0) {
for (size_t i = 0; i < node_proto.input_size(); i++) {
if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) {
Mat blob = getBlob(node_proto, i);
if (i == 1) { // indices, from int32/int64 to float32 for compatibility
blob.convertTo(blob, CV_32F);
}
LayerParams constParams;
constParams.name = node_proto.input(i);
constParams.type = "Const";
constParams.blobs.push_back(blob);
opencv_onnx::NodeProto proto;
proto.add_output(constParams.name);
addLayer(constParams, proto);
}
}
}
addLayer(layerParams, node_proto);
}
void ONNXImporter::parseConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
bool hasVariableInps = false;
@@ -3113,12 +3161,6 @@ void ONNXImporter::parseLayerNorm(LayerParams& layerParams, const opencv_onnx::N
axis = (axis + inputDims) % inputDims;
layerParams.set("axis", axis);
// check if bias existed
bool hasBias = false;
if (node_proto.input_size() > 2)
hasBias = true;
layerParams.set("hasBias", hasBias);
// constants as constant inputs
for (size_t i = 1; i < node_proto.input_size(); i++)
{
@@ -3902,6 +3944,7 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version)
dispatch["Cast"] = &ONNXImporter::parseCast;
dispatch["ConstantFill"] = dispatch["ConstantOfShape"] = &ONNXImporter::parseConstantFill;
dispatch["Gather"] = &ONNXImporter::parseGather;
dispatch["GatherElements"] = &ONNXImporter::parseGatherElements;
dispatch["Concat"] = &ONNXImporter::parseConcat;
dispatch["Resize"] = &ONNXImporter::parseResize;
dispatch["Upsample"] = &ONNXImporter::parseUpsample;
@@ -260,6 +260,11 @@ const tensorflow::AttrValue& getLayerAttr(const tensorflow::NodeDef &layer, cons
return layer.attr().at(name);
}
#if defined(__GNUC__) && (__GNUC__ == 13)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdangling-reference"
#endif
static DataLayout getDataLayout(const tensorflow::NodeDef& layer)
{
if (hasLayerAttr(layer, "data_format"))
@@ -2978,6 +2983,10 @@ static void addConstNodes(tensorflow::GraphDef& net, std::map<String, int>& cons
CV_LOG_DEBUG(NULL, "DNN/TF: layers_to_ignore.size() = " << layers_to_ignore.size());
}
#if defined(__GNUC__) && (__GNUC__ == 13)
#pragma GCC diagnostic pop
#endif
// If all inputs of specific layer have the same data layout we can say that
// this layer's output has this data layout too. Returns DNN_LAYOUT_UNKNOWN otherwise.
DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer)
+5
View File
@@ -455,6 +455,11 @@ TEST_P(DNNTestNetwork, FastNeuralStyle_eccv16)
l1 = 0.4;
lInf = 7.46;
}
else if (target == DNN_TARGET_CUDA)
{
l1 = 7e-4;
lInf = 2e-2;
}
else if (target == DNN_TARGET_CUDA_FP16)
{
l1 = 0.9;
+20 -4
View File
@@ -764,11 +764,15 @@ TEST_F(Layer_RNN_Test, get_set_test)
TEST_P(Test_Caffe_layers, Accum)
{
#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF
throw SkipTestException("Requires patched protobuf");
#else
if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU)
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
testLayerUsingCaffeModels("accum", false, false, 0.0, 0.0, 2);
testLayerUsingCaffeModels("accum_ref", false, false, 0.0, 0.0, 2);
#endif
}
TEST_P(Test_Caffe_layers, FlowWarp)
@@ -788,29 +792,41 @@ TEST_P(Test_Caffe_layers, ChannelNorm)
TEST_P(Test_Caffe_layers, DataAugmentation)
{
#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF
throw SkipTestException("Requires patched protobuf");
#else
if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
testLayerUsingCaffeModels("data_augmentation", true, false);
testLayerUsingCaffeModels("data_augmentation_2x1", true, false);
testLayerUsingCaffeModels("data_augmentation_8x6", true, false);
#endif
}
TEST_P(Test_Caffe_layers, Resample)
{
#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF
throw SkipTestException("Requires patched protobuf");
#else
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2023000000)
if (backend != DNN_BACKEND_OPENCV)
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);
#endif
testLayerUsingCaffeModels("nearest_2inps", false, false, 0.0, 0.0, 2);
testLayerUsingCaffeModels("nearest", false, false);
#endif
}
TEST_P(Test_Caffe_layers, Correlation)
{
#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF
throw SkipTestException("Requires patched protobuf");
#else
if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER,
CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
testLayerUsingCaffeModels("correlation", false, false, 0.0, 0.0, 2);
#endif
}
TEST_P(Test_Caffe_layers, Convolution2Inputs)
@@ -1651,12 +1667,11 @@ private:
int outWidth, outHeight, zoomFactor;
};
#ifndef OPENCV_DNN_EXTERNAL_PROTOBUF
TEST_P(Test_Caffe_layers, Interp)
#else
TEST_P(Test_Caffe_layers, DISABLED_Interp) // requires patched protobuf (available in OpenCV source tree only)
#endif
{
#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF
throw SkipTestException("Requires patched protobuf");
#else
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000)
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception
@@ -1680,6 +1695,7 @@ TEST_P(Test_Caffe_layers, DISABLED_Interp) // requires patched protobuf (availa
// Test an implemented layer.
testLayerUsingCaffeModels("layer_interp", false, false);
#endif
}
INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_Caffe_layers, dnnBackendsAndTargets());
+30
View File
@@ -6,6 +6,7 @@
// Third party copyrights are property of their respective owners.
#include "test_precomp.hpp"
#include "npy_blob.hpp"
#include <opencv2/core/ocl.hpp>
#include <opencv2/core/opencl/ocl_defs.hpp>
#include <opencv2/dnn/layer.details.hpp> // CV_DNN_REGISTER_LAYER_CLASS
@@ -869,6 +870,35 @@ TEST_P(Test_Model_Optimizer, flexible_inputs)
normAssert(ref, out, 0, 0);
}
TEST_P(Test_Model_Optimizer, readONNX)
{
const Backend backendId = get<0>(GetParam());
const Target targetId = get<1>(GetParam());
ASSERT_EQ(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, backendId);
const std::string& model = findDataFile("dnn/onnx/models/convolution.onnx");
std::vector<Net> nets = {
// Old API
readNetFromModelOptimizer(model, ""),
readNet("", model, "dldt"),
// New API
readNetFromModelOptimizer(model),
readNet(model, "", "openvino")
};
Mat inp = blobFromNPY(findDataFile("dnn/onnx/data/input_convolution.npy"));
Mat ref = blobFromNPY(findDataFile("dnn/onnx/data/output_convolution.npy"));
for (int i = 0; i < nets.size(); ++i) {
nets[i].setPreferableTarget(targetId);
nets[i].setInput(inp);
Mat out = nets[i].forward();
normAssert(out, ref, format("Index: %d", i).c_str());
}
}
INSTANTIATE_TEST_CASE_P(/**/, Test_Model_Optimizer,
dnnBackendsAndTargetsIE()
);
@@ -55,6 +55,9 @@
"test_flatten_negative_axis1",
"test_flatten_negative_axis2",
"test_flatten_negative_axis4",
"test_gather_elements_0",
"test_gather_elements_1",
"test_gather_elements_negative_indices",
"test_logsoftmax_default_axis",
"test_maxpool_2d_dilations",
"test_maxpool_2d_same_lower",
@@ -103,7 +103,6 @@
"test_dynamicquantizelinear_min_adjusted",
"test_dynamicquantizelinear_min_adjusted_expanded",
"test_edge_pad",
"test_einsum_batch_diagonal",
"test_einsum_inner_prod",
"test_equal",
"test_equal_bcast",
@@ -115,9 +114,6 @@
"test_gather_0",
"test_gather_1",
"test_gather_2d_indices",
"test_gather_elements_0",
"test_gather_elements_1",
"test_gather_elements_negative_indices",
"test_gather_negative_indices",
"test_gathernd_example_float32",
"test_gathernd_example_int32",
+40 -1
View File
@@ -9,6 +9,7 @@
#include "test_precomp.hpp"
#include "npy_blob.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include <numeric>
namespace opencv_test { namespace {
template<typename TString>
@@ -791,6 +792,11 @@ TEST_P(Test_ONNX_layers, Concatenation)
testONNXModels("concat_const_blobs");
}
TEST_P(Test_ONNX_layers, CumSumExclusiveInplace)
{
testONNXModels("cumsum_exclusive_inplace");
}
TEST_P(Test_ONNX_layers, Eltwise3D)
{
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000)
@@ -1456,6 +1462,11 @@ TEST_P(Test_ONNX_layers, Einsum_2D)
testONNXModels("einsum_2d", npy, 0, 0, false, false, 2);
}
TEST_P(Test_ONNX_layers, Einsum_2D_Ellipses)
{
testONNXModels("einsum_2d_ellipses", npy, 0, 0, false, false, 2);
}
TEST_P(Test_ONNX_layers, Einsum_3D)
{
testONNXModels("einsum_3d", npy, 0, 0, false, false, 2);
@@ -1481,7 +1492,7 @@ TEST_P(Test_ONNX_layers, DISABLED_Einsum_HadamardProduct)
testONNXModels("einsum_hadamard", npy, 0, 0, false, false, 2);
}
TEST_P(Test_ONNX_layers, DISABLED_Einsum_Batch_Diagonal)
TEST_P(Test_ONNX_layers, Einsum_Batch_Diagonal)
{
testONNXModels("einsum_batch_diagonal", npy, 0, 0, false, false, 1);
}
@@ -2135,6 +2146,34 @@ TEST_P(Test_ONNX_nets, Alexnet)
expectNoFallbacksFromIE(net);
}
TEST_P(Test_ONNX_nets, RAFT)
{
applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_DEBUG_VERYLONG, CV_TEST_TAG_MEMORY_2GB);
std::string weight_path = _tf("models/optical_flow_estimation_raft_2023aug.onnx", false);
std::string img0_path = findDataFile(std::string("gpu/opticalflow/frame0.png"));
std::string img1_path = findDataFile(std::string("gpu/opticalflow/frame1.png"));
Size target_size{480, 360};
auto img0 = imread(img0_path);
auto img1 = imread(img1_path);
auto blob0 = blobFromImage(img0, 1.0, target_size, 0, true);
auto blob1 = blobFromImage(img1, 1.0, target_size, 0, true);
auto net = readNet(weight_path);
net.setInput(blob0, "0");
net.setInput(blob1, "1");
std::vector<std::string> outnames{"12007", "12006"};
std::vector<Mat> outs;
net.forward(outs, outnames);
// output 12006 is not checked to save space in opencv_extra since its ref is > 1MB,
// and output 12006 is calculated from 12007 so checking 12007 is sufficient.
std::string ref_12700_path = _tf("data/output_optical_flow_estimation_raft_2023aug.npy");
auto ref0 = blobFromNPY(ref_12700_path);
normAssert(ref0, outs[0], "", 1e-5, 1.8e-4);
}
TEST_P(Test_ONNX_nets, Squeezenet)
{
testONNXModels("squeezenet", pb);
+18 -18
View File
@@ -120,8 +120,8 @@ void FAST_t(InputArray _img, std::vector<KeyPoint>& keypoints, int threshold, bo
for (; j < img.cols - 16 - 3; j += 16, ptr += 16)
{
v_uint8x16 v = v_load(ptr);
v_int8x16 v0 = v_reinterpret_as_s8((v + t) ^ delta);
v_int8x16 v1 = v_reinterpret_as_s8((v - t) ^ delta);
v_int8x16 v0 = v_reinterpret_as_s8(v_xor(v_add(v, t), delta));
v_int8x16 v1 = v_reinterpret_as_s8(v_xor(v_sub(v, t), delta));
v_int8x16 x0 = v_reinterpret_as_s8(v_sub_wrap(v_load(ptr + pixel[0]), delta));
v_int8x16 x1 = v_reinterpret_as_s8(v_sub_wrap(v_load(ptr + pixel[quarterPatternSize]), delta));
@@ -129,15 +129,15 @@ void FAST_t(InputArray _img, std::vector<KeyPoint>& keypoints, int threshold, bo
v_int8x16 x3 = v_reinterpret_as_s8(v_sub_wrap(v_load(ptr + pixel[3*quarterPatternSize]), delta));
v_int8x16 m0, m1;
m0 = (v0 < x0) & (v0 < x1);
m1 = (x0 < v1) & (x1 < v1);
m0 = m0 | ((v0 < x1) & (v0 < x2));
m1 = m1 | ((x1 < v1) & (x2 < v1));
m0 = m0 | ((v0 < x2) & (v0 < x3));
m1 = m1 | ((x2 < v1) & (x3 < v1));
m0 = m0 | ((v0 < x3) & (v0 < x0));
m1 = m1 | ((x3 < v1) & (x0 < v1));
m0 = m0 | m1;
m0 = v_and(v_lt(v0, x0), v_lt(v0, x1));
m1 = v_and(v_lt(x0, v1), v_lt(x1, v1));
m0 = v_or(m0, v_and(v_lt(v0, x1), v_lt(v0, x2)));
m1 = v_or(m1, v_and(v_lt(x1, v1), v_lt(x2, v1)));
m0 = v_or(m0, v_and(v_lt(v0, x2), v_lt(v0, x3)));
m1 = v_or(m1, v_and(v_lt(x2, v1), v_lt(x3, v1)));
m0 = v_or(m0, v_and(v_lt(v0, x3), v_lt(v0, x0)));
m1 = v_or(m1, v_and(v_lt(x3, v1), v_lt(x0, v1)));
m0 = v_or(m0, m1);
if( !v_check_any(m0) )
continue;
@@ -154,18 +154,18 @@ void FAST_t(InputArray _img, std::vector<KeyPoint>& keypoints, int threshold, bo
v_uint8x16 max1 = v_setzero_u8();
for( k = 0; k < N; k++ )
{
v_int8x16 x = v_reinterpret_as_s8(v_load((ptr + pixel[k])) ^ delta);
m0 = v0 < x;
m1 = x < v1;
v_int8x16 x = v_reinterpret_as_s8(v_xor(v_load((ptr + pixel[k])), delta));
m0 = v_lt(v0, x);
m1 = v_lt(x, v1);
c0 = v_sub_wrap(c0, m0) & m0;
c1 = v_sub_wrap(c1, m1) & m1;
c0 = v_and(v_sub_wrap(c0, m0), m0);
c1 = v_and(v_sub_wrap(c1, m1), m1);
max0 = v_max(max0, v_reinterpret_as_u8(c0));
max1 = v_max(max1, v_reinterpret_as_u8(c1));
}
max0 = K16 < v_max(max0, max1);
max0 = v_lt(K16, v_max(max0, max1));
unsigned int m = v_signmask(v_reinterpret_as_s8(max0));
for( k = 0; m > 0 && k < 16; k++, m >>= 1 )
@@ -190,7 +190,7 @@ void FAST_t(InputArray _img, std::vector<KeyPoint>& keypoints, int threshold, bo
a1 = v_min(a1, v_nms);
b1 = v_max(b1, v_nms);
}
curr[j + k] = (uchar)(v_reduce_max(v_max(v_max(a0, a1), v_setzero_s16() - v_min(b0, b1))) - 1);
curr[j + k] = (uchar)(v_reduce_max(v_max(v_max(a0, a1), v_sub(v_setzero_s16(), v_min(b0, b1)))) - 1);
}
}
}
+3 -3
View File
@@ -160,7 +160,7 @@ int cornerScore<16>(const uchar* ptr, const int pixel[], int threshold)
q0 = v_max(q0, v_min(a, v0));
q1 = v_min(q1, v_max(b, v0));
}
q0 = v_max(q0, v_setzero_s16() - q1);
q0 = v_max(q0, v_sub(v_setzero_s16(), q1));
threshold = v_reduce_max(q0) - 1;
}
else
@@ -251,7 +251,7 @@ int cornerScore<12>(const uchar* ptr, const int pixel[], int threshold)
q0 = v_max(q0, v_min(a, v0));
q1 = v_min(q1, v_max(b, v0));
}
q0 = v_max(q0, v_setzero_s16() - q1);
q0 = v_max(q0, v_sub(v_setzero_s16(), q1));
threshold = v_reduce_max(q0) - 1;
}
else
@@ -323,7 +323,7 @@ int cornerScore<8>(const uchar* ptr, const int pixel[], int threshold)
v0 = v_load(d + 5);
q0 = v_max(q0, v_min(a, v0));
q1 = v_min(q1, v_max(b, v0));
q0 = v_max(q0, v_setzero_s16() - q1);
q0 = v_max(q0, v_sub(v_setzero_s16(), q1));
threshold = v_reduce_max(q0) - 1;
}
else
+3 -3
View File
@@ -1,7 +1,7 @@
set(ade_src_dir "${OpenCV_BINARY_DIR}/3rdparty/ade")
set(ade_filename "v0.1.2c.zip")
set(ade_subdir "ade-0.1.2c")
set(ade_md5 "4e996f545f9dddf2348cf593cbb2726f")
set(ade_filename "v0.1.2d.zip")
set(ade_subdir "ade-0.1.2d")
set(ade_md5 "dbb095a8bf3008e91edbbf45d8d34885")
ocv_download(FILENAME ${ade_filename}
HASH ${ade_md5}
URL
@@ -335,7 +335,7 @@ void run_rgb2hsv_impl(uchar out[], const uchar in[], const int sdiv_table[],
// divide and calculate s according to above feature
v_uint32x4 ss[4];
v_uint32x4 vadd = v_setall_u32(1) << (hsv_shift - 1);
v_uint32x4 vadd = v_shl(v_setall_u32(1), (hsv_shift - 1));
v_uint32x4 v_diff_exp[4];
v_diff_exp[0] = v_reinterpret_as_u32(v_and(v_reinterpret_as_u8(v_diff), mask1));
@@ -406,16 +406,16 @@ void run_rgb2hsv_impl(uchar out[], const uchar in[], const int sdiv_table[],
// start computing H-ch
//h = (_vr & (g - b)) + (~_vr & ((_vg & (b - r + 2 * diff)) + ((~_vg) & (r - g + 4 * diff))));
v_int32x4 hh[4];
hh[0] = v_reinterpret_as_s32(v_select(e[0], v_reinterpret_as_s32(gg[0] - bb[0]),
hh[0] = v_reinterpret_as_s32(v_select(e[0], v_reinterpret_as_s32(v_sub(gg[0], bb[0])),
v_select(p[0], v_reinterpret_as_s32(v_add(v_sub(bb[0], rr[0]), v_mul(v_setall_u32(2), vdd[0]))),
v_reinterpret_as_s32(v_add(v_sub(rr[0], gg[0]), v_mul(v_setall_u32(4), vdd[0]))))));
hh[1] = v_reinterpret_as_s32(v_select(e[1], v_reinterpret_as_s32(gg[1] - bb[1]),
hh[1] = v_reinterpret_as_s32(v_select(e[1], v_reinterpret_as_s32(v_sub(gg[1], bb[1])),
v_select(p[1], v_reinterpret_as_s32(v_add(v_sub(bb[1], rr[1]), v_mul(v_setall_u32(2), vdd[1]))),
v_reinterpret_as_s32(v_add(v_sub(rr[1], gg[1]), v_mul(v_setall_u32(4), vdd[1]))))));
hh[2] = v_reinterpret_as_s32(v_select(e[2], v_reinterpret_as_s32(gg[2] - bb[2]),
hh[2] = v_reinterpret_as_s32(v_select(e[2], v_reinterpret_as_s32(v_sub(gg[2], bb[2])),
v_select(p[2], v_reinterpret_as_s32(v_add(v_sub(bb[2], rr[2]), v_mul(v_setall_u32(2), vdd[2]))),
v_reinterpret_as_s32(v_add(v_sub(rr[2], gg[2]), v_mul(v_setall_u32(4), vdd[2]))))));
hh[3] = v_reinterpret_as_s32(v_select(e[3], v_reinterpret_as_s32(gg[3] - bb[3]),
hh[3] = v_reinterpret_as_s32(v_select(e[3], v_reinterpret_as_s32(v_sub(gg[3], bb[3])),
v_select(p[3], v_reinterpret_as_s32(v_add(v_sub(bb[3], rr[3]), v_mul(v_setall_u32(2), vdd[3]))),
v_reinterpret_as_s32(v_add(v_sub(rr[3], gg[3]), v_mul(v_setall_u32(4), vdd[3]))))));
@@ -433,16 +433,16 @@ void run_rgb2hsv_impl(uchar out[], const uchar in[], const int sdiv_table[],
// check for negative H
v_int32x4 v_h_less_0[4];
v_h_less_0[0] = (hh[0] < v_setall_s32(0));
v_h_less_0[1] = (hh[1] < v_setall_s32(0));
v_h_less_0[2] = (hh[2] < v_setall_s32(0));
v_h_less_0[3] = (hh[3] < v_setall_s32(0));
v_h_less_0[0] = (v_lt(hh[0], v_setall_s32(0)));
v_h_less_0[1] = (v_lt(hh[1], v_setall_s32(0)));
v_h_less_0[2] = (v_lt(hh[2], v_setall_s32(0)));
v_h_less_0[3] = (v_lt(hh[3], v_setall_s32(0)));
v_int32x4 v_h_180[4];
v_h_180[0] = hh[0] + v_setall_s32(180);
v_h_180[1] = hh[1] + v_setall_s32(180);
v_h_180[2] = hh[2] + v_setall_s32(180);
v_h_180[3] = hh[3] + v_setall_s32(180);
v_h_180[0] = v_add(hh[0], v_setall_s32(180));
v_h_180[1] = v_add(hh[1], v_setall_s32(180));
v_h_180[2] = v_add(hh[2], v_setall_s32(180));
v_h_180[3] = v_add(hh[3], v_setall_s32(180));
hh[0] = v_select(v_h_less_0[0], v_h_180[0], hh[0]);
hh[1] = v_select(v_h_less_0[1], v_h_180[1], hh[1]);
@@ -64,7 +64,7 @@ CV_ALWAYS_INLINE void calcRowLinear32FC1Impl(float *dst[],
bool xRatioEq1 = inSz.width == outSz.width;
bool yRatioEq1 = inSz.height == outSz.height;
constexpr int nlanes = v_float32x8::nlanes;
const int nlanes = VTraits<v_float32x8>::vlanes();
if (!xRatioEq1 && !yRatioEq1)
{
@@ -142,6 +142,11 @@ namespace
std::unordered_set<CycleCausers, CycleHasher> cycle_causers;
};
#if defined(__GNUC__) && (__GNUC__ == 13)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdangling-reference"
#endif
bool canMerge(const GIslandModel::Graph &g,
const ade::NodeHandle &a_nh,
const ade::NodeHandle &slot_nh,
@@ -191,6 +196,10 @@ namespace
return true;
}
#if defined(__GNUC__) && (__GNUC__ == 13)
#pragma GCC diagnostic pop
#endif
inline bool isProducedBy(const ade::NodeHandle &slot,
const ade::NodeHandle &island)
{
@@ -18,6 +18,14 @@ namespace cv {
namespace gapi {
namespace wip {
namespace onevpl {
// With gcc13, std::unique_ptr(FILE, decltype(&fclose)> causes ignored-attributes warning.
// See https://stackoverflow.com/questions/76849365/can-we-add-attributes-to-standard-function-declarations-without-breaking-standar
#if defined(__GNUC__) && (__GNUC__ == 13)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wignored-attributes"
#endif
struct GAPI_EXPORTS FileDataProvider : public IDataProvider {
using file_ptr = std::unique_ptr<FILE, decltype(&fclose)>;
@@ -34,6 +42,11 @@ private:
mfx_codec_id_type codec;
const uint32_t bitstream_data_size;
};
#if defined(__GNUC__) && (__GNUC__ == 13)
#pragma GCC diagnostic pop
#endif
} // namespace onevpl
} // namespace wip
} // namespace gapi
@@ -337,8 +337,9 @@ See cv::imreadmulti for the list of supported formats and flags description.
@param buf Input array or vector of bytes.
@param flags The same flags as in cv::imread, see cv::ImreadModes.
@param mats A vector of Mat objects holding each page, if more than one.
@param range A continuous selection of pages.
*/
CV_EXPORTS_W bool imdecodemulti(InputArray buf, int flags, CV_OUT std::vector<Mat>& mats);
CV_EXPORTS_W bool imdecodemulti(InputArray buf, int flags, CV_OUT std::vector<Mat>& mats, const cv::Range& range = Range::all());
/** @brief Encodes an image into a memory buffer.
+15 -12
View File
@@ -8,6 +8,7 @@
#include <avif/avif.h>
#include <fstream>
#include <memory>
#include <opencv2/core/utils/configuration.private.hpp>
#include "opencv2/imgproc.hpp"
@@ -147,18 +148,6 @@ AvifDecoder::~AvifDecoder() {
size_t AvifDecoder::signatureLength() const { return kAvifSignatureSize; }
bool AvifDecoder::checkSignature(const String &signature) const {
avifDecoder *decoder = avifDecoderCreate();
if (!decoder) return false;
avifDecoderSetIOMemory(decoder,
reinterpret_cast<const uint8_t *>(signature.c_str()),
signature.size());
decoder->io->sizeHint = 1e9;
const avifResult status = avifDecoderParse(decoder);
avifDecoderDestroy(decoder);
return (status == AVIF_RESULT_OK || status == AVIF_RESULT_TRUNCATED_DATA);
}
#define OPENCV_AVIF_CHECK_STATUS(X, ENCDEC) \
{ \
const avifResult status = (X); \
@@ -170,6 +159,20 @@ bool AvifDecoder::checkSignature(const String &signature) const {
} \
}
bool AvifDecoder::checkSignature(const String &signature) const {
std::unique_ptr<avifDecoder, decltype(&avifDecoderDestroy)> decoder(
avifDecoderCreate(), avifDecoderDestroy);
if (!decoder) return false;
OPENCV_AVIF_CHECK_STATUS(
avifDecoderSetIOMemory(
decoder.get(), reinterpret_cast<const uint8_t *>(signature.c_str()),
signature.size()),
decoder);
decoder->io->sizeHint = 1e9;
const avifResult status = avifDecoderParse(decoder.get());
return (status == AVIF_RESULT_OK || status == AVIF_RESULT_TRUNCATED_DATA);
}
ImageDecoder AvifDecoder::newDecoder() const { return makePtr<AvifDecoder>(); }
bool AvifDecoder::readHeader() {
+11 -2
View File
@@ -1077,12 +1077,21 @@ imdecodemulti_(const Mat& buf, int flags, std::vector<Mat>& mats, int start, int
return !mats.empty();
}
bool imdecodemulti(InputArray _buf, int flags, CV_OUT std::vector<Mat>& mats)
bool imdecodemulti(InputArray _buf, int flags, CV_OUT std::vector<Mat>& mats, const Range& range)
{
CV_TRACE_FUNCTION();
Mat buf = _buf.getMat();
return imdecodemulti_(buf, flags, mats, 0, -1);
if (range == Range::all())
{
return imdecodemulti_(buf, flags, mats, 0, -1);
}
else
{
CV_CheckGE(range.start, 0, "Range start cannot be negative.");
CV_CheckGT(range.size(), 0, "Range cannot be empty.");
return imdecodemulti_(buf, flags, mats, range.start, range.size());
}
}
bool imencode( const String& ext, InputArray _image,
+61 -6
View File
@@ -965,7 +965,7 @@ TEST_P(Imgcodecs_Tiff_Modes, decode_multipage)
}
}
TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer)
TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer_all_pages)
{
const int mode = GetParam();
const string root = cvtest::TS::ptr()->get_data_path();
@@ -984,13 +984,14 @@ TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer)
FILE* fp = fopen(filename.c_str(), "rb");
ASSERT_TRUE(fp != NULL);
fseek(fp, 0, SEEK_END);
long pos = ftell(fp);
std::vector<uchar> buf;
buf.resize((size_t)pos);
const size_t file_size = ftell(fp);
fseek(fp, 0, SEEK_SET);
buf.resize(fread(&buf[0], 1, buf.size(), fp));
std::vector<uchar> buf(file_size);
const size_t actual_read = fread(&buf[0], 1, file_size, fp);
fclose(fp);
ASSERT_EQ(file_size, actual_read);
ASSERT_EQ(file_size, static_cast<size_t>(buf.size()));
bool res = imdecodemulti(buf, mode, pages);
ASSERT_TRUE(res == true);
@@ -1002,6 +1003,60 @@ TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer)
}
}
TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer_selected_pages)
{
const int mode = GetParam();
const string root = cvtest::TS::ptr()->get_data_path();
const string filename = root + "readwrite/multipage.tif";
const string page_files[] = {
"readwrite/multipage_p1.tif",
"readwrite/multipage_p2.tif",
"readwrite/multipage_p3.tif",
"readwrite/multipage_p4.tif",
"readwrite/multipage_p5.tif",
"readwrite/multipage_p6.tif"
};
const size_t page_count = sizeof(page_files) / sizeof(page_files[0]);
FILE* fp = fopen(filename.c_str(), "rb");
ASSERT_TRUE(fp != NULL);
fseek(fp, 0, SEEK_END);
const size_t file_size = ftell(fp);
fseek(fp, 0, SEEK_SET);
std::vector<uchar> buf(file_size);
const size_t actual_read = fread(&buf[0], 1, file_size, fp);
fclose(fp);
ASSERT_EQ(file_size, actual_read);
ASSERT_EQ(file_size, static_cast<size_t>(buf.size()));
const Range range(1, page_count - 1);
ASSERT_GE(range.size(), 1);
vector<Mat> middle_pages_from_imread;
for (int page_i = range.start; page_i < range.end; page_i++)
{
const Mat page = imread(root + page_files[page_i], mode);
middle_pages_from_imread.push_back(page);
}
ASSERT_EQ(
static_cast<size_t>(range.size()),
static_cast<size_t>(middle_pages_from_imread.size())
);
vector<Mat> middle_pages_from_imdecodemulti;
const bool res = imdecodemulti(buf, mode, middle_pages_from_imdecodemulti, range);
ASSERT_TRUE(res == true);
EXPECT_EQ(middle_pages_from_imread.size(), middle_pages_from_imdecodemulti.size());
for (int i = 0, e = range.size(); i < e; i++)
{
EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0),
middle_pages_from_imread[i],
middle_pages_from_imdecodemulti[i]);
}
}
const int all_modes[] =
{
IMREAD_UNCHANGED,
+3 -3
View File
@@ -202,9 +202,9 @@ typedef TestBaseWithParam<tuple<MatType, Size, double> > MatInfo_Size_Scale_Area
PERF_TEST_P(MatInfo_Size_Scale_Area, ResizeArea,
testing::Combine(
testing::Values(CV_8UC1, CV_8UC4),
testing::Values(szVGA, szqHD, sz720p),
testing::Values(2.4, 3.4, 1.3)
testing::Values(CV_8UC1, CV_8UC3, CV_8UC4),
testing::Values(szVGA, szqHD, sz720p, sz1080p, sz2160p),
testing::Values(0.1, 0.25, 0.81)
)
)
{
+17 -17
View File
@@ -140,9 +140,9 @@ public:
#if CV_SIMD128
v_uint32x4 rval = v_setall_u32(sptr[j]);
v_uint32x4 val(ksptr0[j], ksptr1[j], ksptr2[j], ksptr3[j]);
v_float32x4 w = kweight4 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)));
v_float32x4 w = v_mul(kweight4, v_lut(this->color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))));
wsum[j] += v_reduce_sum(w);
sum[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(val)) * w);
sum[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(val)), w));
#else
int rval = sptr[j];
@@ -407,11 +407,11 @@ public:
v_uint32x4 b(ksptr0[0], ksptr1[0], ksptr2[0], ksptr3[0]);
v_uint32x4 g(ksptr0[1], ksptr1[1], ksptr2[1], ksptr3[1]);
v_uint32x4 r(ksptr0[2], ksptr1[2], ksptr2[2], ksptr3[2]);
v_float32x4 w = kweight4 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(b, rb) + v_absdiff(g, rg) + v_absdiff(r, rr)));
v_float32x4 w = v_mul(kweight4, v_lut(this->color_weight, v_reinterpret_as_s32(v_add(v_add(v_absdiff(b, rb), v_absdiff(g, rg)), v_absdiff(r, rr)))));
wsum[j] += v_reduce_sum(w);
sum_b[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(b)) * w);
sum_g[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(g)) * w);
sum_r[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(r)) * w);
sum_b[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(b)), w));
sum_g[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(g)), w));
sum_r[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(r)), w));
#else
int rb = rsptr[0], rg = rsptr[1], rr = rsptr[2];
@@ -661,12 +661,12 @@ public:
v_float32x4 rval = v_setall_f32(sptr[j]);
v_float32x4 val(ksptr0[j], ksptr1[j], ksptr2[j], ksptr3[j]);
v_float32x4 knan = v_not_nan(val);
v_float32x4 alpha = (v_absdiff(val, rval) * sindex4) & v_not_nan(rval) & knan;
v_float32x4 alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex4), v_not_nan(rval)), knan);
v_int32x4 idx = v_trunc(alpha);
alpha -= v_cvt_f32(idx);
v_float32x4 w = (kweight4 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one4 - alpha))) & knan;
alpha = v_sub(alpha, v_cvt_f32(idx));
v_float32x4 w = v_and(v_mul(kweight4, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one4, alpha)))), knan);
wsum[j] += v_reduce_sum(w);
sum[j] += v_reduce_sum((val & knan) * w);
sum[j] += v_reduce_sum(v_mul(v_and(val, knan), w));
#else
float rval = sptr[j];
@@ -862,15 +862,15 @@ public:
v_float32x4 kb(ksptr0[0], ksptr1[0], ksptr2[0], ksptr3[0]);
v_float32x4 kg(ksptr0[1], ksptr1[1], ksptr2[1], ksptr3[1]);
v_float32x4 kr(ksptr0[2], ksptr1[2], ksptr2[2], ksptr3[2]);
v_float32x4 knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr);
v_float32x4 alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex4) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan;
v_float32x4 knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr));
v_float32x4 alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex4), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan);
v_int32x4 idx = v_trunc(alpha);
alpha -= v_cvt_f32(idx);
v_float32x4 w = (kweight4 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one4 - alpha))) & knan;
alpha = v_sub(alpha, v_cvt_f32(idx));
v_float32x4 w = v_and(v_mul(kweight4, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one4, alpha)))), knan);
wsum[j] += v_reduce_sum(w);
sum_b[j] += v_reduce_sum((kb & knan) * w);
sum_g[j] += v_reduce_sum((kg & knan) * w);
sum_r[j] += v_reduce_sum((kr & knan) * w);
sum_b[j] += v_reduce_sum(v_mul(v_and(kb, knan), w));
sum_g[j] += v_reduce_sum(v_mul(v_and(kg, knan), w));
sum_r[j] += v_reduce_sum(v_mul(v_and(kr, knan), w));
#else
float rb = rsptr[0], rg = rsptr[1], rr = rsptr[2];
bool r_NAN = cvIsNaN(rb) || cvIsNaN(rg) || cvIsNaN(rr);
+32 -32
View File
@@ -315,7 +315,7 @@ struct ColumnSum<int, uchar> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes)
for (; i <= width - VTraits<v_int32x4>::vlanes(); i += VTraits<v_int32x4>::vlanes())
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -357,10 +357,10 @@ struct ColumnSum<int, uchar> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes )
for( ; i <= width-VTraits<v_uint16x8>::vlanes(); i+=VTraits<v_uint16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale)));
v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale)));
@@ -369,7 +369,7 @@ struct ColumnSum<int, uchar> :
v_pack_store(D + i, v_dst);
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -396,16 +396,16 @@ struct ColumnSum<int, uchar> :
v_store(SUM + i + VTraits<v_int32>::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits<v_int32>::vlanes())));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes )
for( ; i <= width-VTraits<v_uint16x8>::vlanes(); i+=VTraits<v_uint16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_uint16x8 v_dst = v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01));
v_pack_store(D + i, v_dst);
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -486,7 +486,7 @@ public BaseColumnFilter
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width - v_uint16x8::nlanes; i += v_uint16x8::nlanes )
for( ; i <= width - VTraits<v_uint16x8>::vlanes(); i += VTraits<v_uint16x8>::vlanes() )
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -546,13 +546,13 @@ public BaseColumnFilter
v_uint32x4 ds4 = v_setall_u32((unsigned)ds);
v_uint16x8 dd8 = v_setall_u16((ushort)dd);
for( ; i <= width-v_uint8x16::nlanes; i+=v_uint8x16::nlanes )
for( ; i <= width-VTraits<v_uint8x16>::vlanes(); i+=VTraits<v_uint8x16>::vlanes() )
{
v_uint16x8 _sm0 = v_load(Sm + i);
v_uint16x8 _sm1 = v_load(Sm + i + v_uint16x8::nlanes);
v_uint16x8 _sm1 = v_load(Sm + i + VTraits<v_uint16x8>::vlanes());
v_uint16x8 _s0 = v_add_wrap(v_load(SUM + i), v_load(Sp + i));
v_uint16x8 _s1 = v_add_wrap(v_load(SUM + i + v_uint16x8::nlanes), v_load(Sp + i + v_uint16x8::nlanes));
v_uint16x8 _s1 = v_add_wrap(v_load(SUM + i + VTraits<v_uint16x8>::vlanes()), v_load(Sp + i + VTraits<v_uint16x8>::vlanes()));
v_uint32x4 _s00, _s01, _s10, _s11;
@@ -572,7 +572,7 @@ public BaseColumnFilter
v_store(D + i, v_pack_u(r0, r1));
v_store(SUM + i, _s0);
v_store(SUM + i + v_uint16x8::nlanes, _s1);
v_store(SUM + i + VTraits<v_uint16x8>::vlanes(), _s1);
}
#endif
#endif
@@ -649,7 +649,7 @@ struct ColumnSum<int, short> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width - VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -689,17 +689,17 @@ struct ColumnSum<int, short> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes )
for( ; i <= width-VTraits<v_int16x8>::vlanes(); i+=VTraits<v_int16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale));
v_int32x4 v_s01d = v_round(v_mul(v_cvt_f32(v_s01), v_scale));
v_store(D + i, v_pack(v_s0d, v_s01d));
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -725,15 +725,15 @@ struct ColumnSum<int, short> :
v_store(SUM + i + VTraits<v_int32>::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits<v_int32>::vlanes())));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes )
for( ; i <= width-VTraits<v_int16x8>::vlanes(); i+=VTraits<v_int16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_store(D + i, v_pack(v_s0, v_s01));
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -798,7 +798,7 @@ struct ColumnSum<int, ushort> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes)
for (; i <= width - VTraits<v_int32x4>::vlanes(); i += VTraits<v_int32x4>::vlanes())
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -838,17 +838,17 @@ struct ColumnSum<int, ushort> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes )
for( ; i <= width-VTraits<v_uint16x8>::vlanes(); i+=VTraits<v_uint16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale)));
v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale)));
v_store(D + i, v_pack(v_s0d, v_s01d));
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -874,15 +874,15 @@ struct ColumnSum<int, ushort> :
v_store(SUM + i + VTraits<v_int32>::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits<v_int32>::vlanes())));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes )
for( ; i <= width-VTraits<v_uint16x8>::vlanes(); i+=VTraits<v_uint16x8>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes));
v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits<v_int32x4>::vlanes()), v_load(Sp + i + VTraits<v_int32x4>::vlanes()));
v_store(D + i, v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01)));
v_store(SUM + i, v_sub(v_s0, v_load(Sm + i)));
v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes)));
v_store(SUM + i + VTraits<v_int32x4>::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits<v_int32x4>::vlanes())));
}
#endif
#endif
@@ -945,7 +945,7 @@ struct ColumnSum<int, int> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width - VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -981,7 +981,7 @@ struct ColumnSum<int, int> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width-VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale));
@@ -1010,7 +1010,7 @@ struct ColumnSum<int, int> :
v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width-VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
@@ -1079,7 +1079,7 @@ struct ColumnSum<int, float> :
v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width - VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i)));
}
@@ -1115,7 +1115,7 @@ struct ColumnSum<int, float> :
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
v_float32x4 v_scale = v_setall_f32((float)_scale);
for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes)
for (; i <= width - VTraits<v_int32x4>::vlanes(); i += VTraits<v_int32x4>::vlanes())
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_store(D + i, v_mul(v_cvt_f32(v_s0), v_scale));
@@ -1142,7 +1142,7 @@ struct ColumnSum<int, float> :
v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i)));
}
#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16
for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes )
for( ; i <= width-VTraits<v_int32x4>::vlanes(); i+=VTraits<v_int32x4>::vlanes() )
{
v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i));
v_store(D + i, v_cvt_f32(v_s0));
+151 -148
View File
@@ -66,7 +66,7 @@ template<typename _Tp> static inline cv::v_float32 splineInterpolate(const cv::v
ix = v_shl<2>(ix);
v_float32 t0, t1, t2, t3;
// assume that v_float32::nlanes == v_int32::nlanes
// assume that VTraits<v_float32>::vlanes() == VTraits<v_int32>::vlanes()
if(VTraits<v_float32>::vlanes() == 4)
{
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) idx[4];
@@ -1388,16 +1388,16 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin
v_uint16x8& outA, v_uint16x8& outB, v_uint16x8& outC)
{
//LUT idx of origin pt of cube
v_uint16x8 idxsX = inX >> (lab_base_shift - lab_lut_shift);
v_uint16x8 idxsY = inY >> (lab_base_shift - lab_lut_shift);
v_uint16x8 idxsZ = inZ >> (lab_base_shift - lab_lut_shift);
v_uint16x8 idxsX = v_shr<lab_base_shift - lab_lut_shift>(inX);
v_uint16x8 idxsY = v_shr<lab_base_shift - lab_lut_shift>(inY);
v_uint16x8 idxsZ = v_shr<lab_base_shift - lab_lut_shift>(inZ);
//x, y, z are [0; TRILINEAR_BASE)
const uint16_t bitMask = (1 << trilinear_shift) - 1;
v_uint16x8 bitMaskReg = v_setall_u16(bitMask);
v_uint16x8 fracX = (inX >> (lab_base_shift - 8 - 1)) & bitMaskReg;
v_uint16x8 fracY = (inY >> (lab_base_shift - 8 - 1)) & bitMaskReg;
v_uint16x8 fracZ = (inZ >> (lab_base_shift - 8 - 1)) & bitMaskReg;
v_uint16x8 fracX = v_and(v_shr<lab_base_shift - 8 - 1>(inX), bitMaskReg);
v_uint16x8 fracY = v_and(v_shr<lab_base_shift - 8 - 1>(inY), bitMaskReg);
v_uint16x8 fracZ = v_and(v_shr<lab_base_shift - 8 - 1>(inZ), bitMaskReg);
//load values to interpolate for pix0, pix1, .., pix7
v_int16x8 a0, a1, a2, a3, a4, a5, a6, a7;
@@ -1407,9 +1407,9 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin
v_uint32x4 addrDw0, addrDw1, addrDw10, addrDw11;
v_mul_expand(v_setall_u16(3*8), idxsX, addrDw0, addrDw1);
v_mul_expand(v_setall_u16(3*8*LAB_LUT_DIM), idxsY, addrDw10, addrDw11);
addrDw0 += addrDw10; addrDw1 += addrDw11;
addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11);
v_mul_expand(v_setall_u16(3*8*LAB_LUT_DIM*LAB_LUT_DIM), idxsZ, addrDw10, addrDw11);
addrDw0 += addrDw10; addrDw1 += addrDw11;
addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11);
uint32_t CV_DECL_ALIGNED(16) addrofs[8];
v_store_aligned(addrofs, addrDw0);
@@ -1431,9 +1431,9 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin
v_int16x8 w0, w1, w2, w3, w4, w5, w6, w7;
v_mul_expand(v_setall_u16(8), fracX, addrDw0, addrDw1);
v_mul_expand(v_setall_u16(8*TRILINEAR_BASE), fracY, addrDw10, addrDw11);
addrDw0 += addrDw10; addrDw1 += addrDw11;
addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11);
v_mul_expand(v_setall_u16(8*TRILINEAR_BASE*TRILINEAR_BASE), fracZ, addrDw10, addrDw11);
addrDw0 += addrDw10; addrDw1 += addrDw11;
addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11);
v_store_aligned(addrofs, addrDw0);
v_store_aligned(addrofs + 4, addrDw1);
@@ -1476,7 +1476,8 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1
const int16_t* LUT,
v_uint16& outA, v_uint16& outB, v_uint16& outC)
{
const int vsize = VTraits<v_uint16>::max_nlanes;
const int vsize = VTraits<v_uint16>::vlanes();
const int vsize_max = VTraits<v_uint16>::max_nlanes;
// LUT idx of origin pt of cube
v_uint16 tx = v_shr<lab_base_shift - lab_lut_shift>(inX);
@@ -1492,7 +1493,7 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1
baseIdx0 = v_add(v_add(btmp00, btmp10), btmp20);
baseIdx1 = v_add(v_add(btmp01, btmp11), btmp21);
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vbaseIdx[vsize];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vbaseIdx[vsize_max];
v_store_aligned(vbaseIdx + 0*vsize/2, baseIdx0);
v_store_aligned(vbaseIdx + 1*vsize/2, baseIdx1);
@@ -1513,13 +1514,13 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1
trilinearIdx0 = v_add(v_add(v_shl<3>(fracX0), v_shl<3 + trilinear_shift>(fracY0)), v_shl<3 + trilinear_shift * 2>(fracZ0));
trilinearIdx1 = v_add(v_add(v_shl<3>(fracX1), v_shl<3 + trilinear_shift>(fracY1)), v_shl<3 + trilinear_shift * 2>(fracZ1));
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vtrilinearIdx[vsize];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vtrilinearIdx[vsize_max];
v_store_aligned(vtrilinearIdx + 0*vsize/2, trilinearIdx0);
v_store_aligned(vtrilinearIdx + 1*vsize/2, trilinearIdx1);
v_uint32 a0, a1, b0, b1, c0, c1;
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) va[vsize], vb[vsize], vc[vsize];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) va[vsize_max], vb[vsize_max], vc[vsize_max];
for(int j = 0; j < vsize; j++)
{
const int16_t* baseLUT = LUT + vbaseIdx[j];
@@ -1649,11 +1650,11 @@ struct RGB2Lab_b
vL = v_shr<lab_shift2>(vL);
/* int a = CV_DESCALE( 500*(fX - fY) + 128*(1 << lab_shift2), lab_shift2 );*/
va = v_fma(vfX - vfY, v_setall_s32(500), v_setall_s32(abShift+labDescaleShift));
va = v_fma(v_sub(vfX, vfY), v_setall_s32(500), v_setall_s32(abShift+labDescaleShift));
va = v_shr<lab_shift2>(va);
/* int b = CV_DESCALE( 200*(fY - fZ) + 128*(1 << lab_shift2), lab_shift2 );*/
vb = v_fma(vfY - vfZ, v_setall_s32(200), v_setall_s32(abShift+labDescaleShift));
vb = v_fma(v_sub(vfY, vfZ), v_setall_s32(200), v_setall_s32(abShift+labDescaleShift));
vb = v_shr<lab_shift2>(vb);
}
#endif // CV_NEON
@@ -1675,8 +1676,8 @@ struct RGB2Lab_b
#if CV_NEON
// On each loop, we load nlanes of RGB/A v_uint8s and store nlanes of
// Lab v_uint8s
for(; i <= n - v_uint8::nlanes; i += v_uint8::nlanes,
src += scn*v_uint8::nlanes, dst += 3*v_uint8::nlanes )
for(; i <= n - VTraits<v_uint8>::vlanes(); i += VTraits<v_uint8>::vlanes(),
src += scn*VTraits<v_uint8>::vlanes(), dst += 3*VTraits<v_uint8>::vlanes() )
{
// Load 4 batches of 4 src
v_uint8 vRi, vGi, vBi;
@@ -1712,7 +1713,7 @@ struct RGB2Lab_b
#endif // CV_NEON
#if CV_SIMD
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
const int xyzDescaleShift = 1 << (lab_shift - 1);
v_int16 vXYZdescale = vx_setall_s16(xyzDescaleShift);
v_int16 cxrg, cxb1, cyrg, cyb1, czrg, czb1;
@@ -1752,7 +1753,7 @@ struct RGB2Lab_b
v_expand(drgb[k], qrgb[k*2+0], qrgb[k*2+1]);
}
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vdrgb[vsize*3];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vdrgb[VTraits<v_uint8>::max_nlanes*3];
for(int k = 0; k < 12; k++)
{
v_store_aligned(vdrgb + k*vsize/4, qrgb[k]);
@@ -1784,14 +1785,14 @@ struct RGB2Lab_b
v_uint32 x[4], y[4], z[4];
for(int j = 0; j < 4; j++)
{
x[j] = v_reinterpret_as_u32(v_dotprod(rg[j], cxrg) + v_dotprod(bd[j], cxb1)) >> lab_shift;
y[j] = v_reinterpret_as_u32(v_dotprod(rg[j], cyrg) + v_dotprod(bd[j], cyb1)) >> lab_shift;
z[j] = v_reinterpret_as_u32(v_dotprod(rg[j], czrg) + v_dotprod(bd[j], czb1)) >> lab_shift;
x[j] = v_shr<xyz_shift>(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], cxrg), v_dotprod(bd[j], cxb1))));
y[j] = v_shr<xyz_shift>(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], cyrg), v_dotprod(bd[j], cyb1))));
z[j] = v_shr<xyz_shift>(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], czrg), v_dotprod(bd[j], czb1))));
}
// [fX, fY, fZ] = LabCbrtTab_b[vx, vy, vz]
// [4 per X, 4 per Y, 4 per Z]
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vxyz[vsize*3];
uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vxyz[VTraits<v_uint8>::max_nlanes*3];
for(int j = 0; j < 4; j++)
{
v_store_aligned(vxyz + (0*4+j)*vsize/4, x[j]);
@@ -1822,7 +1823,7 @@ struct RGB2Lab_b
v_uint32 vLshift = vx_setall_u32((uint32_t)(Lshift + labDescaleShift));
for(int k = 0; k < 4; k++)
{
vL[k] = (vL[k] + vLshift) >> lab_shift2;
vL[k] = v_shr<lab_shift2>(v_add(vL[k], vLshift));
}
v_uint16 L0, L1;
L0 = v_pack(vL[0], vL[1]);
@@ -1846,7 +1847,7 @@ struct RGB2Lab_b
v_int32 abShift = vx_setall_s32(128*(1 << lab_shift2) + labDescaleShift);
for(int k = 0; k < 8; k++)
{
ab[k] = (ab[k] + abShift) >> lab_shift2;
ab[k] = v_shr<lab_shift2>(v_add(ab[k], abShift));
}
v_int16 a0, a1, b0, b1;
a0 = v_pack(ab[0], ab[1]); a1 = v_pack(ab[2], ab[3]);
@@ -1941,7 +1942,7 @@ struct RGB2Lab_f
#if CV_SIMD
if(enablePackedLab)
{
const int vsize = v_float32::nlanes;
const int vsize = VTraits<v_float32>::vlanes();
static const int nPixels = vsize*2;
for(; i < n - 3*nPixels; i += 3*nPixels, src += scn*nPixels)
{
@@ -1973,8 +1974,8 @@ struct RGB2Lab_f
#undef clipv
/* int iR = R*LAB_BASE, iG = G*LAB_BASE, iB = B*LAB_BASE, iL, ia, ib; */
v_float32 basef = vx_setall_f32(LAB_BASE);
rvec0 *= basef, gvec0 *= basef, bvec0 *= basef;
rvec1 *= basef, gvec1 *= basef, bvec1 *= basef;
rvec0 = v_mul(rvec0, basef), gvec0 = v_mul(gvec0, basef), bvec0 = v_mul(bvec0, basef);
rvec1 = v_mul(rvec1, basef), gvec1 = v_mul(gvec1, basef), bvec1 = v_mul(bvec1, basef);
v_int32 irvec0, igvec0, ibvec0, irvec1, igvec1, ibvec1;
irvec0 = v_round(rvec0); irvec1 = v_round(rvec1);
@@ -2004,8 +2005,8 @@ struct RGB2Lab_f
/* dst[i] = L*100.0f */
v_float32 v100dBase = vx_setall_f32(100.0f/LAB_BASE);
l_vec0 = l_vec0*v100dBase;
l_vec1 = l_vec1*v100dBase;
l_vec0 = v_mul(l_vec0, v100dBase);
l_vec1 = v_mul(l_vec1, v100dBase);
/*
dst[i + 1] = a*256.0f - 128.0f;
dst[i + 2] = b*256.0f - 128.0f;
@@ -2043,8 +2044,8 @@ struct RGB2Lab_f
static const float _a = (softfloat(16) / softfloat(116));
int i = 0;
#if CV_SIMD
const int vsize = v_float32::nlanes;
const int nrepeats = vsize == 4 ? 2 : 1;
const int vsize = VTraits<v_float32>::vlanes();
const int nrepeats = VTraits<v_float32>::nlanes == 4 ? 2 : 1;
v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2);
v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4), vc5 = vx_setall_f32(C5);
v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8);
@@ -2080,9 +2081,9 @@ struct RGB2Lab_f
v_float32 vgscale = vx_setall_f32(gscale);
for (int k = 0; k < nrepeats; k++)
{
R[k] = splineInterpolate(R[k]*vgscale, gammaTab, GAMMA_TAB_SIZE);
G[k] = splineInterpolate(G[k]*vgscale, gammaTab, GAMMA_TAB_SIZE);
B[k] = splineInterpolate(B[k]*vgscale, gammaTab, GAMMA_TAB_SIZE);
R[k] = splineInterpolate(v_mul(R[k], vgscale), gammaTab, GAMMA_TAB_SIZE);
G[k] = splineInterpolate(v_mul(G[k], vgscale), gammaTab, GAMMA_TAB_SIZE);
B[k] = splineInterpolate(v_mul(B[k], vgscale), gammaTab, GAMMA_TAB_SIZE);
}
}
@@ -2090,26 +2091,26 @@ struct RGB2Lab_f
v_float32 FX[nrepeats], FY[nrepeats], FZ[nrepeats];
for (int k = 0; k < nrepeats; k++)
{
X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, B[k]*vc2));
Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, B[k]*vc5));
Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, B[k]*vc8));
X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, v_mul(B[k], vc2)));
Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, v_mul(B[k], vc5)));
Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, v_mul(B[k], vc8)));
// use spline interpolation instead of direct calculation
v_float32 vTabScale = vx_setall_f32(LabCbrtTabScale);
FX[k] = splineInterpolate(X[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE);
FY[k] = splineInterpolate(Y[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE);
FZ[k] = splineInterpolate(Z[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE);
FX[k] = splineInterpolate(v_mul(X[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE);
FY[k] = splineInterpolate(v_mul(Y[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE);
FZ[k] = splineInterpolate(v_mul(Z[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE);
}
v_float32 L[nrepeats], a[nrepeats], b[nrepeats];
for (int k = 0; k < nrepeats; k++)
{
// 7.787f = (29/3)^3/(29*4), 0.008856f = (6/29)^3, 903.3 = (29/3)^3
v_float32 mask = Y[k] > (vx_setall_f32(0.008856f));
v_float32 mask = v_gt(Y[k], (vx_setall_f32(0.008856f)));
v_float32 v116 = vx_setall_f32(116.f), vm16 = vx_setall_f32(-16.f);
L[k] = v_select(mask, v_fma(v116, FY[k], vm16), vx_setall_f32(903.3f)*Y[k]);
a[k] = vx_setall_f32(500.f) * (FX[k] - FY[k]);
b[k] = vx_setall_f32(200.f) * (FY[k] - FZ[k]);
L[k] = v_select(mask, v_fma(v116, FY[k], vm16), v_mul(vx_setall_f32(903.3f),Y[k]));
a[k] = v_mul(vx_setall_f32(500.F), v_sub(FX[k], FY[k]));
b[k] = v_mul(vx_setall_f32(200.F), v_sub(FY[k], FZ[k]));
v_store_interleave(dst + k*3*vsize, L[k], a[k], b[k]);
}
@@ -2204,7 +2205,7 @@ struct Lab2RGBfloat
float alpha = ColorChannel<float>::max();
#if CV_SIMD
const int vsize = v_float32::nlanes;
const int vsize = VTraits<v_float32>::vlanes();
const int nrepeats = 2;
v_float32 v16_116 = vx_setall_f32(16.0f / 116.0f);
for( ; i <= n-vsize*nrepeats;
@@ -2221,14 +2222,14 @@ struct Lab2RGBfloat
v_float32 vlThresh = vx_setall_f32(lThresh);
for(int k = 0; k < nrepeats; k++)
{
limask[k] = li[k] <= vlThresh;
limask[k] = v_le(li[k], vlThresh);
}
v_float32 ylo[nrepeats], yhi[nrepeats], fylo[nrepeats], fyhi[nrepeats];
// 903.3 = (29/3)^3, 7.787 = (29/3)^3/(29*4)
v_float32 vinv903 = vx_setall_f32(1.f/903.3f);
for(int k = 0; k < nrepeats; k++)
{
ylo[k] = li[k] * vinv903;
ylo[k] = v_mul(li[k], vinv903);
}
v_float32 v7787 = vx_setall_f32(7.787f);
for(int k = 0; k < nrepeats; k++)
@@ -2238,11 +2239,11 @@ struct Lab2RGBfloat
v_float32 v16 = vx_setall_f32(16.0f), vinv116 = vx_setall_f32(1.f/116.0f);
for(int k = 0; k < nrepeats; k++)
{
fyhi[k] = (li[k] + v16) * vinv116;
fyhi[k] = v_mul(v_add(li[k], v16), vinv116);
}
for(int k = 0; k < nrepeats; k++)
{
yhi[k] = fyhi[k] * fyhi[k] * fyhi[k];
yhi[k] = v_mul(fyhi[k], fyhi[k], fyhi[k]);
}
for(int k = 0; k < nrepeats; k++)
{
@@ -2265,9 +2266,9 @@ struct Lab2RGBfloat
for (int j = 0; j < 2; j++)
{
v_float32 f = fxz[k*2+j];
v_float32 fmask = f <= vfTresh;
v_float32 flo = (f - v16_116) * vinv7787;
v_float32 fhi = f*f*f;
v_float32 fmask = v_le(f, vfTresh);
v_float32 flo = v_mul(v_sub(f, v16_116), vinv7787);
v_float32 fhi = v_mul(v_mul(f, f), f);
fxz[k*2+j] = v_select(fmask, flo, fhi);
}
}
@@ -2281,9 +2282,9 @@ struct Lab2RGBfloat
v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8);
for(int k = 0; k < nrepeats; k++)
{
ro[k] = v_fma(vc0, x[k], v_fma(vc1, y[k], vc2 * z[k]));
go[k] = v_fma(vc3, x[k], v_fma(vc4, y[k], vc5 * z[k]));
bo[k] = v_fma(vc6, x[k], v_fma(vc7, y[k], vc8 * z[k]));
ro[k] = v_fma(vc0, x[k], v_fma(vc1, y[k], v_mul(vc2, z[k])));
go[k] = v_fma(vc3, x[k], v_fma(vc4, y[k], v_mul(vc5, z[k])));
bo[k] = v_fma(vc6, x[k], v_fma(vc7, y[k], v_mul(vc8, z[k])));
}
v_float32 one = vx_setall_f32(1.f), zero = vx_setzero_f32();
for(int k = 0; k < nrepeats; k++)
@@ -2298,9 +2299,9 @@ struct Lab2RGBfloat
v_float32 vgscale = vx_setall_f32(gscale);
for(int k = 0; k < nrepeats; k++)
{
ro[k] *= vgscale;
go[k] *= vgscale;
bo[k] *= vgscale;
ro[k] = v_mul(ro[k], vgscale);
go[k] = v_mul(go[k], vgscale);
bo[k] = v_mul(bo[k], vgscale);
}
for(int k = 0; k < nrepeats; k++)
@@ -2500,8 +2501,8 @@ struct Lab2RGBinteger
for(int k = 0; k < 4; k++)
{
yf[k] = v_lut((const int*)LabToYF_b, lq[k]);
y[k] = yf[k] & mask16;
ify[k] = v_reinterpret_as_s32(v_reinterpret_as_u32(yf[k]) >> 16);
y[k] = v_and(yf[k], mask16);
ify[k] = v_reinterpret_as_s32(v_shr(v_reinterpret_as_u32(yf[k]), 16));
}
v_int16 ify0, ify1;
@@ -2516,18 +2517,18 @@ struct Lab2RGBinteger
v_uint16 mulA = vx_setall_u16(53687);
v_uint32 ma[4];
v_uint32 addA = vx_setall_u32(1 << 7);
v_mul_expand((a0 + (a0 << 2)), mulA, ma[0], ma[1]);
v_mul_expand((a1 + (a1 << 2)), mulA, ma[2], ma[3]);
adiv0 = v_reinterpret_as_s16(v_pack(((ma[0] + addA) >> 13), ((ma[1] + addA) >> 13)));
adiv1 = v_reinterpret_as_s16(v_pack(((ma[2] + addA) >> 13), ((ma[3] + addA) >> 13)));
v_mul_expand((v_add(a0, v_shl<2>(a0))), mulA, ma[0], ma[1]);
v_mul_expand((v_add(a1, v_shl<2>(a1))), mulA, ma[2], ma[3]);
adiv0 = v_reinterpret_as_s16(v_pack((v_shr<13>(v_add(ma[0], addA))), (v_shr<13>(v_add(ma[1], addA)))));
adiv1 = v_reinterpret_as_s16(v_pack((v_shr<13>(v_add(ma[2], addA))), (v_shr<13>(v_add(ma[3], addA)))));
v_uint16 mulB = vx_setall_u16(41943);
v_uint32 mb[4];
v_uint32 addB = vx_setall_u32(1 << 4);
v_mul_expand(b0, mulB, mb[0], mb[1]);
v_mul_expand(b1, mulB, mb[2], mb[3]);
bdiv0 = v_reinterpret_as_s16(v_pack((mb[0] + addB) >> 9, (mb[1] + addB) >> 9));
bdiv1 = v_reinterpret_as_s16(v_pack((mb[2] + addB) >> 9, (mb[3] + addB) >> 9));
bdiv0 = v_reinterpret_as_s16(v_pack(v_shr<9>(v_add(mb[0], addB)), v_shr<9>(v_add(mb[1], addB))));
bdiv1 = v_reinterpret_as_s16(v_pack(v_shr<9>(v_add(mb[2], addB)), v_shr<9>(v_add(mb[3], addB))));
// 0 <= adiv <= 8356, 0 <= bdiv <= 20890
/* x = ifxz[0]; y = y; z = ifxz[1]; */
@@ -2570,7 +2571,7 @@ struct Lab2RGBinteger
{
bool srgb = issRGB;
ushort* tab = sRGBInvGammaTab_b;
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
v_uint8 valpha = vx_setall_u8(alpha);
v_int32 vc[9];
for(int k = 0; k < 9; k++)
@@ -2592,9 +2593,9 @@ struct Lab2RGBinteger
v_int32 rq[4], gq[4], bq[4];
for(int k = 0; k < 4; k++)
{
rq[k] = (vc[0] * xq[k] + vc[1] * yq[k] + vc[2] * zq[k] + vdescale) >> shift;
gq[k] = (vc[3] * xq[k] + vc[4] * yq[k] + vc[5] * zq[k] + vdescale) >> shift;
bq[k] = (vc[6] * xq[k] + vc[7] * yq[k] + vc[8] * zq[k] + vdescale) >> shift;
rq[k] = v_shr<shift>(v_add(v_add(v_add(v_mul(vc[0], xq[k]), v_mul(vc[1], yq[k])), v_mul(vc[2], zq[k])), vdescale));
gq[k] = v_shr<shift>(v_add(v_add(v_add(v_mul(vc[3], xq[k]), v_mul(vc[4], yq[k])), v_mul(vc[5], zq[k])), vdescale));
bq[k] = v_shr<shift>(v_add(v_add(v_add(v_mul(vc[6], xq[k]), v_mul(vc[7], yq[k])), v_mul(vc[8], zq[k])), vdescale));
}
//limit indices in table and then substitute
@@ -2611,7 +2612,7 @@ struct Lab2RGBinteger
if(srgb)
{
// [RRR... , GGG... , BBB...]
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vidx[vsize*3];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vidx[VTraits<v_uint8>::max_nlanes*3];
for (int k = 0; k < 4; k++)
v_store_aligned(vidx + 0*vsize + k*vsize/4, rq[k]);
for (int k = 0; k < 4; k++)
@@ -2631,9 +2632,9 @@ struct Lab2RGBinteger
// rgb = (rgb*255) >> inv_gamma_shift
for(int k = 0; k < 4; k++)
{
rq[k] = ((rq[k] << 8) - rq[k]) >> inv_gamma_shift;
gq[k] = ((gq[k] << 8) - gq[k]) >> inv_gamma_shift;
bq[k] = ((bq[k] << 8) - bq[k]) >> inv_gamma_shift;
rq[k] = v_shr((v_sub(v_shl(rq[k], 8), rq[k])), inv_gamma_shift);
gq[k] = v_shr((v_sub(v_shl(gq[k], 8), gq[k])), inv_gamma_shift);
bq[k] = v_shr((v_sub(v_shl(bq[k], 8), bq[k])), inv_gamma_shift);
}
rgb[0] = v_reinterpret_as_u16(v_pack(rq[0], rq[1]));
rgb[1] = v_reinterpret_as_u16(v_pack(rq[2], rq[3]));
@@ -2730,13 +2731,13 @@ struct Lab2RGB_b
static const softfloat fl = softfloat(100)/f255;
#if CV_SIMD
const int fsize = v_float32::nlanes;
const int fsize = VTraits<v_float32>::vlanes();
v_float32 vl = vx_setall_f32((float)fl);
v_float32 va = vx_setall_f32(1.f);
v_float32 vb = vx_setall_f32(1.f);
v_float32 vaLow = vx_setall_f32(-128.f), vbLow = vx_setall_f32(-128.f);
//TODO: fix that when v_interleave is available
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3];
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits<v_float32>::max_nlanes*3], interTmpA[VTraits<v_float32>::max_nlanes*3];
v_store_interleave(interTmpM, vl, va, vb);
v_store_interleave(interTmpA, vx_setzero_f32(), vaLow, vbLow);
v_float32 mluv[3], aluv[3];
@@ -2754,7 +2755,7 @@ struct Lab2RGB_b
j = 0;
#if CV_SIMD
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
for( ; j <= (dn - vsize)*3; j += 3*vsize )
{
v_uint8 s0, s1, s2;
@@ -2808,7 +2809,7 @@ struct Lab2RGB_b
v_int32 vi[4*3];
for(int k = 0; k < 4*3; k++)
{
vi[k] = v_round(vf[k]*v255);
vi[k] = v_round(v_mul(vf[k], v255));
}
v_uint8 rgb[3];
@@ -2830,7 +2831,7 @@ struct Lab2RGB_b
for(int k = 0; k < 4; k++)
{
vf[k] = vx_load_aligned(buf + j + k*fsize);
vi[k] = v_round(vf[k]*v255);
vi[k] = v_round(v_mul(vf[k], v255));
}
v_store(dst, v_pack_u(v_pack(vi[0], vi[1]), v_pack(vi[2], vi[3])));
}
@@ -2910,8 +2911,8 @@ struct RGB2Luvfloat
C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8];
#if CV_SIMD
const int vsize = v_float32::nlanes;
const int nrepeats = vsize == 4 ? 2 : 1;
const int vsize = VTraits<v_float32>::vlanes();
const int nrepeats = VTraits<v_float32>::nlanes == 4 ? 2 : 1;
for( ; i <= n-vsize*nrepeats;
i+= vsize*nrepeats, src += scn*vsize*nrepeats, dst += 3*vsize*nrepeats)
{
@@ -2944,9 +2945,9 @@ struct RGB2Luvfloat
v_float32 vgscale = vx_setall_f32(gscale);
for (int k = 0; k < nrepeats; k++)
{
R[k] *= vgscale;
G[k] *= vgscale;
B[k] *= vgscale;
R[k] = v_mul(R[k], vgscale);
G[k] = v_mul(G[k], vgscale);
B[k] = v_mul(B[k], vgscale);
}
for (int k = 0; k < nrepeats; k++)
@@ -2963,27 +2964,27 @@ struct RGB2Luvfloat
v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8);
for (int k = 0; k < nrepeats; k++)
{
X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, B[k]*vc2));
Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, B[k]*vc5));
Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, B[k]*vc8));
X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, v_mul(B[k], vc2)));
Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, v_mul(B[k], vc5)));
Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, v_mul(B[k], vc8)));
}
v_float32 L[nrepeats], u[nrepeats], v[nrepeats];
v_float32 vmun = vx_setall_f32(-un), vmvn = vx_setall_f32(-vn);
for (int k = 0; k < nrepeats; k++)
{
L[k] = splineInterpolate(Y[k]*vx_setall_f32(LabCbrtTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE);
L[k] = splineInterpolate(v_mul(Y[k], vx_setall_f32(LabCbrtTabScale)), LabCbrtTab, LAB_CBRT_TAB_SIZE);
// L = 116.f*L - 16.f;
L[k] = v_fma(L[k], vx_setall_f32(116.f), vx_setall_f32(-16.f));
v_float32 d;
// d = (4*13) / max(X + 15 * Y + 3 * Z, FLT_EPSILON)
d = v_fma(Y[k], vx_setall_f32(15.f), v_fma(Z[k], vx_setall_f32(3.f), X[k]));
d = vx_setall_f32(4.f*13.f) / v_max(d, vx_setall_f32(FLT_EPSILON));
d = v_div(vx_setall_f32(4.F * 13.F), v_max(d, vx_setall_f32(FLT_EPSILON)));
// u = L*(X*d - un)
u[k] = L[k]*v_fma(X[k], d, vmun);
u[k] = v_mul(L[k], v_fma(X[k], d, vmun));
// v = L*((9*0.25f)*Y*d - vn);
v[k] = L[k]*v_fma(vx_setall_f32(9.f*0.25f)*Y[k], d, vmvn);
v[k] = v_mul(L[k], v_fma(v_mul(vx_setall_f32(9.F * 0.25F), Y[k]), d, vmvn));
}
for (int k = 0; k < nrepeats; k++)
@@ -3099,8 +3100,8 @@ struct Luv2RGBfloat
float _un = un, _vn = vn;
#if CV_SIMD
const int vsize = v_float32::nlanes;
const int nrepeats = vsize == 4 ? 2 : 1;
const int vsize = VTraits<v_float32>::vlanes();
const int nrepeats = VTraits<v_float32>::nlanes == 4 ? 2 : 1;
for( ; i <= n - vsize*nrepeats;
i += vsize*nrepeats, src += vsize*3*nrepeats, dst += dcn*vsize*nrepeats)
{
@@ -3120,13 +3121,13 @@ struct Luv2RGBfloat
v_float32 Ylo, Yhi;
// ((L + 16)/116)^3
Ylo = (L[k] + v16) * v116inv;
Ylo = Ylo*Ylo*Ylo;
Ylo = v_mul(v_add(L[k], v16), v116inv);
Ylo = v_mul(v_mul(Ylo, Ylo), Ylo);
// L*(3./29.)^3
Yhi = L[k] * v903inv;
Yhi = v_mul(L[k], v903inv);
// Y = (L <= 8) ? Y0 : Y1;
Y[k] = v_select(L[k] >= vx_setall_f32(8.f), Ylo, Yhi);
Y[k] = v_select(v_ge(L[k], vx_setall_f32(8.f)), Ylo, Yhi);
}
v_float32 v4inv = vx_setall_f32(0.25f), v3 = vx_setall_f32(3.f);
@@ -3135,18 +3136,18 @@ struct Luv2RGBfloat
v_float32 up, vp;
// up = 3*(u + L*_un);
up = v3*(v_fma(L[k], vx_setall_f32(_un), u[k]));
up = v_mul(v3, v_fma(L[k], vx_setall_f32(_un), u[k]));
// vp = 0.25/(v + L*_vn);
vp = v4inv/(v_fma(L[k], vx_setall_f32(_vn), v[k]));
vp = v_div(v4inv, v_fma(L[k], vx_setall_f32(_vn), v[k]));
// vp = max(-0.25, min(0.25, vp));
vp = v_max(vx_setall_f32(-0.25f), v_min(v4inv, vp));
//X = 3*up*vp; // (*Y) is done later
X[k] = v3*up*vp;
X[k] = v_mul(v_mul(v3, up), vp);
//Z = ((12*13*L - up)*vp - 5); // (*Y) is done later
// xor flips the sign, works like unary minus
Z[k] = v_fma(v_fma(L[k], vx_setall_f32(12.f*13.f), (vx_setall_f32(-0.f) ^ up)), vp, vx_setall_f32(-5.f));
Z[k] = v_fma(v_fma(L[k], vx_setall_f32(12.f*13.f), (v_xor(vx_setall_f32(-0.F), up))), vp, vx_setall_f32(-5.f));
}
v_float32 R[nrepeats], G[nrepeats], B[nrepeats];
@@ -3156,9 +3157,9 @@ struct Luv2RGBfloat
for(int k = 0; k < nrepeats; k++)
{
// R = (X*C0 + C1 + Z*C2)*Y; // here (*Y) is done
R[k] = v_fma(Z[k], vc2, v_fma(X[k], vc0, vc1))*Y[k];
G[k] = v_fma(Z[k], vc5, v_fma(X[k], vc3, vc4))*Y[k];
B[k] = v_fma(Z[k], vc8, v_fma(X[k], vc6, vc7))*Y[k];
R[k] = v_mul(v_fma(Z[k], vc2, v_fma(X[k], vc0, vc1)), Y[k]);
G[k] = v_mul(v_fma(Z[k], vc5, v_fma(X[k], vc3, vc4)), Y[k]);
B[k] = v_mul(v_fma(Z[k], vc8, v_fma(X[k], vc6, vc7)), Y[k]);
}
v_float32 vzero = vx_setzero_f32(), v1 = vx_setall_f32(1.f);
@@ -3174,9 +3175,9 @@ struct Luv2RGBfloat
v_float32 vgscale = vx_setall_f32(gscale);
for(int k = 0; k < nrepeats; k++)
{
R[k] *= vgscale;
G[k] *= vgscale;
B[k] *= vgscale;
R[k] = v_mul(R[k], vgscale);
G[k] = v_mul(G[k], vgscale);
B[k] = v_mul(B[k], vgscale);
}
for(int k = 0; k < nrepeats; k++)
{
@@ -3285,7 +3286,7 @@ struct RGB2Luvinterpolate
#if CV_SIMD
if(enablePackedRGB2Luv)
{
const int vsize = v_uint16::nlanes;
const int vsize = VTraits<v_uint16>::vlanes();
static const int nPixels = vsize*2;
for(; i < n - 3*nPixels; i += 3*nPixels, src += scn*nPixels)
{
@@ -3315,9 +3316,9 @@ struct RGB2Luvinterpolate
v_expand(r, r0, r1);
v_expand(g, g0, g1);
v_expand(b, b0, b1);
r0 = r0 << (lab_base_shift - 8); r1 = r1 << (lab_base_shift - 8);
g0 = g0 << (lab_base_shift - 8); g1 = g1 << (lab_base_shift - 8);
b0 = b0 << (lab_base_shift - 8); b1 = b1 << (lab_base_shift - 8);
r0 = v_shl<lab_base_shift - 8>(r0); r1 = v_shl<lab_base_shift - 8>(r1);
g0 = v_shl<lab_base_shift - 8>(g0); g1 = v_shl<lab_base_shift - 8>(g1);
b0 = v_shl<lab_base_shift - 8>(b0); b1 = v_shl<lab_base_shift - 8>(b1);
/*
int L, u, v;
@@ -3332,9 +3333,9 @@ struct RGB2Luvinterpolate
dst[i+1] = saturate_cast<uchar>(u/baseDiv);
dst[i+2] = saturate_cast<uchar>(v/baseDiv);
*/
l0 = l0 >> (lab_base_shift - 8); l1 = l1 >> (lab_base_shift - 8);
u0 = u0 >> (lab_base_shift - 8); u1 = u1 >> (lab_base_shift - 8);
v0 = v0 >> (lab_base_shift - 8); v1 = v1 >> (lab_base_shift - 8);
l0 = v_shr<lab_base_shift - 8>(l0); l1 = v_shr<lab_base_shift - 8>(l1);
u0 = v_shr<lab_base_shift - 8>(u0); u1 = v_shr<lab_base_shift - 8>(u1);
v0 = v_shr<lab_base_shift - 8>(v0); v1 = v_shr<lab_base_shift - 8>(v1);
v_uint8 l = v_pack(l0, l1);
v_uint8 u = v_pack(u0, u1);
v_uint8 v = v_pack(v0, v1);
@@ -3405,12 +3406,12 @@ struct RGB2Luv_b
static const softfloat su = -uLow*f255/uRange;
static const softfloat sv = -vLow*f255/vRange;
#if CV_SIMD
const int fsize = v_float32::nlanes;
const int fsize = VTraits<v_float32>::vlanes();
v_float32 ml = vx_setall_f32((float)fL), al = vx_setzero_f32();
v_float32 mu = vx_setall_f32((float)fu), au = vx_setall_f32((float)su);
v_float32 mv = vx_setall_f32((float)fv), av = vx_setall_f32((float)sv);
//TODO: fix that when v_interleave is available
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3];
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits<v_float32>::max_nlanes*3], interTmpA[VTraits<v_float32>::max_nlanes*3];
v_store_interleave(interTmpM, ml, mu, mv);
v_store_interleave(interTmpA, al, au, av);
v_float32 mluv[3], aluv[3];
@@ -3452,7 +3453,7 @@ struct RGB2Luv_b
v_float32 f[3*4];
for(int k = 0; k < 3*4; k++)
{
f[k] = v_cvt_f32(q[k])*v255inv;
f[k] = v_mul(v_cvt_f32(q[k]), v255inv);
}
for(int k = 0; k < 4; k++)
@@ -3478,8 +3479,8 @@ struct RGB2Luv_b
v_int32 q0, q1;
v_expand(v_reinterpret_as_s16(d), q0, q1);
v_store_aligned(buf + j + 0*fsize, v_cvt_f32(q0)*v255inv);
v_store_aligned(buf + j + 1*fsize, v_cvt_f32(q1)*v255inv);
v_store_aligned(buf + j + 0*fsize, v_mul(v_cvt_f32(q0), v255inv));
v_store_aligned(buf + j + 1*fsize, v_mul(v_cvt_f32(q1), v255inv));
}
for( ; j < dn*bufChannels; j++, src++ )
{
@@ -3633,7 +3634,8 @@ struct Luv2RGBinteger
inline void processLuvToXYZ(const v_uint8& lv, const v_uint8& uv, const v_uint8& vv,
v_int32 (&x)[4], v_int32 (&y)[4], v_int32 (&z)[4]) const
{
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
const int vsize_max = VTraits<v_uint8>::max_nlanes;
v_uint16 lv0, lv1;
v_expand(lv, lv0, lv1);
@@ -3646,7 +3648,7 @@ struct Luv2RGBinteger
v_int32 mask16 = vx_setall_s32(0xFFFF);
for(int k = 0; k < 4; k++)
{
y[k] = v_lut((const int*)LabToYF_b, v_reinterpret_as_s32(lq[k])) & mask16;
y[k] = v_and(v_lut((const int *)LabToYF_b, v_reinterpret_as_s32(lq[k])), mask16);
}
v_int32 up[4], vp[4];
@@ -3657,10 +3659,10 @@ struct Luv2RGBinteger
v_expand(vv, vv0, vv1);
// LL*256
v_uint16 ll0, ll1;
ll0 = lv0 << 8; ll1 = lv1 << 8;
ll0 = v_shl<8>(lv0); ll1 = v_shl<8>(lv1);
v_uint16 upidx0, upidx1, vpidx0, vpidx1;
upidx0 = ll0 + uv0; upidx1 = ll1 + uv1;
vpidx0 = ll0 + vv0; vpidx1 = ll1 + vv1;
upidx0 = v_add(ll0, uv0); upidx1 = v_add(ll1, uv1);
vpidx0 = v_add(ll0, vv0); vpidx1 = v_add(ll1, vv1);
v_uint32 upidx[4], vpidx[4];
v_expand(upidx0, upidx[0], upidx[1]); v_expand(upidx1, upidx[2], upidx[3]);
v_expand(vpidx0, vpidx[0], vpidx[1]); v_expand(vpidx1, vpidx[2], vpidx[3]);
@@ -3672,7 +3674,7 @@ struct Luv2RGBinteger
// long long int vpl = LUVLUT.LvToVpl_b[LL*256+v];
v_int64 vpl[8];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vpidxstore[vsize];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vpidxstore[vsize_max];
for(int k = 0; k < 4; k++)
{
v_store_aligned(vpidxstore + k*vsize/4, v_reinterpret_as_s32(vpidx[k]));
@@ -3684,12 +3686,13 @@ struct Luv2RGBinteger
// not all 64-bit arithmetic is available in univ. intrinsics
// need to handle it with scalar code
int64_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vvpl[vsize];
int64_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vvpl[vsize_max];
for(int k = 0; k < 8; k++)
{
v_store_aligned(vvpl + k*vsize/8, vpl[k]);
}
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vup[vsize], vvp[vsize], vx[vsize], vy[vsize], vzm[vsize];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vup[vsize_max], vvp[vsize_max],
vx[vsize_max], vy[vsize_max], vzm[vsize_max];
for(int k = 0; k < 4; k++)
{
v_store_aligned(vup + k*vsize/4, up[k]);
@@ -3724,7 +3727,7 @@ struct Luv2RGBinteger
// z = zm/256 + zm/65536;
for (int k = 0; k < 4; k++)
{
z[k] = (zm[k] >> 8) + (zm[k] >> 16);
z[k] = v_add(v_shr<8>(zm[k]), v_shr<16>(zm[k]));
}
// (x, z) = clip((x, z), min=0, max=2*BASE)
@@ -3751,7 +3754,7 @@ struct Luv2RGBinteger
{
ushort* tab = sRGBInvGammaTab_b;
bool srgb = issRGB;
static const int vsize = v_uint8::nlanes;
static const int vsize = VTraits<v_uint8>::vlanes();
const int descaleShift = 1 << (shift-1);
v_int16 vdescale = vx_setall_s16(descaleShift);
v_int16 vc[9];
@@ -3771,12 +3774,12 @@ struct Luv2RGBinteger
// fixing 16bit signed multiplication
// by subtracting 2^(base_shift-1) and then adding result back
v_int32 dummy32, fm[3];
v_expand(vc[0]+vc[1]+vc[2], fm[0], dummy32);
v_expand(vc[3]+vc[4]+vc[5], fm[1], dummy32);
v_expand(vc[6]+vc[7]+vc[8], fm[2], dummy32);
fm[0] = fm[0] << (base_shift-1);
fm[1] = fm[1] << (base_shift-1);
fm[2] = fm[2] << (base_shift-1);
v_expand(v_add(vc[0],vc[1],vc[2]), fm[0], dummy32);
v_expand(v_add(vc[3],vc[4],vc[5]), fm[1], dummy32);
v_expand(v_add(vc[6],vc[7],vc[8]), fm[2], dummy32);
fm[0] = v_shl(fm[0], (base_shift-1));
fm[1] = v_shl(fm[1], (base_shift-1));
fm[2] = v_shl(fm[2], (base_shift-1));
for (; i <= n-vsize; i += vsize, src += 3*vsize, dst += dcn*vsize)
{
@@ -3816,15 +3819,15 @@ struct Luv2RGBinteger
// a bit faster than one loop for all
for(int k = 0; k < 4; k++)
{
i_rgb[k+4*0] = (v_dotprod(xy[k], crxy) + v_dotprod(zd[k], crz1) + fm[0]) >> shift;
i_rgb[k+4*0] = v_shr<shift>(v_add(v_add(v_dotprod(xy[k], crxy), v_dotprod(zd[k], crz1)), fm[0]));
}
for(int k = 0; k < 4; k++)
{
i_rgb[k+4*1] = (v_dotprod(xy[k], cgxy) + v_dotprod(zd[k], cgz1) + fm[1]) >> shift;
i_rgb[k+4*1] = v_shr<shift>(v_add(v_add(v_dotprod(xy[k], cgxy), v_dotprod(zd[k], cgz1)), fm[1]));
}
for(int k = 0; k < 4; k++)
{
i_rgb[k+4*2] = (v_dotprod(xy[k], cbxy) + v_dotprod(zd[k], cbz1) + fm[2]) >> shift;
i_rgb[k+4*2] = v_shr<shift>(v_add(v_add(v_dotprod(xy[k], cbxy), v_dotprod(zd[k], cbz1)), fm[2]));
}
// [rrggbb]
@@ -3842,7 +3845,7 @@ struct Luv2RGBinteger
if(srgb)
{
// [rr.., gg.., bb..]
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) rgbshifts[3*vsize];
int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) rgbshifts[3*VTraits<v_uint8>::max_nlanes];
for(int k = 0; k < 12; k++)
{
v_store_aligned(rgbshifts + k*vsize/4, i_rgb[k]);
@@ -3857,7 +3860,7 @@ struct Luv2RGBinteger
// rgb = (rgb*255) >> inv_gamma_shift
for(int k = 0; k < 12; k++)
{
i_rgb[k] = ((i_rgb[k] << 8) - i_rgb[k]) >> inv_gamma_shift;
i_rgb[k] = v_shr((v_sub((v_shl(i_rgb[k], 8)), i_rgb[k])), inv_gamma_shift);
}
for(int k = 0; k < 6; k++)
@@ -3940,13 +3943,13 @@ struct Luv2RGB_b
static const softfloat fv = vRange/f255;
#if CV_SIMD
const int fsize = v_float32::nlanes;
const int fsize = VTraits<v_float32>::vlanes();
v_float32 vl = vx_setall_f32((float)fl);
v_float32 vu = vx_setall_f32((float)fu);
v_float32 vv = vx_setall_f32((float)fv);
v_float32 vuLow = vx_setall_f32((float)uLow), vvLow = vx_setall_f32((float)vLow);
//TODO: fix that when v_interleave is available
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3];
float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits<v_float32>::max_nlanes*3], interTmpA[VTraits<v_float32>::max_nlanes*3];
v_store_interleave(interTmpM, vl, vu, vv);
v_store_interleave(interTmpA, vx_setzero_f32(), vuLow, vvLow);
v_float32 mluv[3], aluv[3];
@@ -3964,7 +3967,7 @@ struct Luv2RGB_b
j = 0;
#if CV_SIMD
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
for( ; j <= (dn - vsize)*3; j += 3*vsize )
{
v_uint8 s0, s1, s2;
@@ -4017,7 +4020,7 @@ struct Luv2RGB_b
v_int32 vi[4*3];
for(int k = 0; k < 4*3; k++)
{
vi[k] = v_round(vf[k]*v255);
vi[k] = v_round(v_mul(vf[k], v255));
}
v_uint8 rgb[3];
@@ -4039,7 +4042,7 @@ struct Luv2RGB_b
for(int k = 0; k < 4; k++)
{
vf[k] = vx_load_aligned(buf + j + k*fsize);
vi[k] = v_round(vf[k]*v255);
vi[k] = v_round(v_mul(vf[k], v255));
}
v_store(dst, v_pack_u(v_pack(vi[0], vi[1]), v_pack(vi[2], vi[3])));
}
+6 -6
View File
@@ -882,7 +882,7 @@ struct RGBA2mRGBA<uchar>
int i = 0;
#if CV_SIMD
const int vsize = v_uint8::nlanes;
const int vsize = VTraits<v_uint8>::vlanes();
v_uint8 amask = v_reinterpret_as_u8(vx_setall_u32(0xFF000000));
v_uint16 vh = vx_setall_u16(half_val+1);
@@ -901,27 +901,27 @@ struct RGBA2mRGBA<uchar>
v_uint16 a16[4];
for(int j = 0; j < 4; j++)
a16[j] = v_reinterpret_as_u16(v[j] & amask);
a16[j] = v_reinterpret_as_u16(v_and(v[j], amask));
v_uint32 a32[4];
for(int j = 0; j < 4; j++)
a32[j] = v_reinterpret_as_u32(a16[j] | (a16[j] >> 8));
a32[j] = v_reinterpret_as_u32(v_or(a16[j], (v_shr(a16[j], 8))));
v_uint8 a[4];
for(int j = 0; j < 4; j++)
a[j] = v_reinterpret_as_u8(a32[j] | (a32[j] >> 16));
a[j] = v_reinterpret_as_u8(v_or(a32[j], (v_shr(a32[j], 16))));
v_uint16 m[8];
for(int j = 0; j < 4; j++)
v_mul_expand(v[j], a[j], m[j], m[j+4]);
for(int j = 0; j < 8; j++)
m[j] += vh;
m[j] = v_add(m[j], vh);
// div 255: (v+1+(v>>8))>8
// +1 is in vh, has no effect on (v>>8)
for(int j = 0; j < 8; j++)
m[j] = (m[j] + (m[j] >> 8)) >> 8;
m[j] = v_shr((v_add(m[j], (v_shr(m[j], 8)))), 8);
v_uint8 d[4];
for(int j = 0; j < 4; j++)
+148 -146
View File
@@ -188,21 +188,21 @@ public:
v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step));
v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2));
v_uint16x8 b1 = ((r0 << 8) >> 7) + ((r2 << 8) >> 7);
v_uint16x8 b0 = v_rotate_right<1>(b1) + b1;
b1 = v_rotate_right<1>(b1) << 1;
v_uint16x8 b1 = v_add(v_shr<7>(v_shl<8>(r0)), v_shr<7>(v_shl<8>(r2)));
v_uint16x8 b0 = v_add(v_rotate_right<1>(b1), b1);
b1 = v_shl<1>(v_rotate_right<1>(b1));
v_uint16x8 g0 = (r0 >> 7) + (r2 >> 7);
v_uint16x8 g1 = (r1 << 8) >> 7;
g0 += v_rotate_right<1>(g1) + g1;
g1 = v_rotate_right<1>(g1) << 2;
v_uint16x8 g0 = v_add(v_shr<7>(r0), v_shr<7>(r2));
v_uint16x8 g1 = v_shr<7>(v_shl<8>(r1));
g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1));
g1 = v_shl<2>(v_rotate_right<1>(g1));
r0 = r1 >> 8;
r1 = (v_rotate_right<1>(r0) + r0) << 2;
r0 = r0 << 3;
r0 = v_shr<8>(r1);
r1 = v_shl<2>(v_add(v_rotate_right<1>(r0), r0));
r0 = v_shl<3>(r0);
g0 = (v_mul_hi(b0, _b2y) + v_mul_hi(g0, _g2y) + v_mul_hi(r0, _r2y)) >> 2;
g1 = (v_mul_hi(b1, _b2y) + v_mul_hi(g1, _g2y) + v_mul_hi(r1, _r2y)) >> 2;
g0 = v_shr<2>(v_add(v_add(v_mul_hi(b0, _b2y), v_mul_hi(g0, _g2y)), v_mul_hi(r0, _r2y)));
g1 = v_shr<2>(v_add(v_add(v_mul_hi(b1, _b2y), v_mul_hi(g1, _g2y)), v_mul_hi(r1, _r2y)));
v_uint8x16 pack_lo, pack_hi;
v_zip(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g0)),
v_pack_u(v_reinterpret_as_s16(g1), v_reinterpret_as_s16(g1)),
@@ -269,31 +269,31 @@ public:
v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step));
v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2));
v_uint16x8 b1 = (r0 & masklo) + (r2 & masklo);
v_uint16x8 b1 = v_add(v_and(r0, masklo), v_and(r2, masklo));
v_uint16x8 nextb1 = v_rotate_right<1>(b1);
v_uint16x8 b0 = b1 + nextb1;
b1 = (nextb1 + delta1) >> 1;
b0 = (b0 + delta2) >> 2;
v_uint16x8 b0 = v_add(b1, nextb1);
b1 = v_shr<1>(v_add(nextb1, delta1));
b0 = v_shr<2>(v_add(b0, delta2));
// b0 b2 ... b14 b1 b3 ... b15
b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1)));
v_uint16x8 g0 = (r0 >> 8) + (r2 >> 8);
v_uint16x8 g1 = r1 & masklo;
g0 += v_rotate_right<1>(g1) + g1;
v_uint16x8 g0 = v_add(v_shr<8>(r0), v_shr<8>(r2));
v_uint16x8 g1 = v_and(r1, masklo);
g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1));
g1 = v_rotate_right<1>(g1);
g0 = (g0 + delta2) >> 2;
g0 = v_shr<2>(v_add(g0, delta2));
// g0 g2 ... g14 g1 g3 ... g15
g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g1)));
r0 = r1 >> 8;
r1 = v_rotate_right<1>(r0) + r0;
r1 = (r1 + delta1) >> 1;
r0 = v_shr<8>(r1);
r1 = v_add(v_rotate_right<1>(r0), r0);
r1 = v_shr<1>(v_add(r1, delta1));
// r0 r2 ... r14 r1 r3 ... r15
r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1)));
b1 = (b0 ^ r0) & mask;
b0 = b0 ^ b1;
r0 = r0 ^ b1;
b1 = v_and(v_xor(b0, r0), mask);
b0 = v_xor(b0, b1);
r0 = v_xor(r0, b1);
// b1 g1 b3 g3 b5 g5...
v_uint8x16 pack_lo, pack_hi;
@@ -402,31 +402,31 @@ public:
v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step));
v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2));
v_uint16x8 b1 = (r0 & masklo) + (r2 & masklo);
v_uint16x8 b1 = v_add(v_and(r0, masklo), v_and(r2, masklo));
v_uint16x8 nextb1 = v_rotate_right<1>(b1);
v_uint16x8 b0 = b1 + nextb1;
b1 = (nextb1 + delta1) >> 1;
b0 = (b0 + delta2) >> 2;
v_uint16x8 b0 = v_add(b1, nextb1);
b1 = v_shr<1>(v_add(nextb1, delta1));
b0 = v_shr<2>(v_add(b0, delta2));
// b0 b2 ... b14 b1 b3 ... b15
b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1)));
v_uint16x8 g0 = (r0 >> 8) + (r2 >> 8);
v_uint16x8 g1 = r1 & masklo;
g0 += v_rotate_right<1>(g1) + g1;
v_uint16x8 g0 = v_add(v_shr<8>(r0), v_shr<8>(r2));
v_uint16x8 g1 = v_and(r1, masklo);
g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1));
g1 = v_rotate_right<1>(g1);
g0 = (g0 + delta2) >> 2;
g0 = v_shr<2>(v_add(g0, delta2));
// g0 g2 ... g14 g1 g3 ... g15
g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g1)));
r0 = r1 >> 8;
r1 = v_rotate_right<1>(r0) + r0;
r1 = (r1 + delta1) >> 1;
r0 = v_shr<8>(r1);
r1 = v_add(v_rotate_right<1>(r0), r0);
r1 = v_shr<1>(v_add(r1, delta1));
// r0 r2 ... r14 r1 r3 ... r15
r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1)));
b1 = (b0 ^ r0) & mask;
b0 = b0 ^ b1;
r0 = r0 ^ b1;
b1 = v_and(v_xor(b0, r0), mask);
b0 = v_xor(b0, b1);
r0 = v_xor(r0, b1);
// b1 g1 b3 g3 b5 g5...
v_uint8x16 pack_lo, pack_hi;
@@ -498,40 +498,40 @@ public:
v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step));
v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2));
v_uint16x8 b1 = (r0 & masklow) + (r2 & masklow);
v_uint16x8 b1 = v_add(v_and(r0, masklow), v_and(r2, masklow));
v_uint16x8 nextb1 = v_rotate_right<1>(b1);
v_uint16x8 b0 = b1 + nextb1;
b1 = (nextb1 + delta1) >> 1;
b0 = (b0 + delta2) >> 2;
v_uint16x8 b0 = v_add(b1, nextb1);
b1 = v_shr<1>(v_add(nextb1, delta1));
b0 = v_shr<2>(v_add(b0, delta2));
// b0 b2 ... b14 b1 b3 ... b15
b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1)));
// vertical sum
v_uint16x8 r0g = r0 >> 8;
v_uint16x8 r2g = r2 >> 8;
v_uint16x8 sumv = ((r0g + r2g) + delta1) >> 1;
v_uint16x8 r0g = v_shr<8>(r0);
v_uint16x8 r2g = v_shr<8>(r2);
v_uint16x8 sumv = v_shr<1>(v_add(v_add(r0g, r2g), delta1));
// horizontal sum
v_uint16x8 g1 = r1 & masklow;
v_uint16x8 g1 = v_and(r1, masklow);
v_uint16x8 nextg1 = v_rotate_right<1>(g1);
v_uint16x8 sumg = (g1 + nextg1 + delta1) >> 1;
v_uint16x8 sumg = v_shr<1>(v_add(v_add(g1, nextg1), delta1));
// gradients
v_uint16x8 gradv = (r0g - r2g) + (r2g - r0g);
v_uint16x8 gradg = (nextg1 - g1) + (g1 - nextg1);
v_uint16x8 gmask = gradg > gradv;
v_uint16x8 g0 = (gmask & sumv) + (sumg & (gmask ^ full));
v_uint16x8 gradv = v_add(v_sub(r0g, r2g), v_sub(r2g, r0g));
v_uint16x8 gradg = v_add(v_sub(nextg1, g1), v_sub(g1, nextg1));
v_uint16x8 gmask = v_gt(gradg, gradv);
v_uint16x8 g0 = v_add(v_and(gmask, sumv), v_and(sumg, v_xor(gmask, full)));
// g0 g2 ... g14 g1 g3 ...
g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(nextg1)));
r0 = r1 >> 8;
r1 = v_rotate_right<1>(r0) + r0;
r1 = (r1 + delta1) >> 1;
r0 = v_shr<8>(r1);
r1 = v_add(v_rotate_right<1>(r0), r0);
r1 = v_shr<1>(v_add(r1, delta1));
// r0 r2 ... r14 r1 r3 ... r15
r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1)));
b1 = (b0 ^ r0) & mask;
b0 = b0 ^ b1;
r0 = r0 ^ b1;
b1 = v_and(v_xor(b0, r0), mask);
b0 = v_xor(b0, b1);
r0 = v_xor(r0, b1);
// b1 g1 b3 g3 b5 g5...
v_uint8x16 pack_lo, pack_hi;
@@ -1060,19 +1060,19 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
v_uint16x8 b0, b1, b2, b3, b4, b5, b6;
b0 = (v_absdiff(s2, s8)<<1) + v_absdiff(s1, s7) + v_absdiff(s3, s9);
b1 = (v_absdiff(s4, s6)<<1) + v_absdiff(s1, s3) + v_absdiff(s7, s9);
b2 = v_absdiff(s3, s7)<<1;
b3 = v_absdiff(s1, s9)<<1;
b0 = v_add(v_add(v_shl<1>(v_absdiff(s2, s8)), v_absdiff(s1, s7)), v_absdiff(s3, s9));
b1 = v_add(v_add(v_shl<1>(v_absdiff(s4, s6)), v_absdiff(s1, s3)), v_absdiff(s7, s9));
b2 = v_shl<1>(v_absdiff(s3, s7));
b3 = v_shl<1>(v_absdiff(s1, s9));
v_store(brow, b0);
v_store(brow + N, b1);
v_store(brow + N2, b2);
v_store(brow + N3, b3);
b4 = b2 + v_absdiff(s2, s4) + v_absdiff(s6, s8);
b5 = b3 + v_absdiff(s2, s6) + v_absdiff(s4, s8);
b6 = (s2 + s4 + s6 + s8)>>1;
b4 = v_add(v_add(b2, v_absdiff(s2, s4)), v_absdiff(s6, s8));
b5 = v_add(v_add(b3, v_absdiff(s2, s6)), v_absdiff(s4, s8));
b6 = v_shr<1>(v_add(v_add(v_add(s2, s4), s6), s8));
v_store(brow + N4, b4);
v_store(brow + N5, b5);
@@ -1279,7 +1279,7 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
v_uint16x8 one = v_setall_u16(1), z = v_setzero_u16();
v_float32x4 _0_5 = v_setall_f32(0.5f);
#define v_merge_u16(a, b) (((a) & v_reinterpret_as_u16(emask)) | ((b) & v_reinterpret_as_u16(omask))) //(aA_aA_aA_aA) * (bB_bB_bB_bB) => (bA_bA_bA_bA)
#define v_merge_u16(a, b) (v_or((v_and((a), v_reinterpret_as_u16(emask))), (v_and((b), v_reinterpret_as_u16(omask))))) //(aA_aA_aA_aA) * (bB_bB_bB_bB) => (bA_bA_bA_bA)
#define v_cvt_s16f32_lo(a) v_cvt_f32(v_expand_low(v_reinterpret_as_s16(a))) //(1,2,3,4,5,6,7,8) => (1f,2f,3f,4f)
#define v_cvt_s16f32_hi(a) v_cvt_f32(v_expand_high(v_reinterpret_as_s16(a))) //(1,2,3,4,5,6,7,8) => (5f,6f,7f,8f)
@@ -1287,16 +1287,16 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
for( ; i <= N - 10; i += 8, srow += 8, brow0 += 8, brow1 += 8, brow2 += 8 )
{
//int gradN = brow0[0] + brow1[0];
v_uint16x8 gradN = v_load(brow0) + v_load(brow1);
v_uint16x8 gradN = v_add(v_load(brow0), v_load(brow1));
//int gradS = brow1[0] + brow2[0];
v_uint16x8 gradS = v_load(brow1) + v_load(brow2);
v_uint16x8 gradS = v_add(v_load(brow1), v_load(brow2));
//int gradW = brow1[N-1] + brow1[N];
v_uint16x8 gradW = v_load(brow1+N-1) + v_load(brow1+N);
v_uint16x8 gradW = v_add(v_load(brow1 + N - 1), v_load(brow1 + N));
//int gradE = brow1[N+1] + brow1[N];
v_uint16x8 gradE = v_load(brow1+N+1) + v_load(brow1+N);
v_uint16x8 gradE = v_add(v_load(brow1 + N + 1), v_load(brow1 + N));
//int minGrad = std::min(std::min(std::min(gradN, gradS), gradW), gradE);
//int maxGrad = std::max(std::max(std::max(gradN, gradS), gradW), gradE);
@@ -1307,14 +1307,14 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
//int gradNE = brow0[N4+1] + brow1[N4];
//int gradNE = brow0[N2] + brow0[N2+1] + brow1[N2] + brow1[N2+1];
grad0 = v_load(brow0+N4+1) + v_load(brow1+N4);
grad1 = v_load(brow0+N2) + v_load(brow0+N2+1) + v_load(brow1+N2) + v_load(brow1+N2+1);
grad0 = v_add(v_load(brow0 + N4 + 1), v_load(brow1 + N4));
grad1 = v_add(v_add(v_add(v_load(brow0 + N2), v_load(brow0 + N2 + 1)), v_load(brow1 + N2)), v_load(brow1 + N2 + 1));
v_uint16x8 gradNE = v_merge_u16(grad0, grad1);
//int gradSW = brow1[N4] + brow2[N4-1];
//int gradSW = brow1[N2] + brow1[N2-1] + brow2[N2] + brow2[N2-1];
grad0 = v_load(brow2+N4-1) + v_load(brow1+N4);
grad1 = v_load(brow2+N2) + v_load(brow2+N2-1) + v_load(brow1+N2) + v_load(brow1+N2-1);
grad0 = v_add(v_load(brow2 + N4 - 1), v_load(brow1 + N4));
grad1 = v_add(v_add(v_add(v_load(brow2 + N2), v_load(brow2 + N2 - 1)), v_load(brow1 + N2)), v_load(brow1 + N2 - 1));
v_uint16x8 gradSW = v_merge_u16(grad0, grad1);
minGrad = v_min(v_min(minGrad, gradNE), gradSW);
@@ -1322,21 +1322,21 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
//int gradNW = brow0[N5-1] + brow1[N5];
//int gradNW = brow0[N3] + brow0[N3-1] + brow1[N3] + brow1[N3-1];
grad0 = v_load(brow0+N5-1) + v_load(brow1+N5);
grad1 = v_load(brow0+N3) + v_load(brow0+N3-1) + v_load(brow1+N3) + v_load(brow1+N3-1);
grad0 = v_add(v_load(brow0 + N5 - 1), v_load(brow1 + N5));
grad1 = v_add(v_add(v_add(v_load(brow0 + N3), v_load(brow0 + N3 - 1)), v_load(brow1 + N3)), v_load(brow1 + N3 - 1));
v_uint16x8 gradNW = v_merge_u16(grad0, grad1);
//int gradSE = brow1[N5] + brow2[N5+1];
//int gradSE = brow1[N3] + brow1[N3+1] + brow2[N3] + brow2[N3+1];
grad0 = v_load(brow2+N5+1) + v_load(brow1+N5);
grad1 = v_load(brow2+N3) + v_load(brow2+N3+1) + v_load(brow1+N3) + v_load(brow1+N3+1);
grad0 = v_add(v_load(brow2 + N5 + 1), v_load(brow1 + N5));
grad1 = v_add(v_add(v_add(v_load(brow2 + N3), v_load(brow2 + N3 + 1)), v_load(brow1 + N3)), v_load(brow1 + N3 + 1));
v_uint16x8 gradSE = v_merge_u16(grad0, grad1);
minGrad = v_min(v_min(minGrad, gradNW), gradSE);
maxGrad = v_max(v_max(maxGrad, gradNW), gradSE);
//int T = minGrad + maxGrad/2;
v_uint16x8 T = v_max((maxGrad >> 1), one) + minGrad;
v_uint16x8 T = v_add(v_max((v_shr<1>(maxGrad)), one), minGrad);
v_uint16x8 RGs = z, GRs = z, Bs = z, ng = z;
@@ -1361,133 +1361,135 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code )
v_uint16x8 t0, t1, mask;
// gradN ***********************************************
mask = (T > gradN); // mask = T>gradN
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradN)
mask = (v_gt(T, gradN)); // mask = T>gradN
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradN)
t0 = (x3 << 1); // srow[-bstep]*2
t1 = v_load_expand(srow - bstep*2) + x0; // srow[-bstep*2] + srow[0]
t0 = (v_shl<1>(x3)); // srow[-bstep]*2
t1 = v_add(v_load_expand(srow - bstep * 2), x0); // srow[-bstep*2] + srow[0]
// RGs += (srow[-bstep*2] + srow[0]) * (T>gradN)
RGs += (t1 & mask);
RGs = v_add(RGs, v_and(t1, mask));
// GRs += {srow[-bstep]*2; (srow[-bstep*2-1] + srow[-bstep*2+1])} * (T>gradN)
GRs += (v_merge_u16(t0, x2 + x4) & mask);
GRs = v_add(GRs, (v_and(v_merge_u16(t0, v_add(x2, x4)), mask)));
// Bs += {(srow[-bstep-1]+srow[-bstep+1]); srow[-bstep]*2 } * (T>gradN)
Bs += (v_merge_u16(x1 + x5, t0) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_add(x1, x5), t0), mask));
// gradNE **********************************************
mask = (T > gradNE); // mask = T>gradNE
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradNE)
mask = (v_gt(T, gradNE)); // mask = T>gradNE
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradNE)
t0 = (x5 << 1); // srow[-bstep+1]*2
t1 = v_load_expand(srow - bstep*2+2) + x0; // srow[-bstep*2+2] + srow[0]
t0 = (v_shl<1>(x5)); // srow[-bstep+1]*2
t1 = v_add(v_load_expand(srow - bstep * 2 + 2), x0); // srow[-bstep*2+2] + srow[0]
// RGs += {(srow[-bstep*2+2] + srow[0]); srow[-bstep+1]*2} * (T>gradNE)
RGs += (v_merge_u16(t1, t0) & mask);
RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask));
// GRs += {brow0[N6+1]; (srow[-bstep*2+1] + srow[1])} * (T>gradNE)
GRs += (v_merge_u16(v_load(brow0+N6+1), x4 + x7) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow0+N6+1), v_add(x4, x7)), mask));
// Bs += {srow[-bstep+1]*2; (srow[-bstep] + srow[-bstep+2])} * (T>gradNE)
Bs += (v_merge_u16(t0, x3 + x6) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(t0, v_add(x3, x6)), mask));
// gradE ***********************************************
mask = (T > gradE); // mask = T>gradE
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradE)
mask = (v_gt(T, gradE)); // mask = T>gradE
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradE)
t0 = (x7 << 1); // srow[1]*2
t1 = v_load_expand(srow +2) + x0; // srow[2] + srow[0]
t0 = (v_shl<1>(x7)); // srow[1]*2
t1 = v_add(v_load_expand(srow + 2), x0); // srow[2] + srow[0]
// RGs += (srow[2] + srow[0]) * (T>gradE)
RGs += (t1 & mask);
RGs = v_add(RGs, v_and(t1, mask));
// GRs += (srow[1]*2) * (T>gradE)
GRs += (t0 & mask);
GRs = v_add(GRs, v_and(t0, mask));
// Bs += {(srow[-bstep+1]+srow[bstep+1]); (srow[-bstep+2]+srow[bstep+2])} * (T>gradE)
Bs += (v_merge_u16(x5 + x9, x6 + x8) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_add(x5, x9), v_add(x6, x8)), mask));
// gradSE **********************************************
mask = (T > gradSE); // mask = T>gradSE
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradSE)
mask = (v_gt(T, gradSE)); // mask = T>gradSE
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradSE)
t0 = (x9 << 1); // srow[bstep+1]*2
t1 = v_load_expand(srow + bstep*2+2) + x0; // srow[bstep*2+2] + srow[0]
t0 = (v_shl<1>(x9)); // srow[bstep+1]*2
t1 = v_add(v_load_expand(srow + bstep * 2 + 2), x0); // srow[bstep*2+2] + srow[0]
// RGs += {(srow[bstep*2+2] + srow[0]); srow[bstep+1]*2} * (T>gradSE)
RGs += (v_merge_u16(t1, t0) & mask);
RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask));
// GRs += {brow2[N6+1]; (srow[1]+srow[bstep*2+1])} * (T>gradSE)
GRs += (v_merge_u16(v_load(brow2+N6+1), x7 + x10) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow2+N6+1), v_add(x7, x10)), mask));
// Bs += {srow[bstep+1]*2; (srow[bstep+2]+srow[bstep])} * (T>gradSE)
Bs += (v_merge_u16((x9 << 1), x8 + x11) & mask);
Bs = v_add(Bs, v_and(v_merge_u16((v_shl<1>(x9)), v_add(x8, x11)), mask));
// gradS ***********************************************
mask = (T > gradS); // mask = T>gradS
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradS)
mask = (v_gt(T, gradS)); // mask = T>gradS
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradS)
t0 = (x11 << 1); // srow[bstep]*2
t1 = v_load_expand(srow + bstep*2) + x0; // srow[bstep*2]+srow[0]
t0 = (v_shl<1>(x11)); // srow[bstep]*2
t1 = v_add(v_load_expand(srow + bstep * 2), x0); // srow[bstep*2]+srow[0]
// RGs += (srow[bstep*2]+srow[0]) * (T>gradS)
RGs += (t1 & mask);
RGs = v_add(RGs, v_and(t1, mask));
// GRs += {srow[bstep]*2; (srow[bstep*2+1]+srow[bstep*2-1])} * (T>gradS)
GRs += (v_merge_u16(t0, x10 + x12) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(t0, v_add(x10, x12)), mask));
// Bs += {(srow[bstep+1]+srow[bstep-1]); srow[bstep]*2} * (T>gradS)
Bs += (v_merge_u16(x9 + x13, t0) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_add(x9, x13), t0), mask));
// gradSW **********************************************
mask = (T > gradSW); // mask = T>gradSW
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradSW)
mask = (v_gt(T, gradSW)); // mask = T>gradSW
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradSW)
t0 = (x13 << 1); // srow[bstep-1]*2
t1 = v_load_expand(srow + bstep*2-2) + x0; // srow[bstep*2-2]+srow[0]
t0 = (v_shl<1>(x13)); // srow[bstep-1]*2
t1 = v_add(v_load_expand(srow + bstep * 2 - 2), x0); // srow[bstep*2-2]+srow[0]
// RGs += {(srow[bstep*2-2]+srow[0]); srow[bstep-1]*2} * (T>gradSW)
RGs += (v_merge_u16(t1, t0) & mask);
RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask));
// GRs += {brow2[N6-1]; (srow[bstep*2-1]+srow[-1])} * (T>gradSW)
GRs += (v_merge_u16(v_load(brow2+N6-1), x12 + x15) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow2+N6-1), v_add(x12, x15)), mask));
// Bs += {srow[bstep-1]*2; (srow[bstep]+srow[bstep-2])} * (T>gradSW)
Bs += (v_merge_u16(t0, x11 + x14) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(t0, v_add(x11, x14)), mask));
// gradW ***********************************************
mask = (T > gradW); // mask = T>gradW
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradW)
mask = (v_gt(T, gradW)); // mask = T>gradW
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradW)
t0 = (x15 << 1); // srow[-1]*2
t1 = v_load_expand(srow -2) + x0; // srow[-2]+srow[0]
t0 = (v_shl<1>(x15)); // srow[-1]*2
t1 = v_add(v_load_expand(srow - 2), x0); // srow[-2]+srow[0]
// RGs += (srow[-2]+srow[0]) * (T>gradW)
RGs += (t1 & mask);
RGs = v_add(RGs, v_and(t1, mask));
// GRs += (srow[-1]*2) * (T>gradW)
GRs += (t0 & mask);
GRs = v_add(GRs, v_and(t0, mask));
// Bs += {(srow[-bstep-1]+srow[bstep-1]); (srow[bstep-2]+srow[-bstep-2])} * (T>gradW)
Bs += (v_merge_u16(x1 + x13, x14 + x16) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_add(x1, x13), v_add(x14, x16)), mask));
// gradNW **********************************************
mask = (T > gradNW); // mask = T>gradNW
ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradNW)
mask = (v_gt(T, gradNW)); // mask = T>gradNW
ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradNW)
t0 = (x1 << 1); // srow[-bstep-1]*2
t1 = v_load_expand(srow -bstep*2-2) + x0; // srow[-bstep*2-2]+srow[0]
t0 = (v_shl<1>(x1)); // srow[-bstep-1]*2
t1 = v_add(v_load_expand(srow - bstep * 2 - 2), x0); // srow[-bstep*2-2]+srow[0]
// RGs += {(srow[-bstep*2-2]+srow[0]); srow[-bstep-1]*2} * (T>gradNW)
RGs += (v_merge_u16(t1, t0) & mask);
RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask));
// GRs += {brow0[N6-1]; (srow[-bstep*2-1]+srow[-1])} * (T>gradNW)
GRs += (v_merge_u16(v_load(brow0+N6-1), x2 + x15) & mask);
GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow0+N6-1), v_add(x2, x15)), mask));
// Bs += {srow[-bstep-1]*2; (srow[-bstep]+srow[-bstep-2])} * (T>gradNW)
Bs += (v_merge_u16((x1 << 1), x3 + x16) & mask);
Bs = v_add(Bs, v_and(v_merge_u16(v_shl<1>(x1), v_add(x3, x16)), mask));
v_float32x4 ngf0 = _0_5 / v_cvt_s16f32_lo(ng);
v_float32x4 ngf1 = _0_5 / v_cvt_s16f32_hi(ng);
v_float32x4 ngf0 = v_div(_0_5, v_cvt_s16f32_lo(ng));
v_float32x4 ngf1 = v_div(_0_5, v_cvt_s16f32_hi(ng));
// now interpolate r, g & b
t0 = v_reinterpret_as_u16(v_reinterpret_as_s16(GRs) - v_reinterpret_as_s16(RGs));
t1 = v_reinterpret_as_u16(v_reinterpret_as_s16(Bs) - v_reinterpret_as_s16(RGs));
t0 = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(GRs), v_reinterpret_as_s16(RGs)));
t1 = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(Bs), v_reinterpret_as_s16(RGs)));
t0 = v_reinterpret_as_u16(v_reinterpret_as_s16(x0) +
t0 = v_reinterpret_as_u16(
v_add(v_reinterpret_as_s16(x0),
v_pack(
v_round(v_cvt_s16f32_lo(t0) * ngf0),
v_round(v_cvt_s16f32_hi(t0) * ngf1)));
v_round(v_mul(v_cvt_s16f32_lo(t0), ngf0)),
v_round(v_mul(v_cvt_s16f32_hi(t0), ngf1)))));
t1 = v_reinterpret_as_u16(v_reinterpret_as_s16(x0) +
t1 = v_reinterpret_as_u16(
v_add(v_reinterpret_as_s16(x0),
v_pack(
v_round(v_cvt_s16f32_lo(t1) * ngf0),
v_round(v_cvt_s16f32_hi(t1) * ngf1)));
v_round(v_mul(v_cvt_s16f32_lo(t1), ngf0)),
v_round(v_mul(v_cvt_s16f32_hi(t1), ngf1)))));
x1 = v_merge_u16(x0, t0);
x2 = v_merge_u16(t0, x0);
+19 -19
View File
@@ -1084,9 +1084,9 @@ struct SymmColumnVec_32s8u
i += VTraits<v_uint16>::vlanes();
}
#if CV_SIMD_WIDTH > 16
while( i <= width - 4 /*v_int32x4::nlanes*/ )
while( i <= width - 4 /*VTraits<v_int32x4>::vlanes()*/ )
#else
if( i <= width - v_int32::nlanes )
if( i <= width - VTraits<v_int32>::vlanes() )
#endif
{
v_float32 s0 = v_muladd(v_cvt_f32(vx_load(src[0] + i)), vx_setall_f32(ky[0]), vx_setall_f32(delta));
@@ -1140,9 +1140,9 @@ struct SymmColumnVec_32s8u
i += VTraits<v_uint16>::vlanes();
}
#if CV_SIMD_WIDTH > 16
while( i <= width - 4 /*v_int32x4::nlanes*/ )
while( i <= width - 4 /*VTraits<v_int32x4>::vlanes()*/ )
#else
if( i <= width - v_int32::nlanes )
if( i <= width - VTraits<v_int32>::vlanes() )
#endif
{
v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i))), vx_setall_f32(ky[1]), vx_setall_f32(delta));
@@ -1321,23 +1321,23 @@ struct SymmColumnSmallVec_32s16s
{
v_int32 k0 = vx_setall_s32((int)ky[0]), k1 = vx_setall_s32((int)ky[1]);
v_int32 d4 = vx_setall_s32(d);
for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes )
for( ; i <= width - 2*VTraits<v_int16>::vlanes(); i += 2*VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + v_int32::nlanes), k0, d4))));
v_store(dst + i + v_int16::nlanes, v_pack(v_muladd(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + 2*v_int32::nlanes), k0, d4)),
v_muladd(vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + 3*v_int32::nlanes), k0, d4))));
v_store(dst + i, v_pack(v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + VTraits<v_int32>::vlanes()), k0, d4))));
v_store(dst + i + VTraits<v_int16>::vlanes(), v_pack(v_muladd(v_add(vx_load(S0 + i + 2 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 2 * VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + 2*VTraits<v_int32>::vlanes()), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + 3 * VTraits<v_int32>::vlanes()), vx_load(S2 + i + 3 * VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + 3*VTraits<v_int32>::vlanes()), k0, d4))));
}
if( i <= width - v_int16::nlanes )
if( i <= width - VTraits<v_int16>::vlanes() )
{
v_store(dst + i, v_pack(v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + v_int32::nlanes), k0, d4))));
i += v_int16::nlanes;
v_store(dst + i, v_pack(v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)),
v_muladd(v_add(vx_load(S0 + i + VTraits<v_int32>::vlanes()), vx_load(S2 + i + VTraits<v_int32>::vlanes())), k1, v_muladd(vx_load(S1 + i + VTraits<v_int32>::vlanes()), k0, d4))));
i += VTraits<v_int16>::vlanes();
}
if( i <= width - v_int32::nlanes )
if( i <= width - VTraits<v_int32>::vlanes() )
{
v_pack_store(dst + i, v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4)));
i += v_int32::nlanes;
v_pack_store(dst + i, v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)));
i += VTraits<v_int32>::vlanes();
}
}
#endif
@@ -2237,9 +2237,9 @@ struct FilterVec_8u
i += VTraits<v_uint16>::vlanes();
}
#if CV_SIMD_WIDTH > 16
while( i <= width - 4 /*v_int32x4::nlanes*/ )
while( i <= width - 4 /*VTraits<v_int32x4>::vlanes()*/ )
#else
if( i <= width - v_int32::nlanes )
if( i <= width - VTraits<v_int32>::vlanes() )
#endif
{
v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[0] + i))), vx_setall_f32(kf[0]), vx_setall_f32(delta));
@@ -2248,7 +2248,7 @@ struct FilterVec_8u
v_int32 s32 = v_round(s0);
v_int16 s16 = v_pack(s32, s32);
*(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16)));
i += 4 /*v_int32x4::nlanes*/ ;
i += 4 /*VTraits<v_int32x4>::vlanes()*/ ;
}
return i;
}
+4 -4
View File
@@ -2106,7 +2106,7 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method )
v_float32 v_s11 = vx_setzero_f32();
v_float32 v_s12 = vx_setzero_f32();
v_float32 v_s22 = vx_setzero_f32();
for (; j <= len - v_float32::nlanes; j += v_float32::nlanes)
for (; j <= len - VTraits<v_float32>::vlanes(); j += VTraits<v_float32>::vlanes())
{
v_float32 v_a = vx_load(h1 + j);
v_float32 v_b = vx_load(h2 + j);
@@ -2147,10 +2147,10 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method )
result += v_reduce_sum(v_result);
#elif CV_SIMD
v_float32 v_result = vx_setzero_f32();
for (; j <= len - v_float32::nlanes; j += v_float32::nlanes)
for (; j <= len - VTraits<v_float32>::vlanes(); j += VTraits<v_float32>::vlanes())
{
v_float32 v_src = v_min(vx_load(h1 + j), vx_load(h2 + j));
v_result += v_src;
v_result = v_add(v_result, v_src);
}
result += v_reduce_sum(v_result);
#endif
@@ -2187,7 +2187,7 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method )
v_float32 v_s1 = vx_setzero_f32();
v_float32 v_s2 = vx_setzero_f32();
v_float32 v_result = vx_setzero_f32();
for (; j <= len - v_float32::nlanes; j += v_float32::nlanes)
for (; j <= len - VTraits<v_float32>::vlanes(); j += VTraits<v_float32>::vlanes())
{
v_float32 v_a = vx_load(h1 + j);
v_float32 v_b = vx_load(h2 + j);
+136 -136
View File
@@ -455,7 +455,7 @@ struct RemapVec_8u
v_int32x4 delta = v_setall_s32(INTER_REMAP_COEF_SCALE / 2);
v_int16x8 xy2ofs = v_reinterpret_as_s16(v_setall_s32(cn + (sstep << 16)));
int CV_DECL_ALIGNED(16) iofs0[4], iofs1[4];
const uchar* src_limit_8bytes = _src.datalimit - v_int16x8::nlanes;
const uchar* src_limit_8bytes = _src.datalimit - VTraits<v_int16x8>::vlanes();
#define CV_PICK_AND_PACK_RGB(ptr, offset, result) \
{ \
const uchar* const p = ((const uchar*)ptr) + (offset); \
@@ -483,7 +483,7 @@ struct RemapVec_8u
v_uint8x16 rrggbbaa, dummy; \
v_uint16x8 rrggbbaa8, dummy8; \
v_uint8x16 rgba0 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p), 0, 0, 0)); \
v_uint8x16 rgba1 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p + v_int32x4::nlanes), 0, 0, 0)); \
v_uint8x16 rgba1 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p + VTraits<v_int32x4>::vlanes()), 0, 0, 0)); \
v_zip(rgba0, rgba1, rrggbbaa, dummy); \
v_expand(rrggbbaa, rrggbbaa8, dummy8); \
result = v_reinterpret_as_s16(rrggbbaa8); \
@@ -534,8 +534,8 @@ struct RemapVec_8u
v3 = v_dotprod(v_reinterpret_as_s16(v3), v_reinterpret_as_s16(d2), delta);
v2 = v_dotprod(v_reinterpret_as_s16(v2), v_reinterpret_as_s16(c2), v3);
v0 = v0 >> INTER_REMAP_COEF_BITS;
v2 = v2 >> INTER_REMAP_COEF_BITS;
v0 = v_shr<INTER_REMAP_COEF_BITS>(v0);
v2 = v_shr<INTER_REMAP_COEF_BITS>(v2);
v_pack_u_store(D + x, v_pack(v0, v2));
}
}
@@ -563,8 +563,8 @@ struct RemapVec_8u
CV_PICK_AND_PACK_RGB(S0, iofs0[1], u1);
CV_PICK_AND_PACK_RGB(S1, iofs0[1], v1);
v_int32x4 result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS;
v_int32x4 result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS;
v_int32x4 result0 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u0, w00, v_dotprod(v0, w01, delta)));
v_int32x4 result1 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u1, w10, v_dotprod(v1, w11, delta)));
result0 = v_rotate_left<1>(result0);
v_int16x8 result8 = v_pack(result0, result1);
@@ -581,8 +581,8 @@ struct RemapVec_8u
CV_PICK_AND_PACK_RGB(S0, iofs0[3], u1);
CV_PICK_AND_PACK_RGB(S1, iofs0[3], v1);
result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS;
result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS;
result0 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u0, w00, v_dotprod(v0, w01, delta)));
result1 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u1, w10, v_dotprod(v1, w11, delta)));
result0 = v_rotate_left<1>(result0);
result8 = v_pack(result0, result1);
@@ -613,8 +613,8 @@ struct RemapVec_8u
CV_PICK_AND_PACK_RGBA(S0, iofs0[1], u1);
CV_PICK_AND_PACK_RGBA(S1, iofs0[1], v1);
v_int32x4 result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS;
v_int32x4 result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS;
v_int32x4 result0 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u0, w00, v_dotprod(v0, w01, delta)));
v_int32x4 result1 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u1, w10, v_dotprod(v1, w11, delta)));
v_int16x8 result8 = v_pack(result0, result1);
v_pack_u_store(D, result8);
@@ -627,8 +627,8 @@ struct RemapVec_8u
CV_PICK_AND_PACK_RGBA(S0, iofs0[3], u1);
CV_PICK_AND_PACK_RGBA(S1, iofs0[3], v1);
result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS;
result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS;
result0 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u0, w00, v_dotprod(v0, w01, delta)));
result1 = v_shr<INTER_REMAP_COEF_BITS>(v_dotprod(u1, w10, v_dotprod(v1, w11, delta)));
result8 = v_pack(result0, result1);
v_pack_u_store(D + 8, result8);
}
@@ -1164,7 +1164,7 @@ public:
#if CV_SIMD128
{
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x1 <= bcols - span * 2; x1 += span * 2 )
{
v_int32x4 ix0 = v_round(v_load(sX + x1));
@@ -1206,9 +1206,9 @@ public:
#if CV_SIMD128
{
v_uint16x8 v_scale = v_setall_u16(INTER_TAB_SIZE2 - 1);
int span = v_uint16x8::nlanes;
int span = VTraits<v_uint16x8>::vlanes();
for( ; x1 <= bcols - span; x1 += span )
v_store((unsigned short*)(A + x1), v_load(sA + x1) & v_scale);
v_store((unsigned short*)(A + x1), v_and(v_load(sA + x1), v_scale));
}
#endif
for( ; x1 < bcols; x1++ )
@@ -1224,16 +1224,16 @@ public:
{
v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE);
v_int32x4 v_scale2 = v_setall_s32(INTER_TAB_SIZE - 1);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x1 <= bcols - span * 2; x1 += span * 2 )
{
v_int32x4 v_sx0 = v_round(v_scale * v_load(sX + x1));
v_int32x4 v_sy0 = v_round(v_scale * v_load(sY + x1));
v_int32x4 v_sx1 = v_round(v_scale * v_load(sX + x1 + span));
v_int32x4 v_sy1 = v_round(v_scale * v_load(sY + x1 + span));
v_uint16x8 v_sx8 = v_reinterpret_as_u16(v_pack(v_sx0 & v_scale2, v_sx1 & v_scale2));
v_uint16x8 v_sy8 = v_reinterpret_as_u16(v_pack(v_sy0 & v_scale2, v_sy1 & v_scale2));
v_uint16x8 v_v = v_shl<INTER_BITS>(v_sy8) | (v_sx8);
v_int32x4 v_sx0 = v_round(v_mul(v_scale, v_load(sX + x1)));
v_int32x4 v_sy0 = v_round(v_mul(v_scale, v_load(sY + x1)));
v_int32x4 v_sx1 = v_round(v_mul(v_scale, v_load(sX + x1 + span)));
v_int32x4 v_sy1 = v_round(v_mul(v_scale, v_load(sY + x1 + span)));
v_uint16x8 v_sx8 = v_reinterpret_as_u16(v_pack(v_and(v_sx0, v_scale2), v_and(v_sx1, v_scale2)));
v_uint16x8 v_sy8 = v_reinterpret_as_u16(v_pack(v_and(v_sy0, v_scale2), v_and(v_sy1, v_scale2)));
v_uint16x8 v_v = v_or(v_shl<INTER_BITS>(v_sy8), v_sx8);
v_store(A + x1, v_v);
v_int16x8 v_d0 = v_pack(v_shr<INTER_BITS>(v_sx0), v_shr<INTER_BITS>(v_sx1));
@@ -1261,18 +1261,18 @@ public:
{
v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE);
v_int32x4 v_scale2 = v_setall_s32(INTER_TAB_SIZE - 1), v_scale3 = v_setall_s32(INTER_TAB_SIZE);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x1 <= bcols - span * 2; x1 += span * 2 )
{
v_float32x4 v_fx, v_fy;
v_load_deinterleave(sXY + (x1 << 1), v_fx, v_fy);
v_int32x4 v_sx0 = v_round(v_fx * v_scale);
v_int32x4 v_sy0 = v_round(v_fy * v_scale);
v_int32x4 v_sx0 = v_round(v_mul(v_fx, v_scale));
v_int32x4 v_sy0 = v_round(v_mul(v_fy, v_scale));
v_load_deinterleave(sXY + ((x1 + span) << 1), v_fx, v_fy);
v_int32x4 v_sx1 = v_round(v_fx * v_scale);
v_int32x4 v_sy1 = v_round(v_fy * v_scale);
v_int32x4 v_v0 = v_muladd(v_scale3, (v_sy0 & v_scale2), (v_sx0 & v_scale2));
v_int32x4 v_v1 = v_muladd(v_scale3, (v_sy1 & v_scale2), (v_sx1 & v_scale2));
v_int32x4 v_sx1 = v_round(v_mul(v_fx, v_scale));
v_int32x4 v_sy1 = v_round(v_mul(v_fy, v_scale));
v_int32x4 v_v0 = v_muladd(v_scale3, (v_and(v_sy0, v_scale2)), (v_and(v_sx0, v_scale2)));
v_int32x4 v_v1 = v_muladd(v_scale3, (v_and(v_sy1, v_scale2)), (v_and(v_sx1, v_scale2)));
v_uint16x8 v_v8 = v_reinterpret_as_u16(v_pack(v_v0, v_v1));
v_store(A + x1, v_v8);
v_int16x8 v_dx = v_pack(v_shr<INTER_BITS>(v_sx0), v_shr<INTER_BITS>(v_sx1));
@@ -1941,7 +1941,7 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
{
#if CV_SIMD128
{
int span = v_int16x8::nlanes;
int span = VTraits<v_int16x8>::vlanes();
for( ; x <= size.width - span; x += span )
{
v_int16x8 v_dst[2];
@@ -1973,21 +1973,21 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE);
v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1);
v_int32x4 v_scale3 = v_setall_s32(INTER_TAB_SIZE);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x <= size.width - span * 2; x += span * 2 )
{
v_int32x4 v_ix0 = v_round(v_scale * (v_load(src1f + x)));
v_int32x4 v_ix1 = v_round(v_scale * (v_load(src1f + x + span)));
v_int32x4 v_iy0 = v_round(v_scale * (v_load(src2f + x)));
v_int32x4 v_iy1 = v_round(v_scale * (v_load(src2f + x + span)));
v_int32x4 v_ix0 = v_round(v_mul(v_scale, v_load(src1f + x)));
v_int32x4 v_ix1 = v_round(v_mul(v_scale, v_load(src1f + x + span)));
v_int32x4 v_iy0 = v_round(v_mul(v_scale, v_load(src2f + x)));
v_int32x4 v_iy1 = v_round(v_mul(v_scale, v_load(src2f + x + span)));
v_int16x8 v_dst[2];
v_dst[0] = v_pack(v_shr<INTER_BITS>(v_ix0), v_shr<INTER_BITS>(v_ix1));
v_dst[1] = v_pack(v_shr<INTER_BITS>(v_iy0), v_shr<INTER_BITS>(v_iy1));
v_store_interleave(dst1 + (x << 1), v_dst[0], v_dst[1]);
v_int32x4 v_dst0 = v_muladd(v_scale3, (v_iy0 & v_mask), (v_ix0 & v_mask));
v_int32x4 v_dst1 = v_muladd(v_scale3, (v_iy1 & v_mask), (v_ix1 & v_mask));
v_int32x4 v_dst0 = v_muladd(v_scale3, (v_and(v_iy0, v_mask)), (v_and(v_ix0, v_mask)));
v_int32x4 v_dst1 = v_muladd(v_scale3, (v_and(v_iy1, v_mask)), (v_and(v_ix1, v_mask)));
v_store(dst2 + x, v_pack_u(v_dst0, v_dst1));
}
}
@@ -2008,7 +2008,7 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
if( nninterpolate )
{
#if CV_SIMD128
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
{
for( ; x <= (size.width << 1) - span * 2; x += span * 2 )
v_store(dst1 + x, v_pack(v_round(v_load(src1f + x)),
@@ -2034,16 +2034,16 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE);
v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1);
v_int32x4 v_scale3 = v_setall_s32(INTER_TAB_SIZE);
int span = v_uint16x8::nlanes;
int span = VTraits<v_uint16x8>::vlanes();
for (; x <= size.width - span; x += span )
{
v_float32x4 v_src0[2], v_src1[2];
v_load_deinterleave(src1f + (x << 1), v_src0[0], v_src0[1]);
v_load_deinterleave(src1f + (x << 1) + span, v_src1[0], v_src1[1]);
v_int32x4 v_ix0 = v_round(v_src0[0] * v_scale);
v_int32x4 v_ix1 = v_round(v_src1[0] * v_scale);
v_int32x4 v_iy0 = v_round(v_src0[1] * v_scale);
v_int32x4 v_iy1 = v_round(v_src1[1] * v_scale);
v_int32x4 v_ix0 = v_round(v_mul(v_src0[0], v_scale));
v_int32x4 v_ix1 = v_round(v_mul(v_src1[0], v_scale));
v_int32x4 v_iy0 = v_round(v_mul(v_src0[1], v_scale));
v_int32x4 v_iy1 = v_round(v_mul(v_src1[1], v_scale));
v_int16x8 v_dst[2];
v_dst[0] = v_pack(v_shr<INTER_BITS>(v_ix0), v_shr<INTER_BITS>(v_ix1));
@@ -2051,8 +2051,8 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_store_interleave(dst1 + (x << 1), v_dst[0], v_dst[1]);
v_store(dst2 + x, v_pack_u(
v_muladd(v_scale3, (v_iy0 & v_mask), (v_ix0 & v_mask)),
v_muladd(v_scale3, (v_iy1 & v_mask), (v_ix1 & v_mask))));
v_muladd(v_scale3, (v_and(v_iy0, v_mask)), (v_and(v_ix0, v_mask))),
v_muladd(v_scale3, (v_and(v_iy1, v_mask)), (v_and(v_ix1, v_mask)))));
}
}
#endif
@@ -2074,13 +2074,13 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_uint16x8 v_mask2 = v_setall_u16(INTER_TAB_SIZE2-1);
v_uint32x4 v_zero = v_setzero_u32(), v_mask = v_setall_u32(INTER_TAB_SIZE-1);
v_float32x4 v_scale = v_setall_f32(scale);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x <= size.width - span * 2; x += span * 2 )
{
v_uint32x4 v_fxy1, v_fxy2;
if ( src2 )
{
v_uint16x8 v_src2 = v_load(src2 + x) & v_mask2;
v_uint16x8 v_src2 = v_and(v_load(src2 + x), v_mask2);
v_expand(v_src2, v_fxy1, v_fxy2);
}
else
@@ -2091,9 +2091,9 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_load_deinterleave(src1 + (x << 1), v_src[0], v_src[1]);
v_expand(v_src[0], v_src0[0], v_src0[1]);
v_expand(v_src[1], v_src1[0], v_src1[1]);
#define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32((FXY) & v_mask)),\
#define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32(v_and((FXY), v_mask))),\
v_cvt_f32(v_reinterpret_as_s32(X)))
#define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32((FXY) >> INTER_BITS)),\
#define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32(v_shr<INTER_BITS>((FXY)))),\
v_cvt_f32(v_reinterpret_as_s32(Y)))
v_float32x4 v_dst1 = CV_COMPUTE_MAP_X(v_src0[0], v_fxy1);
v_float32x4 v_dst2 = CV_COMPUTE_MAP_Y(v_src1[0], v_fxy1);
@@ -2123,13 +2123,13 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_int16x8 v_mask2 = v_setall_s16(INTER_TAB_SIZE2-1);
v_int32x4 v_zero = v_setzero_s32(), v_mask = v_setall_s32(INTER_TAB_SIZE-1);
v_float32x4 v_scale = v_setall_f32(scale);
int span = v_int16x8::nlanes;
int span = VTraits<v_int16x8>::vlanes();
for( ; x <= size.width - span; x += span )
{
v_int32x4 v_fxy1, v_fxy2;
if (src2)
{
v_int16x8 v_src2 = v_load((short *)src2 + x) & v_mask2;
v_int16x8 v_src2 = v_and(v_load((short *)src2 + x), v_mask2);
v_expand(v_src2, v_fxy1, v_fxy2);
}
else
@@ -2142,8 +2142,8 @@ void cv::convertMaps( InputArray _map1, InputArray _map2,
v_expand(v_src[0], v_src0[0], v_src0[1]);
v_expand(v_src[1], v_src1[0], v_src1[1]);
#define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32((FXY) & v_mask), v_cvt_f32(X))
#define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32((FXY) >> INTER_BITS), v_cvt_f32(Y))
#define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_and((FXY), v_mask)), v_cvt_f32(X))
#define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_shr<INTER_BITS>((FXY))), v_cvt_f32(Y))
v_dst[0] = CV_COMPUTE_MAP_X(v_src0[0], v_fxy1);
v_dst[1] = CV_COMPUTE_MAP_Y(v_src1[0], v_fxy1);
v_store_interleave(dst1f + (x << 1), v_dst[0], v_dst[1]);
@@ -2234,12 +2234,12 @@ public:
#if CV_SIMD128
{
v_int32x4 v_X0 = v_setall_s32(X0), v_Y0 = v_setall_s32(Y0);
int span = v_uint16x8::nlanes;
int span = VTraits<v_uint16x8>::vlanes();
for( ; x1 <= bw - span; x1 += span )
{
v_int16x8 v_dst[2];
#define CV_CONVERT_MAP(ptr,offset,shift) v_pack(v_shr<AB_BITS>(shift+v_load(ptr + offset)),\
v_shr<AB_BITS>(shift+v_load(ptr + offset + 4)))
#define CV_CONVERT_MAP(ptr,offset,shift) v_pack(v_shr<AB_BITS>(v_add(shift,v_load(ptr + offset))),\
v_shr<AB_BITS>(v_add(shift,v_load(ptr + offset + 4))))
v_dst[0] = CV_CONVERT_MAP(adelta, x+x1, v_X0);
v_dst[1] = CV_CONVERT_MAP(bdelta, x+x1, v_Y0);
#undef CV_CONVERT_MAP
@@ -2272,21 +2272,21 @@ public:
{
v_int32x4 v__X0 = v_setall_s32(X0), v__Y0 = v_setall_s32(Y0);
v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1);
int span = v_float32x4::nlanes;
int span = VTraits<v_float32x4>::vlanes();
for( ; x1 <= bw - span * 2; x1 += span * 2 )
{
v_int32x4 v_X0 = v_shr<AB_BITS - INTER_BITS>(v__X0 + v_load(adelta + x + x1));
v_int32x4 v_Y0 = v_shr<AB_BITS - INTER_BITS>(v__Y0 + v_load(bdelta + x + x1));
v_int32x4 v_X1 = v_shr<AB_BITS - INTER_BITS>(v__X0 + v_load(adelta + x + x1 + span));
v_int32x4 v_Y1 = v_shr<AB_BITS - INTER_BITS>(v__Y0 + v_load(bdelta + x + x1 + span));
v_int32x4 v_X0 = v_shr<AB_BITS - INTER_BITS>(v_add(v__X0, v_load(this->adelta + x + x1)));
v_int32x4 v_Y0 = v_shr<AB_BITS - INTER_BITS>(v_add(v__Y0, v_load(this->bdelta + x + x1)));
v_int32x4 v_X1 = v_shr<AB_BITS - INTER_BITS>(v_add(v__X0, v_load(this->adelta + x + x1 + span)));
v_int32x4 v_Y1 = v_shr<AB_BITS - INTER_BITS>(v_add(v__Y0, v_load(this->bdelta + x + x1 + span)));
v_int16x8 v_xy[2];
v_xy[0] = v_pack(v_shr<INTER_BITS>(v_X0), v_shr<INTER_BITS>(v_X1));
v_xy[1] = v_pack(v_shr<INTER_BITS>(v_Y0), v_shr<INTER_BITS>(v_Y1));
v_store_interleave(xy + (x1 << 1), v_xy[0], v_xy[1]);
v_int32x4 v_alpha0 = v_shl<INTER_BITS>(v_Y0 & v_mask) | (v_X0 & v_mask);
v_int32x4 v_alpha1 = v_shl<INTER_BITS>(v_Y1 & v_mask) | (v_X1 & v_mask);
v_int32x4 v_alpha0 = v_or(v_shl<INTER_BITS>(v_and(v_Y0, v_mask)), v_and(v_X0, v_mask));
v_int32x4 v_alpha1 = v_or(v_shl<INTER_BITS>(v_and(v_Y1, v_mask)), v_and(v_X1, v_mask));
v_store(alpha + x1, v_pack(v_alpha0, v_alpha1));
}
}
@@ -2866,16 +2866,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0
v_int32x4 v_X0, v_Y0;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X0 = v_round(v_fX0, v_fX1);
v_Y0 = v_round(v_fY0, v_fY1);
@@ -2885,16 +2885,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0
v_int32x4 v_X1, v_Y1;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X1 = v_round(v_fX0, v_fX1);
v_Y1 = v_round(v_fY0, v_fY1);
@@ -2904,16 +2904,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0
v_int32x4 v_X2, v_Y2;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X2 = v_round(v_fX0, v_fX1);
v_Y2 = v_round(v_fY0, v_fY1);
@@ -2923,16 +2923,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0
v_int32x4 v_X3, v_Y3;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X3 = v_round(v_fX0, v_fX1);
v_Y3 = v_round(v_fY0, v_fY1);
@@ -2987,16 +2987,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph
v_int32x4 v_X0, v_Y0;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X0 = v_round(v_fX0, v_fX1);
v_Y0 = v_round(v_fY0, v_fY1);
@@ -3006,16 +3006,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph
v_int32x4 v_X1, v_Y1;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X1 = v_round(v_fX0, v_fX1);
v_Y1 = v_round(v_fY0, v_fY1);
@@ -3025,16 +3025,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph
v_int32x4 v_X2, v_Y2;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X2 = v_round(v_fX0, v_fX1);
v_Y2 = v_round(v_fY0, v_fY1);
@@ -3044,35 +3044,35 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph
v_int32x4 v_X3, v_Y3;
{
v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_W = v_muladd(v_M6, v_x1, v_W0);
v_W = v_select(v_W != v_zero, v_its / v_W, v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W));
v_x1 += v_2;
v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero);
v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W)));
v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W)));
v_x1 = v_add(v_x1, v_2);
v_X3 = v_round(v_fX0, v_fX1);
v_Y3 = v_round(v_fY0, v_fY1);
}
// store alpha
v_int32x4 v_alpha0 = ((v_Y0 & v_itsi1) << INTER_BITS) + (v_X0 & v_itsi1);
v_int32x4 v_alpha1 = ((v_Y1 & v_itsi1) << INTER_BITS) + (v_X1 & v_itsi1);
v_int32x4 v_alpha0 = v_add(v_shl<INTER_BITS>(v_and(v_Y0, v_itsi1)), v_and(v_X0, v_itsi1));
v_int32x4 v_alpha1 = v_add(v_shl<INTER_BITS>(v_and(v_Y1, v_itsi1)), v_and(v_X1, v_itsi1));
v_store((alpha + x1), v_pack(v_alpha0, v_alpha1));
v_alpha0 = ((v_Y2 & v_itsi1) << INTER_BITS) + (v_X2 & v_itsi1);
v_alpha1 = ((v_Y3 & v_itsi1) << INTER_BITS) + (v_X3 & v_itsi1);
v_alpha0 = v_add(v_shl<INTER_BITS>(v_and(v_Y2, v_itsi1)), v_and(v_X2, v_itsi1));
v_alpha1 = v_add(v_shl<INTER_BITS>(v_and(v_Y3, v_itsi1)), v_and(v_X3, v_itsi1));
v_store((alpha + x1 + 8), v_pack(v_alpha0, v_alpha1));
// convert to 16s
v_X0 = v_reinterpret_as_s32(v_pack(v_X0 >> INTER_BITS, v_X1 >> INTER_BITS));
v_X1 = v_reinterpret_as_s32(v_pack(v_X2 >> INTER_BITS, v_X3 >> INTER_BITS));
v_Y0 = v_reinterpret_as_s32(v_pack(v_Y0 >> INTER_BITS, v_Y1 >> INTER_BITS));
v_Y1 = v_reinterpret_as_s32(v_pack(v_Y2 >> INTER_BITS, v_Y3 >> INTER_BITS));
v_X0 = v_reinterpret_as_s32(v_pack(v_shr<INTER_BITS>(v_X0), v_shr<INTER_BITS>(v_X1)));
v_X1 = v_reinterpret_as_s32(v_pack(v_shr<INTER_BITS>(v_X2), v_shr<INTER_BITS>(v_X3)));
v_Y0 = v_reinterpret_as_s32(v_pack(v_shr<INTER_BITS>(v_Y0), v_shr<INTER_BITS>(v_Y1)));
v_Y1 = v_reinterpret_as_s32(v_pack(v_shr<INTER_BITS>(v_Y2), v_shr<INTER_BITS>(v_Y3)));
v_store_interleave(xy + x1 * 2, (v_reinterpret_as_s16)(v_X0), (v_reinterpret_as_s16)(v_Y0));
v_store_interleave(xy + x1 * 2 + 16, (v_reinterpret_as_s16)(v_X1), (v_reinterpret_as_s16)(v_Y1));
+21 -21
View File
@@ -179,10 +179,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
for (k = 0; k < 16; ++k)
{
#if CV_SIMD256
v_store(H.fine[k], v_mul_wrap(v256_load(h_fine + 16 * n*(16 * c + k)), v256_setall_u16(2 * r + 1)) + v256_load(H.fine[k]));
v_store(H.fine[k], v_mul_wrap(v256_load(h_fine + 16 * n*(16 * c + k)), v_add(v256_setall_u16(2 * r + 1), v256_load(H.fine[k]))));
#elif CV_SIMD128
v_store(H.fine[k], v_mul_wrap(v_load(h_fine + 16 * n*(16 * c + k)), v_setall_u16((ushort)(2 * r + 1))) + v_load(H.fine[k]));
v_store(H.fine[k] + 8, v_mul_wrap(v_load(h_fine + 16 * n*(16 * c + k) + 8), v_setall_u16((ushort)(2 * r + 1))) + v_load(H.fine[k] + 8));
v_store(H.fine[k], v_add(v_mul_wrap(v_load(h_fine + 16 * n * (16 * c + k)), v_setall_u16((ushort)(2 * r + 1))), v_load(H.fine[k])));
v_store(H.fine[k] + 8, v_add(v_mul_wrap(v_load(h_fine + 16 * n * (16 * c + k) + 8), v_setall_u16((ushort)(2 * r + 1))), v_load(H.fine[k] + 8)));
#else
for (int ind = 0; ind < 16; ++ind)
H.fine[k][ind] = (HT)(H.fine[k][ind] + (2 * r + 1) * h_fine[16 * n*(16 * c + k) + ind]);
@@ -199,10 +199,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
for( j = 0; j < 2*r; ++j, px += 16 )
{
#if CV_SIMD256
v_coarse += v256_load(px);
v_coarse = v_add(v_coarse, v256_load(px));
#elif CV_SIMD128
v_coarsel += v_load(px);
v_coarseh += v_load(px + 8);
v_coarsel = v_add(v_coarsel, v_load(px));
v_coarseh = v_add(v_coarseh, v_load(px + 8));
#else
for (int ind = 0; ind < 16; ++ind)
H.coarse[ind] += px[ind];
@@ -216,11 +216,11 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
px = h_coarse + 16 * (n*c + std::min(j + r, n - 1));
#if CV_SIMD256
v_coarse += v256_load(px);
v_coarse = v_add(v_coarse, v256_load(px));
v_store(H.coarse, v_coarse);
#elif CV_SIMD128
v_coarsel += v_load(px);
v_coarseh += v_load(px + 8);
v_coarsel = v_add(v_coarsel, v_load(px));
v_coarseh = v_add(v_coarseh, v_load(px + 8));
v_store(H.coarse, v_coarsel);
v_store(H.coarse + 8, v_coarseh);
#else
@@ -261,10 +261,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
for (luc[k] = HT(j - r); luc[k] < MIN(j + r + 1, n); ++luc[k], px += 16)
{
#if CV_SIMD256
v_fine += v256_load(px);
v_fine = v_add(v_fine, v256_load(px));
#elif CV_SIMD128
v_finel += v_load(px);
v_fineh += v_load(px + 8);
v_finel = v_add(v_finel, v_load(px));
v_fineh = v_add(v_fineh, v_load(px + 8));
#else
for (int ind = 0; ind < 16; ++ind)
H.fine[k][ind] += px[ind];
@@ -275,10 +275,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
{
px = h_fine + 16 * (n*(16 * c + k) + (n - 1));
#if CV_SIMD256
v_fine += v_mul_wrap(v256_load(px), v256_setall_u16(j + r + 1 - n));
v_fine = v_add(v_fine, v_mul_wrap(v256_load(px), v256_setall_u16(j + r + 1 - n)));
#elif CV_SIMD128
v_finel += v_mul_wrap(v_load(px), v_setall_u16((ushort)(j + r + 1 - n)));
v_fineh += v_mul_wrap(v_load(px + 8), v_setall_u16((ushort)(j + r + 1 - n)));
v_finel = v_add(v_finel, v_mul_wrap(v_load(px), v_setall_u16((ushort)(j + r + 1 - n))));
v_fineh = v_add(v_fineh, v_mul_wrap(v_load(px + 8), v_setall_u16((ushort)(j + r + 1 - n))));
#else
for (int ind = 0; ind < 16; ++ind)
H.fine[k][ind] = (HT)(H.fine[k][ind] + (j + r + 1 - n) * px[ind]);
@@ -298,10 +298,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
for ( ; luc[k] < j+r+1; ++luc[k] )
{
#if CV_SIMD256
v_fine = v_fine + v256_load(px + 16 * MIN(luc[k], n - 1)) - v256_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0));
v_fine = v_sub(v_add(v_fine, v256_load(px + 16 * MIN(luc[k], n - 1))), v256_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0)));
#elif CV_SIMD128
v_finel = v_finel + v_load(px + 16 * MIN(luc[k], n - 1) ) - v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0));
v_fineh = v_fineh + v_load(px + 16 * MIN(luc[k], n - 1) + 8) - v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0) + 8);
v_finel = v_sub(v_add(v_finel, v_load(px + 16 * MIN(luc[k], n - 1) )), v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0)));
v_fineh = v_sub(v_add(v_fineh, v_load(px + 16 * MIN(luc[k], n - 1) + 8)), v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0) + 8));
#else
for (int ind = 0; ind < 16; ++ind)
H.fine[k][ind] += px[16 * MIN(luc[k], n - 1) + ind] - px[16 * MAX(luc[k] - 2 * r - 1, 0) + ind];
@@ -312,12 +312,12 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize )
px = h_coarse + 16 * (n*c + MAX(j - r, 0));
#if CV_SIMD256
v_store(H.fine[k], v_fine);
v_coarse -= v256_load(px);
v_coarse = v_sub(v_coarse, v256_load(px));
#elif CV_SIMD128
v_store(H.fine[k], v_finel);
v_store(H.fine[k] + 8, v_fineh);
v_coarsel -= v_load(px);
v_coarseh -= v_load(px + 8);
v_coarsel = v_sub(v_coarsel, v_load(px));
v_coarseh = v_sub(v_coarseh, v_load(px + 8));
#else
for (int ind = 0; ind < 16; ++ind)
H.coarse[ind] -= px[ind];
+10 -10
View File
@@ -236,12 +236,12 @@ struct MomentsInTile_SIMD<uchar, int, int>
v_int16x8 p = v_reinterpret_as_s16(v_load_expand(ptr + x));
v_int16x8 sx = v_mul_wrap(qx, qx);
qx0 += v_reinterpret_as_u32(p);
qx0 = v_add(qx0, v_reinterpret_as_u32(p));
qx1 = v_reinterpret_as_u32(v_dotprod(p, qx, v_reinterpret_as_s32(qx1)));
qx2 = v_reinterpret_as_u32(v_dotprod(p, sx, v_reinterpret_as_s32(qx2)));
qx3 = v_reinterpret_as_u32(v_dotprod(v_mul_wrap(p, qx), sx, v_reinterpret_as_s32(qx3)));
qx += dx;
qx = v_add(qx, dx);
}
x0 = v_reduce_sum(qx0);
@@ -276,19 +276,19 @@ struct MomentsInTile_SIMD<ushort, int, int64>
{
v_int32x4 v_src = v_reinterpret_as_s32(v_load_expand(ptr + x));
v_x0 += v_reinterpret_as_u32(v_src);
v_x1 += v_reinterpret_as_u32(v_src * v_ix0);
v_x0 = v_add(v_x0, v_reinterpret_as_u32(v_src));
v_x1 = v_add(v_x1, v_reinterpret_as_u32(v_mul(v_src, v_ix0)));
v_int32x4 v_ix1 = v_ix0 * v_ix0;
v_x2 += v_reinterpret_as_u32(v_src * v_ix1);
v_int32x4 v_ix1 = v_mul(v_ix0, v_ix0);
v_x2 = v_add(v_x2, v_reinterpret_as_u32(v_mul(v_src, v_ix1)));
v_ix1 = v_ix0 * v_ix1;
v_src = v_src * v_ix1;
v_ix1 = v_mul(v_ix0, v_ix1);
v_src = v_mul(v_src, v_ix1);
v_uint64x2 v_lo, v_hi;
v_expand(v_reinterpret_as_u32(v_src), v_lo, v_hi);
v_x3 += v_lo + v_hi;
v_x3 = v_add(v_x3, v_add(v_lo, v_hi));
v_ix0 += v_delta;
v_ix0 = v_add(v_ix0, v_delta);
}
x0 = v_reduce_sum(v_x0);
+7 -7
View File
@@ -463,7 +463,7 @@ template<> int PyrDownVecV<int, uchar>(int** src, uchar* dst, int width)
}
#if CV_SIMD128
typedef int CV_DECL_ALIGNED(1) unaligned_int;
for ( ; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes)
for ( ; x <= width - VTraits<v_int32x4>::vlanes(); x += VTraits<v_int32x4>::vlanes())
{
v_int32x4 r0, r1, r2, r3, r4, t0;
r0 = v_load(row0 + x);
@@ -473,7 +473,7 @@ template<> int PyrDownVecV<int, uchar>(int** src, uchar* dst, int width)
r4 = v_load(row4 + x);
t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2)));
*((unaligned_int*) (dst + x)) = v_reinterpret_as_s32(v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16())).get0();
*((unaligned_int*) (dst + x)) = v_get0(v_reinterpret_as_s32(v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16())));
}
#else
for (; x <= width - 1; x += 1)
@@ -615,15 +615,15 @@ template <> int PyrUpVecV<int, uchar>(int** src, uchar** dst, int width)
}
#if CV_SIMD128
typedef int CV_DECL_ALIGNED(1) unaligned_int;
for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes)
for (; x <= width - VTraits<v_int32x4>::vlanes(); x += VTraits<v_int32x4>::vlanes())
{
v_int32 v_r00 = vx_load(row0 + x),
v_r10 = vx_load(row1 + x),
v_r20 = vx_load(row2 + x);
v_int32 v_2r10 = v_add(v_r10, v_r10);
v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20)));
*(unaligned_int*)(dst0 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0();
*(unaligned_int*)(dst1 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16())).get0();
*(unaligned_int*)(dst0 + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())));
*(unaligned_int*)(dst1 + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16())));
}
#else
for (; x <= width - 1; x += 1)
@@ -754,14 +754,14 @@ template <> int PyrUpVecVOneRow<int, uchar>(int** src, uchar* dst, int width)
}
#if CV_SIMD128
typedef int CV_DECL_ALIGNED(1) unaligned_int;
for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes)
for (; x <= width - VTraits<v_int32x4>::vlanes(); x += VTraits<v_int32x4>::vlanes())
{
v_int32 v_r00 = vx_load(row0 + x),
v_r10 = vx_load(row1 + x),
v_r20 = vx_load(row2 + x);
v_int32 v_2r10 = v_add(v_r10, v_r10);
v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20)));
*(unaligned_int*)(dst + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0();
*(unaligned_int*)(dst + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())));
}
#else
for (; x <= width - 1; x += 1)
+126 -31
View File
@@ -2473,7 +2473,7 @@ public:
v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5);
v_uint16 bl, gl, rl;
#if CV_SIMD_WIDTH == 16
bl = t0 + t3; gl = t1 + t4; rl = t2 + t5;
bl = v_add(t0, t3); gl = v_add(t1, t4); rl = v_add(t2, t5);
#elif CV_SIMD_WIDTH == 32
v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5);
bl = v_add(s0, s3); gl = v_add(s1, s4); rl = v_add(s2, s5);
@@ -2493,7 +2493,7 @@ public:
v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5);
v_uint16 bh, gh, rh;
#if CV_SIMD_WIDTH == 16
bh = t0 + t3; gh = t1 + t4; rh = t2 + t5;
bh = v_add(t0, t3); gh = v_add(t1, t4); rh = v_add(t2, t5);
#elif CV_SIMD_WIDTH == 32
v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5);
bh = v_add(s0, s3); gh = v_add(s1, s4); rh = v_add(s2, s5);
@@ -2566,7 +2566,7 @@ public:
v_rshr_pack_store<2>(D, r0 + v_rotate_left<1>(r1, r0));
}
#else
v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3));
v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_load_expand(S0), v_load_expand(S0 + 3)), v_load_expand(S1)), v_load_expand(S1 + 3)));
#endif
#elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64
for ( ; dx <= w - 3*VTraits<v_uint16>::vlanes(); dx += 3*VTraits<v_uint16>::vlanes(), S0 += 6*VTraits<v_uint16>::vlanes(), S1 += 6*VTraits<v_uint16>::vlanes(), D += 3*VTraits<v_uint16>::vlanes())
@@ -2609,7 +2609,7 @@ public:
}
#elif CV_SIMD_WIDTH >= 64
v_uint32 masklow = vx_setall_u32(0x0000ffff);
for ( ; dx <= w - 3*v_uint16::nlanes; dx += 3*v_uint16::nlanes, S0 += 6*v_uint16::nlanes, S1 += 6*v_uint16::nlanes, D += 3*v_uint16::nlanes)
for ( ; dx <= w - 3*VTraits<v_uint16>::vlanes(); dx += 3*VTraits<v_uint16>::vlanes(), S0 += 6*VTraits<v_uint16>::vlanes(), S1 += 6*VTraits<v_uint16>::vlanes(), D += 3*VTraits<v_uint16>::vlanes())
{
v_uint16 b0, g0, r0, b1, g1, r1;
v_load_deinterleave(S0, b0, g0, r0);
@@ -2617,8 +2617,8 @@ public:
v_uint32 bl = (v_reinterpret_as_u32(b0) >> 16) + (v_reinterpret_as_u32(b0) & masklow) + (v_reinterpret_as_u32(b1) >> 16) + (v_reinterpret_as_u32(b1) & masklow);
v_uint32 gl = (v_reinterpret_as_u32(g0) >> 16) + (v_reinterpret_as_u32(g0) & masklow) + (v_reinterpret_as_u32(g1) >> 16) + (v_reinterpret_as_u32(g1) & masklow);
v_uint32 rl = (v_reinterpret_as_u32(r0) >> 16) + (v_reinterpret_as_u32(r0) & masklow) + (v_reinterpret_as_u32(r1) >> 16) + (v_reinterpret_as_u32(r1) & masklow);
v_load_deinterleave(S0 + 3*v_uint16::nlanes, b0, g0, r0);
v_load_deinterleave(S1 + 3*v_uint16::nlanes, b1, g1, r1);
v_load_deinterleave(S0 + 3*VTraits<v_uint16>::vlanes(), b0, g0, r0);
v_load_deinterleave(S1 + 3*VTraits<v_uint16>::vlanes(), b1, g1, r1);
v_uint32 bh = (v_reinterpret_as_u32(b0) >> 16) + (v_reinterpret_as_u32(b0) & masklow) + (v_reinterpret_as_u32(b1) >> 16) + (v_reinterpret_as_u32(b1) & masklow);
v_uint32 gh = (v_reinterpret_as_u32(g0) >> 16) + (v_reinterpret_as_u32(g0) & masklow) + (v_reinterpret_as_u32(g1) >> 16) + (v_reinterpret_as_u32(g1) & masklow);
v_uint32 rh = (v_reinterpret_as_u32(r0) >> 16) + (v_reinterpret_as_u32(r0) & masklow) + (v_reinterpret_as_u32(r1) >> 16) + (v_reinterpret_as_u32(r1) & masklow);
@@ -2630,7 +2630,7 @@ public:
{
CV_Assert(cn == 4);
#if CV_SIMD_WIDTH >= 64
for ( ; dx <= w - v_uint16::nlanes; dx += v_uint16::nlanes, S0 += 2*v_uint16::nlanes, S1 += 2*v_uint16::nlanes, D += v_uint16::nlanes)
for ( ; dx <= w - VTraits<v_uint16>::vlanes(); dx += VTraits<v_uint16>::vlanes(), S0 += 2*VTraits<v_uint16>::vlanes(), S1 += 2*VTraits<v_uint16>::vlanes(), D += VTraits<v_uint16>::vlanes())
{
v_uint64 r00, r01, r10, r11;
v_load_deinterleave((uint64_t*)S0, r00, r01);
@@ -2652,7 +2652,7 @@ public:
r0 = v_add(r0, r2); r1 = v_add(r1, r3);
v_uint32 v_d;
#if CV_SIMD_WIDTH == 16
v_d = r0 + r1;
v_d = v_add(r0, r1);
#elif CV_SIMD_WIDTH == 32
v_uint32 t0, t1;
v_recombine(r0, r1, t0, t1);
@@ -2697,7 +2697,7 @@ public:
{
#if CV_SIMD_WIDTH == 16
for ( ; dx <= w - 4; dx += 3, S0 += 6, S1 += 6, D += 3)
v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3));
v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_load_expand(S0), v_load_expand(S0 + 3)), v_load_expand(S1)), v_load_expand(S1 + 3)));
#elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64
for ( ; dx <= w - 3*VTraits<v_int16>::vlanes(); dx += 3*VTraits<v_int16>::vlanes(), S0 += 6*VTraits<v_int16>::vlanes(), S1 += 6*VTraits<v_int16>::vlanes(), D += 3*VTraits<v_int16>::vlanes())
{
@@ -2738,7 +2738,7 @@ public:
v_store_interleave(D, v_rshr_pack<2>(bl, bh), v_rshr_pack<2>(gl, gh), v_rshr_pack<2>(rl, rh));
}
#elif CV_SIMD_WIDTH >= 64
for ( ; dx <= w - 3*v_int16::nlanes; dx += 3*v_int16::nlanes, S0 += 6*v_int16::nlanes, S1 += 6*v_int16::nlanes, D += 3*v_int16::nlanes)
for ( ; dx <= w - 3*VTraits<v_int16>::vlanes(); dx += 3*VTraits<v_int16>::vlanes(), S0 += 6*VTraits<v_int16>::vlanes(), S1 += 6*VTraits<v_int16>::vlanes(), D += 3*VTraits<v_int16>::vlanes())
{
v_int16 b0, g0, r0, b1, g1, r1;
v_load_deinterleave(S0, b0, g0, r0);
@@ -2746,8 +2746,8 @@ public:
v_int32 bl = (v_reinterpret_as_s32(b0) >> 16) + ((v_reinterpret_as_s32(b0) << 16) >> 16) + (v_reinterpret_as_s32(b1) >> 16) + ((v_reinterpret_as_s32(b1) << 16) >> 16);
v_int32 gl = (v_reinterpret_as_s32(g0) >> 16) + ((v_reinterpret_as_s32(g0) << 16) >> 16) + (v_reinterpret_as_s32(g1) >> 16) + ((v_reinterpret_as_s32(g1) << 16) >> 16);
v_int32 rl = (v_reinterpret_as_s32(r0) >> 16) + ((v_reinterpret_as_s32(r0) << 16) >> 16) + (v_reinterpret_as_s32(r1) >> 16) + ((v_reinterpret_as_s32(r1) << 16) >> 16);
v_load_deinterleave(S0 + 3*v_int16::nlanes, b0, g0, r0);
v_load_deinterleave(S1 + 3*v_int16::nlanes, b1, g1, r1);
v_load_deinterleave(S0 + 3*VTraits<v_int16>::vlanes(), b0, g0, r0);
v_load_deinterleave(S1 + 3*VTraits<v_int16>::vlanes(), b1, g1, r1);
v_int32 bh = (v_reinterpret_as_s32(b0) >> 16) + ((v_reinterpret_as_s32(b0) << 16) >> 16) + (v_reinterpret_as_s32(b1) >> 16) + ((v_reinterpret_as_s32(b1) << 16) >> 16);
v_int32 gh = (v_reinterpret_as_s32(g0) >> 16) + ((v_reinterpret_as_s32(g0) << 16) >> 16) + (v_reinterpret_as_s32(g1) >> 16) + ((v_reinterpret_as_s32(g1) << 16) >> 16);
v_int32 rh = (v_reinterpret_as_s32(r0) >> 16) + ((v_reinterpret_as_s32(r0) << 16) >> 16) + (v_reinterpret_as_s32(r1) >> 16) + ((v_reinterpret_as_s32(r1) << 16) >> 16);
@@ -2779,7 +2779,7 @@ public:
r3 = v_add(vx_load_expand(S0 + 3 * VTraits<v_int32>::vlanes()), vx_load_expand(S1 + 3 * VTraits<v_int32>::vlanes()));
v_int32 dl, dh;
#if CV_SIMD_WIDTH == 16
dl = r0 + r1; dh = r2 + r3;
dl = v_add(r0, r1); dh = v_add(r2, r3);
#elif CV_SIMD_WIDTH == 32
v_int32 t0, t1, t2, t3;
v_recombine(r0, r1, t0, t1); v_recombine(r2, r3, t2, t3);
@@ -2829,14 +2829,14 @@ struct ResizeAreaFastVec_SIMD_32f
{
#if CV_SIMD_WIDTH == 16
v_float32 v_025 = vx_setall_f32(0.25f);
for (; dx <= w - v_float32::nlanes; dx += v_float32::nlanes, S0 += 2*v_float32::nlanes, S1 += 2*v_float32::nlanes, D += v_float32::nlanes)
v_store(D, ((vx_load(S0) + vx_load(S0 + v_float32::nlanes)) + (vx_load(S1) + vx_load(S1 + v_float32::nlanes))) * v_025);
for (; dx <= w - VTraits<v_float32>::vlanes(); dx += VTraits<v_float32>::vlanes(), S0 += 2*VTraits<v_float32>::vlanes(), S1 += 2*VTraits<v_float32>::vlanes(), D += VTraits<v_float32>::vlanes())
v_store(D, v_mul(v_add(v_add(vx_load(S0), vx_load(S0 + VTraits<v_float32>::vlanes())), v_add(vx_load(S1), vx_load(S1 + VTraits<v_float32>::vlanes()))), v_025));
#elif CV_SIMD256
v_float32x8 v_025 = v256_setall_f32(0.25f);
for (; dx <= w - v_float32x8::nlanes; dx += v_float32x8::nlanes, S0 += 2*v_float32x8::nlanes, S1 += 2*v_float32x8::nlanes, D += v_float32x8::nlanes)
for (; dx <= w - VTraits<v_float32x8>::vlanes(); dx += VTraits<v_float32x8>::vlanes(), S0 += 2*VTraits<v_float32x8>::vlanes(), S1 += 2*VTraits<v_float32x8>::vlanes(), D += VTraits<v_float32x8>::vlanes())
{
v_float32x8 dst0, dst1;
v_recombine(v_add(v256_load(S0), v256_load(S1)), v_add(v256_load(S0 + v_float32x8::nlanes), v256_load(S1 + v_float32x8::nlanes)), dst0, dst1);
v_recombine(v_add(v256_load(S0), v256_load(S1)), v_add(v256_load(S0 + VTraits<v_float32x8>::vlanes()), v256_load(S1 + VTraits<v_float32x8>::vlanes())), dst0, dst1);
v_store(D, v_mul(v_add(dst0, dst1), v_025));
}
#endif
@@ -3019,6 +3019,111 @@ struct DecimateAlpha
};
namespace inter_area {
#if (CV_SIMD || CV_SIMD_SCALABLE)
inline void saturate_store(const float* src, uchar* dst) {
const v_int32 tmp0 = v_round(vx_load(src + 0 * VTraits<v_float32>::vlanes()));
const v_int32 tmp1 = v_round(vx_load(src + 1 * VTraits<v_float32>::vlanes()));
const v_int32 tmp2 = v_round(vx_load(src + 2 * VTraits<v_float32>::vlanes()));
const v_int32 tmp3 = v_round(vx_load(src + 3 * VTraits<v_float32>::vlanes()));
v_store(dst, v_pack(v_pack_u(tmp0, tmp1), v_pack_u(tmp2, tmp3)));
}
inline void saturate_store(const float* src, ushort* dst) {
const v_int32 tmp0 = v_round(vx_load(src + 0 * VTraits<v_float32>::vlanes()));
const v_int32 tmp1 = v_round(vx_load(src + 1 * VTraits<v_float32>::vlanes()));
v_store(dst, v_pack_u(tmp0, tmp1));
}
inline void saturate_store(const float* src, short* dst) {
const v_int32 tmp0 = v_round(vx_load(src + 0 * VTraits<v_float32>::vlanes()));
const v_int32 tmp1 = v_round(vx_load(src + 1 * VTraits<v_float32>::vlanes()));
v_store(dst, v_pack(tmp0, tmp1));
}
static inline v_float32 vx_setall(float coeff) { return vx_setall_f32(coeff); }
template <typename T>
struct VArea {};
template <>
struct VArea<float> {
typedef v_float32 vWT;
};
#endif
#if (CV_SIMD128_64F || CV_SIMD_SCALABLE_64F)
static inline v_float64 vx_setall(double coeff) { return vx_setall_f64(coeff); }
template <>
struct VArea<double> {
typedef v_float64 vWT;
};
#else
inline void mul(const double* buf, int width, double beta, double* sum) {
for (int dx = 0; dx < width; ++dx) {
sum[dx] = beta * buf[dx];
}
}
inline void muladd(const double* buf, int width, double beta, double* sum) {
for (int dx = 0; dx < width; ++dx) {
sum[dx] += beta * buf[dx];
}
}
#endif
template <typename T, typename WT>
inline void saturate_store(const WT* sum, int width, T* D) {
int dx = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int step = VTraits<typename VArea<WT>::vWT>::vlanes() * sizeof(WT) / sizeof(T);
for (; dx + step < width; dx += step) {
saturate_store(sum + dx, D + dx);
}
#endif
for (; dx < width; ++dx) {
D[dx] = saturate_cast<T>(sum[dx]);
}
}
// Optimization when T == WT.
template <typename WT>
inline void saturate_store(const WT* sum, int width, WT* D) {
std::copy(sum, sum + width, D);
}
template <typename WT>
inline void mul(const WT* buf, int width, WT beta, WT* sum) {
int dx = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int step = VTraits<typename VArea<WT>::vWT>::vlanes();
for (; dx + step < width; dx += step) {
vx_store(sum + dx, v_mul(vx_setall(beta), vx_load(buf + dx)));
}
#endif
for (; dx < width; ++dx) {
sum[dx] = beta * buf[dx];
}
}
template <typename WT>
inline void muladd(const WT* buf, int width, WT beta, WT* sum) {
int dx = 0;
#if (CV_SIMD || CV_SIMD_SCALABLE)
const int step = VTraits<typename VArea<WT>::vWT>::vlanes();
for (; dx + step < width; dx += step) {
vx_store(sum + dx, v_add(vx_load(sum + dx), v_mul(vx_setall(beta), vx_load(buf + dx))));
}
#endif
for (; dx < width; ++dx) {
sum[dx] += beta * buf[dx];
}
}
} // namespace inter_area
template<typename T, typename WT> class ResizeArea_Invoker :
public ParallelLoopBody
{
@@ -3120,27 +3225,17 @@ public:
if( dy != prev_dy )
{
T* D = dst->template ptr<T>(prev_dy);
for( dx = 0; dx < dsize.width; dx++ )
{
D[dx] = saturate_cast<T>(sum[dx]);
sum[dx] = beta*buf[dx];
}
inter_area::saturate_store(sum, dsize.width, dst->template ptr<T>(prev_dy));
inter_area::mul(buf, dsize.width, beta, sum);
prev_dy = dy;
}
else
{
for( dx = 0; dx < dsize.width; dx++ )
sum[dx] += beta*buf[dx];
inter_area::muladd(buf, dsize.width, beta, sum);
}
}
{
T* D = dst->template ptr<T>(prev_dy);
for( dx = 0; dx < dsize.width; dx++ )
D[dx] = saturate_cast<T>(sum[dx]);
}
inter_area::saturate_store(sum, dsize.width, dst->template ptr<T>(prev_dy));
}
private:
+234 -234
View File
@@ -114,7 +114,7 @@ struct Integral_SIMD<uchar, int, double>
v_int32 prev = vx_setzero_s32();
int j = 0;
for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for ( ; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_int32 el4l, el4h;
@@ -127,8 +127,8 @@ struct Integral_SIMD<uchar, int, double>
el4h.val = _mm256_add_epi32(_mm256_cvtepi16_epi32(_v256_extract_high(vsum)), _mm256_permutevar8x32_epi32(el4l.val, shmask));
prev.val = _mm256_permutevar8x32_epi32(el4h.val, shmask);
#else
el8 += v_rotate_left<1>(el8);
el8 += v_rotate_left<2>(el8);
el8 = v_add(el8, v_rotate_left<1>(el8));
el8 = v_add(el8, v_rotate_left<2>(el8));
#if CV_SIMD_WIDTH >= 32
el8 += v_rotate_left<4>(el8);
#if CV_SIMD_WIDTH == 64
@@ -136,12 +136,12 @@ struct Integral_SIMD<uchar, int, double>
#endif
#endif
v_expand(el8, el4l, el4h);
el4l += prev;
el4h += el4l;
prev = v_broadcast_element<v_int32::nlanes - 1>(el4h);
el4l = v_add(el4l, prev);
el4h = v_add(el4h, el4l);
prev = v_broadcast_highest(el4h);
#endif
v_store(sum_row + j , el4l + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_int32::nlanes, el4h + vx_load(prev_sum_row + j + v_int32::nlanes));
v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_int32>::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes())));
}
for (int v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j)
@@ -162,11 +162,11 @@ struct Integral_SIMD<uchar, int, double>
v_int32 prev_1 = vx_setzero_s32(), prev_2 = vx_setzero_s32();
int j = 0;
for ( ; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn)
for ( ; j + VTraits<v_uint16>::vlanes() * cn <= width; j += VTraits<v_uint16>::vlanes() * cn)
{
v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j));
v_int16 el8_1 = v_src_row & mask;
v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8);
v_int16 el8_1 = v_and(v_src_row, mask);
v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row)));
v_int32 el4l_1, el4h_1, el4l_2, el4h_2;
#if CV_AVX2 && CV_SIMD_WIDTH == 32
__m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2));
@@ -183,10 +183,10 @@ struct Integral_SIMD<uchar, int, double>
prev_1.val = _mm256_permutevar8x32_epi32(el4h_1.val, shmask);
prev_2.val = _mm256_permutevar8x32_epi32(el4h_2.val, shmask);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -197,20 +197,20 @@ struct Integral_SIMD<uchar, int, double>
#endif
v_expand(el8_1, el4l_1, el4h_1);
v_expand(el8_2, el4l_2, el4h_2);
el4l_1 += prev_1;
el4l_2 += prev_2;
el4h_1 += el4l_1;
el4h_2 += el4l_2;
prev_1 = v_broadcast_element<v_int32::nlanes - 1>(el4h_1);
prev_2 = v_broadcast_element<v_int32::nlanes - 1>(el4h_2);
el4l_1 = v_add(el4l_1, prev_1);
el4l_2 = v_add(el4l_2, prev_2);
el4h_1 = v_add(el4h_1, el4l_1);
el4h_2 = v_add(el4h_2, el4l_2);
prev_1 = v_broadcast_highest(el4h_1);
prev_2 = v_broadcast_highest(el4h_2);
#endif
v_int32 el4_1, el4_2, el4_3, el4_4;
v_zip(el4l_1, el4l_2, el4_1, el4_2);
v_zip(el4h_1, el4h_2, el4_3, el4_4);
v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_int32::nlanes , el4_2 + vx_load(prev_sum_row + j + v_int32::nlanes ));
v_store(sum_row + j + v_int32::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 2));
v_store(sum_row + j + v_int32::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_int32::nlanes * 3));
v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes())));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 3)));
}
for (int v2 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -230,7 +230,7 @@ struct Integral_SIMD<uchar, int, double>
const uchar * src_row = src + _srcstep * i;
int * prev_sum_row = (int *)((uchar *)sum + _sumstep * i) + cn;
int * sum_row = (int *)((uchar *)sum + _sumstep * (i + 1)) + cn;
int row_cache[v_int32::nlanes * 6];
int row_cache[VTraits<v_int32>::max_nlanes * 6];
sum_row[-1] = sum_row[-2] = sum_row[-3] = 0;
@@ -238,10 +238,10 @@ struct Integral_SIMD<uchar, int, double>
prev_3 = vx_setzero_s32();
int j = 0;
const int j_max =
((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height)
? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave()
: width - v_uint16::nlanes * cn; // v_expand_low
for ( ; j <= j_max; j += v_uint16::nlanes * cn)
((_srcstep * i + (width - VTraits<v_uint16>::vlanes() * cn + VTraits<v_uint8>::vlanes() * cn)) >= _srcstep * height)
? width - VTraits<v_uint8>::vlanes() * cn // uint8 in v_load_deinterleave()
: width - VTraits<v_uint16>::vlanes() * cn; // v_expand_low
for ( ; j <= j_max; j += VTraits<v_uint16>::vlanes() * cn)
{
v_uint8 v_src_row_1, v_src_row_2, v_src_row_3;
v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3);
@@ -270,49 +270,49 @@ struct Integral_SIMD<uchar, int, double>
prev_2.val = _mm256_permutevar8x32_epi32(el4h_2.val, shmask);
prev_3.val = _mm256_permutevar8x32_epi32(el4h_3.val, shmask);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_3 += v_rotate_left<1>(el8_3);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_3 += v_rotate_left<2>(el8_3);
el8_1 = v_add(el8_1,v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2,v_rotate_left<1>(el8_2));
el8_3 = v_add(el8_3,v_rotate_left<1>(el8_3));
el8_1 = v_add(el8_1,v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2,v_rotate_left<2>(el8_2));
el8_3 = v_add(el8_3,v_rotate_left<2>(el8_3));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
el8_3 += v_rotate_left<4>(el8_3);
el8_1 = v_add(el8_1, v_rotate_left<4>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<4>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<4>(el8_3));
#if CV_SIMD_WIDTH == 64
el8_1 += v_rotate_left<8>(el8_1);
el8_2 += v_rotate_left<8>(el8_2);
el8_3 += v_rotate_left<8>(el8_3);
el8_1 = v_add(el8_1, v_rotate_left<8>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<8>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<8>(el8_3));
#endif
#endif
v_expand(el8_1, el4l_1, el4h_1);
v_expand(el8_2, el4l_2, el4h_2);
v_expand(el8_3, el4l_3, el4h_3);
el4l_1 += prev_1;
el4l_2 += prev_2;
el4l_3 += prev_3;
el4h_1 += el4l_1;
el4h_2 += el4l_2;
el4h_3 += el4l_3;
prev_1 = v_broadcast_element<v_int32::nlanes - 1>(el4h_1);
prev_2 = v_broadcast_element<v_int32::nlanes - 1>(el4h_2);
prev_3 = v_broadcast_element<v_int32::nlanes - 1>(el4h_3);
el4l_1 = v_add(el4l_1, prev_1);
el4l_2 = v_add(el4l_2, prev_2);
el4l_3 = v_add(el4l_3, prev_3);
el4h_1 = v_add(el4h_1, el4l_1);
el4h_2 = v_add(el4h_2, el4l_2);
el4h_3 = v_add(el4h_3, el4l_3);
prev_1 = v_broadcast_highest(el4h_1);
prev_2 = v_broadcast_highest(el4h_2);
prev_3 = v_broadcast_highest(el4h_3);
#endif
v_store_interleave(row_cache , el4l_1, el4l_2, el4l_3);
v_store_interleave(row_cache + v_int32::nlanes * 3, el4h_1, el4h_2, el4h_3);
v_store_interleave(row_cache + VTraits<v_int32>::vlanes() * 3, el4h_1, el4h_2, el4h_3);
el4l_1 = vx_load(row_cache );
el4l_2 = vx_load(row_cache + v_int32::nlanes );
el4l_3 = vx_load(row_cache + v_int32::nlanes * 2);
el4h_1 = vx_load(row_cache + v_int32::nlanes * 3);
el4h_2 = vx_load(row_cache + v_int32::nlanes * 4);
el4h_3 = vx_load(row_cache + v_int32::nlanes * 5);
v_store(sum_row + j , el4l_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_int32::nlanes , el4l_2 + vx_load(prev_sum_row + j + v_int32::nlanes ));
v_store(sum_row + j + v_int32::nlanes * 2, el4l_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 2));
v_store(sum_row + j + v_int32::nlanes * 3, el4h_1 + vx_load(prev_sum_row + j + v_int32::nlanes * 3));
v_store(sum_row + j + v_int32::nlanes * 4, el4h_2 + vx_load(prev_sum_row + j + v_int32::nlanes * 4));
v_store(sum_row + j + v_int32::nlanes * 5, el4h_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 5));
el4l_2 = vx_load(row_cache + VTraits<v_int32>::vlanes() );
el4l_3 = vx_load(row_cache + VTraits<v_int32>::vlanes() * 2);
el4h_1 = vx_load(row_cache + VTraits<v_int32>::vlanes() * 3);
el4h_2 = vx_load(row_cache + VTraits<v_int32>::vlanes() * 4);
el4h_3 = vx_load(row_cache + VTraits<v_int32>::vlanes() * 5);
v_store(sum_row + j , v_add(el4l_1, vx_load(prev_sum_row + j )));
v_store(sum_row + j + VTraits<v_int32>::vlanes() , v_add(el4l_2, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() )));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 2, v_add(el4l_3, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 3, v_add(el4h_1, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 3)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 4, v_add(el4h_2, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 4)));
v_store(sum_row + j + VTraits<v_int32>::vlanes() * 5, v_add(el4h_3, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes() * 5)));
}
for (int v3 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -339,7 +339,7 @@ struct Integral_SIMD<uchar, int, double>
v_int32 prev = vx_setzero_s32();
int j = 0;
for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for ( ; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_int32 el4l, el4h;
@@ -356,8 +356,8 @@ struct Integral_SIMD<uchar, int, double>
#endif
#endif
v_expand(el8, el4l, el4h);
el4l += prev;
el4h += el4l;
el4l = v_add(el4l, prev);
el4h = v_add(el4h, el4l);
#if CV_SIMD_WIDTH == 16
prev = el4h;
#elif CV_SIMD_WIDTH == 32
@@ -368,8 +368,8 @@ struct Integral_SIMD<uchar, int, double>
prev = v_combine_low(t, t);
#endif
#endif
v_store(sum_row + j , el4l + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_int32::nlanes, el4h + vx_load(prev_sum_row + j + v_int32::nlanes));
v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_int32>::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits<v_int32>::vlanes())));
}
for (int v4 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -426,7 +426,7 @@ struct Integral_SIMD<uchar, float, double>
v_float32 prev = vx_setzero_f32();
int j = 0;
for (; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for (; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_float32 el4l, el4h;
@@ -439,8 +439,8 @@ struct Integral_SIMD<uchar, float, double>
el4h.val = _mm256_add_ps(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_v256_extract_high(vsum))), _mm256_permutevar8x32_ps(el4l.val, shmask));
prev.val = _mm256_permutevar8x32_ps(el4h.val, shmask);
#else
el8 += v_rotate_left<1>(el8);
el8 += v_rotate_left<2>(el8);
el8 = v_add(el8, v_rotate_left<1>(el8));
el8 = v_add(el8, v_rotate_left<2>(el8));
#if CV_SIMD_WIDTH >= 32
el8 += v_rotate_left<4>(el8);
#if CV_SIMD_WIDTH == 64
@@ -449,12 +449,12 @@ struct Integral_SIMD<uchar, float, double>
#endif
v_int32 el4li, el4hi;
v_expand(el8, el4li, el4hi);
el4l = v_cvt_f32(el4li) + prev;
el4h = v_cvt_f32(el4hi) + el4l;
prev = v_broadcast_element<v_float32::nlanes - 1>(el4h);
el4l = v_add(v_cvt_f32(el4li), prev);
el4h = v_add(v_cvt_f32(el4hi), el4l);
prev = v_broadcast_highest(el4h);
#endif
v_store(sum_row + j , el4l + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float32::nlanes, el4h + vx_load(prev_sum_row + j + v_float32::nlanes));
v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float32>::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes())));
}
for (float v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j)
@@ -475,11 +475,11 @@ struct Integral_SIMD<uchar, float, double>
v_float32 prev_1 = vx_setzero_f32(), prev_2 = vx_setzero_f32();
int j = 0;
for (; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn)
for (; j + VTraits<v_uint16>::vlanes() * cn <= width; j += VTraits<v_uint16>::vlanes() * cn)
{
v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j));
v_int16 el8_1 = v_src_row & mask;
v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8);
v_int16 el8_1 = v_and(v_src_row, mask);
v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row)));
v_float32 el4l_1, el4h_1, el4l_2, el4h_2;
#if CV_AVX2 && CV_SIMD_WIDTH == 32
__m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2));
@@ -496,10 +496,10 @@ struct Integral_SIMD<uchar, float, double>
prev_1.val = _mm256_permutevar8x32_ps(el4h_1.val, shmask);
prev_2.val = _mm256_permutevar8x32_ps(el4h_2.val, shmask);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -511,20 +511,20 @@ struct Integral_SIMD<uchar, float, double>
v_int32 el4li_1, el4hi_1, el4li_2, el4hi_2;
v_expand(el8_1, el4li_1, el4hi_1);
v_expand(el8_2, el4li_2, el4hi_2);
el4l_1 = v_cvt_f32(el4li_1) + prev_1;
el4l_2 = v_cvt_f32(el4li_2) + prev_2;
el4h_1 = v_cvt_f32(el4hi_1) + el4l_1;
el4h_2 = v_cvt_f32(el4hi_2) + el4l_2;
prev_1 = v_broadcast_element<v_float32::nlanes - 1>(el4h_1);
prev_2 = v_broadcast_element<v_float32::nlanes - 1>(el4h_2);
el4l_1 = v_add(v_cvt_f32(el4li_1), prev_1);
el4l_2 = v_add(v_cvt_f32(el4li_2), prev_2);
el4h_1 = v_add(v_cvt_f32(el4hi_1), el4l_1);
el4h_2 = v_add(v_cvt_f32(el4hi_2), el4l_2);
prev_1 = v_broadcast_highest(el4h_1);
prev_2 = v_broadcast_highest(el4h_2);
#endif
v_float32 el4_1, el4_2, el4_3, el4_4;
v_zip(el4l_1, el4l_2, el4_1, el4_2);
v_zip(el4h_1, el4h_2, el4_3, el4_4);
v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float32::nlanes , el4_2 + vx_load(prev_sum_row + j + v_float32::nlanes ));
v_store(sum_row + j + v_float32::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 2));
v_store(sum_row + j + v_float32::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_float32::nlanes * 3));
v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes())));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 3)));
}
for (float v2 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -543,7 +543,7 @@ struct Integral_SIMD<uchar, float, double>
const uchar * src_row = src + _srcstep * i;
float * prev_sum_row = (float *)((uchar *)sum + _sumstep * i) + cn;
float * sum_row = (float *)((uchar *)sum + _sumstep * (i + 1)) + cn;
float row_cache[v_float32::nlanes * 6];
float row_cache[VTraits<v_float32>::max_nlanes * 6];
sum_row[-1] = sum_row[-2] = sum_row[-3] = 0;
@@ -551,10 +551,10 @@ struct Integral_SIMD<uchar, float, double>
prev_3 = vx_setzero_f32();
int j = 0;
const int j_max =
((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height)
? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave()
: width - v_uint16::nlanes * cn; // v_expand_low
for ( ; j <= j_max; j += v_uint16::nlanes * cn)
((_srcstep * i + (width - VTraits<v_uint16>::vlanes() * cn + VTraits<v_uint8>::vlanes() * cn)) >= _srcstep * height)
? width - VTraits<v_uint8>::vlanes() * cn // uint8 in v_load_deinterleave()
: width - VTraits<v_uint16>::vlanes() * cn; // v_expand_low
for ( ; j <= j_max; j += VTraits<v_uint16>::vlanes() * cn)
{
v_uint8 v_src_row_1, v_src_row_2, v_src_row_3;
v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3);
@@ -583,12 +583,12 @@ struct Integral_SIMD<uchar, float, double>
prev_2.val = _mm256_permutevar8x32_ps(el4h_2.val, shmask);
prev_3.val = _mm256_permutevar8x32_ps(el4h_3.val, shmask);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_3 += v_rotate_left<1>(el8_3);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_3 += v_rotate_left<2>(el8_3);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<1>(el8_3));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<2>(el8_3));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -603,30 +603,30 @@ struct Integral_SIMD<uchar, float, double>
v_expand(el8_1, el4li_1, el4hi_1);
v_expand(el8_2, el4li_2, el4hi_2);
v_expand(el8_3, el4li_3, el4hi_3);
el4l_1 = v_cvt_f32(el4li_1) + prev_1;
el4l_2 = v_cvt_f32(el4li_2) + prev_2;
el4l_3 = v_cvt_f32(el4li_3) + prev_3;
el4h_1 = v_cvt_f32(el4hi_1) + el4l_1;
el4h_2 = v_cvt_f32(el4hi_2) + el4l_2;
el4h_3 = v_cvt_f32(el4hi_3) + el4l_3;
prev_1 = v_broadcast_element<v_float32::nlanes - 1>(el4h_1);
prev_2 = v_broadcast_element<v_float32::nlanes - 1>(el4h_2);
prev_3 = v_broadcast_element<v_float32::nlanes - 1>(el4h_3);
el4l_1 = v_add(v_cvt_f32(el4li_1), prev_1);
el4l_2 = v_add(v_cvt_f32(el4li_2), prev_2);
el4l_3 = v_add(v_cvt_f32(el4li_3), prev_3);
el4h_1 = v_add(v_cvt_f32(el4hi_1), el4l_1);
el4h_2 = v_add(v_cvt_f32(el4hi_2), el4l_2);
el4h_3 = v_add(v_cvt_f32(el4hi_3), el4l_3);
prev_1 = v_broadcast_highest(el4h_1);
prev_2 = v_broadcast_highest(el4h_2);
prev_3 = v_broadcast_highest(el4h_3);
#endif
v_store_interleave(row_cache , el4l_1, el4l_2, el4l_3);
v_store_interleave(row_cache + v_float32::nlanes * 3, el4h_1, el4h_2, el4h_3);
v_store_interleave(row_cache + VTraits<v_float32>::vlanes() * 3, el4h_1, el4h_2, el4h_3);
el4l_1 = vx_load(row_cache );
el4l_2 = vx_load(row_cache + v_float32::nlanes );
el4l_3 = vx_load(row_cache + v_float32::nlanes * 2);
el4h_1 = vx_load(row_cache + v_float32::nlanes * 3);
el4h_2 = vx_load(row_cache + v_float32::nlanes * 4);
el4h_3 = vx_load(row_cache + v_float32::nlanes * 5);
v_store(sum_row + j , el4l_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float32::nlanes , el4l_2 + vx_load(prev_sum_row + j + v_float32::nlanes ));
v_store(sum_row + j + v_float32::nlanes * 2, el4l_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 2));
v_store(sum_row + j + v_float32::nlanes * 3, el4h_1 + vx_load(prev_sum_row + j + v_float32::nlanes * 3));
v_store(sum_row + j + v_float32::nlanes * 4, el4h_2 + vx_load(prev_sum_row + j + v_float32::nlanes * 4));
v_store(sum_row + j + v_float32::nlanes * 5, el4h_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 5));
el4l_2 = vx_load(row_cache + VTraits<v_float32>::vlanes() );
el4l_3 = vx_load(row_cache + VTraits<v_float32>::vlanes() * 2);
el4h_1 = vx_load(row_cache + VTraits<v_float32>::vlanes() * 3);
el4h_2 = vx_load(row_cache + VTraits<v_float32>::vlanes() * 4);
el4h_3 = vx_load(row_cache + VTraits<v_float32>::vlanes() * 5);
v_store(sum_row + j , v_add(el4l_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() , v_add(el4l_2, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes())));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 2, v_add(el4l_3, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 3, v_add(el4h_1, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 3)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 4, v_add(el4h_2, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 4)));
v_store(sum_row + j + VTraits<v_float32>::vlanes() * 5, v_add(el4h_3, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes() * 5)));
}
for (float v3 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -652,7 +652,7 @@ struct Integral_SIMD<uchar, float, double>
v_float32 prev = vx_setzero_f32();
int j = 0;
for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for ( ; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_float32 el4l, el4h;
@@ -670,8 +670,8 @@ struct Integral_SIMD<uchar, float, double>
#endif
v_int32 el4li, el4hi;
v_expand(el8, el4li, el4hi);
el4l = v_cvt_f32(el4li) + prev;
el4h = v_cvt_f32(el4hi) + el4l;
el4l = v_add(v_cvt_f32(el4li), prev);
el4h = v_add(v_cvt_f32(el4hi), el4l);
#if CV_SIMD_WIDTH == 16
prev = el4h;
#elif CV_SIMD_WIDTH == 32
@@ -682,8 +682,8 @@ struct Integral_SIMD<uchar, float, double>
prev = v_combine_low(t, t);
#endif
#endif
v_store(sum_row + j , el4l + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float32::nlanes, el4h + vx_load(prev_sum_row + j + v_float32::nlanes));
v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float32>::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits<v_float32>::vlanes())));
}
for (float v4 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -750,7 +750,7 @@ struct Integral_SIMD<uchar, double, double>
v_float64 prev = vx_setzero_f64();
int j = 0;
for (; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for (; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_float64 el4ll, el4lh, el4hl, el4hh;
@@ -767,8 +767,8 @@ struct Integral_SIMD<uchar, double, double>
el4hh.val = _mm256_add_pd(_mm256_cvtepi32_pd(_v256_extract_high(el4h_32)), el4d);
prev.val = _mm256_permute4x64_pd(el4hh.val, 0xff);
#else
el8 += v_rotate_left<1>(el8);
el8 += v_rotate_left<2>(el8);
el8 = v_add(el8, v_rotate_left<1>(el8));
el8 = v_add(el8, v_rotate_left<2>(el8));
#if CV_SIMD_WIDTH >= 32
el8 += v_rotate_left<4>(el8);
#if CV_SIMD_WIDTH == 64
@@ -777,17 +777,17 @@ struct Integral_SIMD<uchar, double, double>
#endif
v_int32 el4li, el4hi;
v_expand(el8, el4li, el4hi);
el4ll = v_cvt_f64(el4li) + prev;
el4lh = v_cvt_f64_high(el4li) + prev;
el4hl = v_cvt_f64(el4hi) + el4ll;
el4hh = v_cvt_f64_high(el4hi) + el4lh;
prev = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh));
// prev = v_broadcast_element<v_float64::nlanes - 1>(el4hh);
el4ll = v_add(v_cvt_f64(el4li), prev);
el4lh = v_add(v_cvt_f64_high(el4li), prev);
el4hl = v_add(v_cvt_f64(el4hi), el4ll);
el4hh = v_add(v_cvt_f64_high(el4hi), el4lh);
prev = vx_setall_f64(v_extract_highest(el4hh));
// prev = v_broadcast_highest(el4hh);
#endif
v_store(sum_row + j , el4ll + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float64::nlanes , el4lh + vx_load(prev_sum_row + j + v_float64::nlanes ));
v_store(sum_row + j + v_float64::nlanes * 2, el4hl + vx_load(prev_sum_row + j + v_float64::nlanes * 2));
v_store(sum_row + j + v_float64::nlanes * 3, el4hh + vx_load(prev_sum_row + j + v_float64::nlanes * 3));
v_store(sum_row + j , v_add(el4ll, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() , v_add(el4lh, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes())));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 2, v_add(el4hl, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 3, v_add(el4hh, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 3)));
}
for (double v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j)
@@ -808,11 +808,11 @@ struct Integral_SIMD<uchar, double, double>
v_float64 prev_1 = vx_setzero_f64(), prev_2 = vx_setzero_f64();
int j = 0;
for (; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn)
for (; j + VTraits<v_uint16>::vlanes() * cn <= width; j += VTraits<v_uint16>::vlanes() * cn)
{
v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j));
v_int16 el8_1 = v_src_row & mask;
v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8);
v_int16 el8_1 = v_and(v_src_row, mask);
v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row)));
v_float64 el4ll_1, el4lh_1, el4hl_1, el4hh_1, el4ll_2, el4lh_2, el4hl_2, el4hh_2;
#if CV_AVX2 && CV_SIMD_WIDTH == 32
__m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2));
@@ -838,10 +838,10 @@ struct Integral_SIMD<uchar, double, double>
prev_1.val = _mm256_permute4x64_pd(el4hh_1.val, 0xff);
prev_2.val = _mm256_permute4x64_pd(el4hh_2.val, 0xff);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -853,32 +853,32 @@ struct Integral_SIMD<uchar, double, double>
v_int32 el4li_1, el4hi_1, el4li_2, el4hi_2;
v_expand(el8_1, el4li_1, el4hi_1);
v_expand(el8_2, el4li_2, el4hi_2);
el4ll_1 = v_cvt_f64(el4li_1) + prev_1;
el4ll_2 = v_cvt_f64(el4li_2) + prev_2;
el4lh_1 = v_cvt_f64_high(el4li_1) + prev_1;
el4lh_2 = v_cvt_f64_high(el4li_2) + prev_2;
el4hl_1 = v_cvt_f64(el4hi_1) + el4ll_1;
el4hl_2 = v_cvt_f64(el4hi_2) + el4ll_2;
el4hh_1 = v_cvt_f64_high(el4hi_1) + el4lh_1;
el4hh_2 = v_cvt_f64_high(el4hi_2) + el4lh_2;
prev_1 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_1));
prev_2 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_2));
// prev_1 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_1);
// prev_2 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_2);
el4ll_1 = v_add(v_cvt_f64(el4li_1), prev_1);
el4ll_2 = v_add(v_cvt_f64(el4li_2), prev_2);
el4lh_1 = v_add(v_cvt_f64_high(el4li_1), prev_1);
el4lh_2 = v_add(v_cvt_f64_high(el4li_2), prev_2);
el4hl_1 = v_add(v_cvt_f64(el4hi_1), el4ll_1);
el4hl_2 = v_add(v_cvt_f64(el4hi_2), el4ll_2);
el4hh_1 = v_add(v_cvt_f64_high(el4hi_1), el4lh_1);
el4hh_2 = v_add(v_cvt_f64_high(el4hi_2), el4lh_2);
prev_1 = vx_setall_f64(v_extract_highest(el4hh_1));
prev_2 = vx_setall_f64(v_extract_highest(el4hh_2));
// prev_1 = v_broadcast_highest(el4hh_1);
// prev_2 = v_broadcast_highest(el4hh_2);
#endif
v_float64 el4_1, el4_2, el4_3, el4_4, el4_5, el4_6, el4_7, el4_8;
v_zip(el4ll_1, el4ll_2, el4_1, el4_2);
v_zip(el4lh_1, el4lh_2, el4_3, el4_4);
v_zip(el4hl_1, el4hl_2, el4_5, el4_6);
v_zip(el4hh_1, el4hh_2, el4_7, el4_8);
v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float64::nlanes , el4_2 + vx_load(prev_sum_row + j + v_float64::nlanes ));
v_store(sum_row + j + v_float64::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 2));
v_store(sum_row + j + v_float64::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_float64::nlanes * 3));
v_store(sum_row + j + v_float64::nlanes * 4, el4_5 + vx_load(prev_sum_row + j + v_float64::nlanes * 4));
v_store(sum_row + j + v_float64::nlanes * 5, el4_6 + vx_load(prev_sum_row + j + v_float64::nlanes * 5));
v_store(sum_row + j + v_float64::nlanes * 6, el4_7 + vx_load(prev_sum_row + j + v_float64::nlanes * 6));
v_store(sum_row + j + v_float64::nlanes * 7, el4_8 + vx_load(prev_sum_row + j + v_float64::nlanes * 7));
v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes())));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 3)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 4, v_add(el4_5, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 4)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 5, v_add(el4_6, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 5)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 6, v_add(el4_7, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 6)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 7, v_add(el4_8, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 7)));
}
for (double v2 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -897,7 +897,7 @@ struct Integral_SIMD<uchar, double, double>
const uchar * src_row = src + _srcstep * i;
double * prev_sum_row = (double *)((uchar *)sum + _sumstep * i) + cn;
double * sum_row = (double *)((uchar *)sum + _sumstep * (i + 1)) + cn;
double row_cache[v_float64::nlanes * 12];
double row_cache[VTraits<v_float64>::max_nlanes * 12];
sum_row[-1] = sum_row[-2] = sum_row[-3] = 0;
@@ -905,10 +905,10 @@ struct Integral_SIMD<uchar, double, double>
prev_3 = vx_setzero_f64();
int j = 0;
const int j_max =
((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height)
? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave()
: width - v_uint16::nlanes * cn; // v_expand_low
for ( ; j <= j_max; j += v_uint16::nlanes * cn)
((_srcstep * i + (width - VTraits<v_uint16>::vlanes() * cn + VTraits<v_uint8>::vlanes() * cn)) >= _srcstep * height)
? width - VTraits<v_uint8>::vlanes() * cn // uint8 in v_load_deinterleave()
: width - VTraits<v_uint16>::vlanes() * cn; // v_expand_low
for ( ; j <= j_max; j += VTraits<v_uint16>::vlanes() * cn)
{
v_uint8 v_src_row_1, v_src_row_2, v_src_row_3;
v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3);
@@ -951,12 +951,12 @@ struct Integral_SIMD<uchar, double, double>
prev_2.val = _mm256_permute4x64_pd(el4hh_2.val, 0xff);
prev_3.val = _mm256_permute4x64_pd(el4hh_3.val, 0xff);
#else
el8_1 += v_rotate_left<1>(el8_1);
el8_2 += v_rotate_left<1>(el8_2);
el8_3 += v_rotate_left<1>(el8_3);
el8_1 += v_rotate_left<2>(el8_1);
el8_2 += v_rotate_left<2>(el8_2);
el8_3 += v_rotate_left<2>(el8_3);
el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<1>(el8_3));
el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1));
el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2));
el8_3 = v_add(el8_3, v_rotate_left<2>(el8_3));
#if CV_SIMD_WIDTH >= 32
el8_1 += v_rotate_left<4>(el8_1);
el8_2 += v_rotate_left<4>(el8_2);
@@ -971,53 +971,53 @@ struct Integral_SIMD<uchar, double, double>
v_expand(el8_1, el4li_1, el4hi_1);
v_expand(el8_2, el4li_2, el4hi_2);
v_expand(el8_3, el4li_3, el4hi_3);
el4ll_1 = v_cvt_f64(el4li_1) + prev_1;
el4ll_2 = v_cvt_f64(el4li_2) + prev_2;
el4ll_3 = v_cvt_f64(el4li_3) + prev_3;
el4lh_1 = v_cvt_f64_high(el4li_1) + prev_1;
el4lh_2 = v_cvt_f64_high(el4li_2) + prev_2;
el4lh_3 = v_cvt_f64_high(el4li_3) + prev_3;
el4hl_1 = v_cvt_f64(el4hi_1) + el4ll_1;
el4hl_2 = v_cvt_f64(el4hi_2) + el4ll_2;
el4hl_3 = v_cvt_f64(el4hi_3) + el4ll_3;
el4hh_1 = v_cvt_f64_high(el4hi_1) + el4lh_1;
el4hh_2 = v_cvt_f64_high(el4hi_2) + el4lh_2;
el4hh_3 = v_cvt_f64_high(el4hi_3) + el4lh_3;
prev_1 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_1));
prev_2 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_2));
prev_3 = vx_setall_f64(v_extract_n<v_float64::nlanes - 1>(el4hh_3));
// prev_1 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_1);
// prev_2 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_2);
// prev_3 = v_broadcast_element<v_float64::nlanes - 1>(el4hh_3);
el4ll_1 = v_add(v_cvt_f64(el4li_1), prev_1);
el4ll_2 = v_add(v_cvt_f64(el4li_2), prev_2);
el4ll_3 = v_add(v_cvt_f64(el4li_3), prev_3);
el4lh_1 = v_add(v_cvt_f64_high(el4li_1), prev_1);
el4lh_2 = v_add(v_cvt_f64_high(el4li_2), prev_2);
el4lh_3 = v_add(v_cvt_f64_high(el4li_3), prev_3);
el4hl_1 = v_add(v_cvt_f64(el4hi_1), el4ll_1);
el4hl_2 = v_add(v_cvt_f64(el4hi_2), el4ll_2);
el4hl_3 = v_add(v_cvt_f64(el4hi_3), el4ll_3);
el4hh_1 = v_add(v_cvt_f64_high(el4hi_1), el4lh_1);
el4hh_2 = v_add(v_cvt_f64_high(el4hi_2), el4lh_2);
el4hh_3 = v_add(v_cvt_f64_high(el4hi_3), el4lh_3);
prev_1 = vx_setall_f64(v_extract_highest(el4hh_1));
prev_2 = vx_setall_f64(v_extract_highest(el4hh_2));
prev_3 = vx_setall_f64(v_extract_highest(el4hh_3));
// prev_1 = v_broadcast_highest(el4hh_1);
// prev_2 = v_broadcast_highest(el4hh_2);
// prev_3 = v_broadcast_highest(el4hh_3);
#endif
v_store_interleave(row_cache , el4ll_1, el4ll_2, el4ll_3);
v_store_interleave(row_cache + v_float64::nlanes * 3, el4lh_1, el4lh_2, el4lh_3);
v_store_interleave(row_cache + v_float64::nlanes * 6, el4hl_1, el4hl_2, el4hl_3);
v_store_interleave(row_cache + v_float64::nlanes * 9, el4hh_1, el4hh_2, el4hh_3);
v_store_interleave(row_cache + VTraits<v_float64>::vlanes() * 3, el4lh_1, el4lh_2, el4lh_3);
v_store_interleave(row_cache + VTraits<v_float64>::vlanes() * 6, el4hl_1, el4hl_2, el4hl_3);
v_store_interleave(row_cache + VTraits<v_float64>::vlanes() * 9, el4hh_1, el4hh_2, el4hh_3);
el4ll_1 = vx_load(row_cache );
el4ll_2 = vx_load(row_cache + v_float64::nlanes );
el4ll_3 = vx_load(row_cache + v_float64::nlanes * 2 );
el4lh_1 = vx_load(row_cache + v_float64::nlanes * 3 );
el4lh_2 = vx_load(row_cache + v_float64::nlanes * 4 );
el4lh_3 = vx_load(row_cache + v_float64::nlanes * 5 );
el4hl_1 = vx_load(row_cache + v_float64::nlanes * 6 );
el4hl_2 = vx_load(row_cache + v_float64::nlanes * 7 );
el4hl_3 = vx_load(row_cache + v_float64::nlanes * 8 );
el4hh_1 = vx_load(row_cache + v_float64::nlanes * 9 );
el4hh_2 = vx_load(row_cache + v_float64::nlanes * 10);
el4hh_3 = vx_load(row_cache + v_float64::nlanes * 11);
v_store(sum_row + j , el4ll_1 + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float64::nlanes , el4ll_2 + vx_load(prev_sum_row + j + v_float64::nlanes ));
v_store(sum_row + j + v_float64::nlanes * 2 , el4ll_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 2 ));
v_store(sum_row + j + v_float64::nlanes * 3 , el4lh_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 3 ));
v_store(sum_row + j + v_float64::nlanes * 4 , el4lh_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 4 ));
v_store(sum_row + j + v_float64::nlanes * 5 , el4lh_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 5 ));
v_store(sum_row + j + v_float64::nlanes * 6 , el4hl_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 6 ));
v_store(sum_row + j + v_float64::nlanes * 7 , el4hl_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 7 ));
v_store(sum_row + j + v_float64::nlanes * 8 , el4hl_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 8 ));
v_store(sum_row + j + v_float64::nlanes * 9 , el4hh_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 9 ));
v_store(sum_row + j + v_float64::nlanes * 10, el4hh_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 10));
v_store(sum_row + j + v_float64::nlanes * 11, el4hh_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 11));
el4ll_2 = vx_load(row_cache + VTraits<v_float64>::vlanes() );
el4ll_3 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 2 );
el4lh_1 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 3 );
el4lh_2 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 4 );
el4lh_3 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 5 );
el4hl_1 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 6 );
el4hl_2 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 7 );
el4hl_3 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 8 );
el4hh_1 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 9 );
el4hh_2 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 10);
el4hh_3 = vx_load(row_cache + VTraits<v_float64>::vlanes() * 11);
v_store(sum_row + j , v_add(el4ll_1, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() , v_add(el4ll_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes())));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 2 , v_add(el4ll_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 3 , v_add(el4lh_1, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 3)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 4 , v_add(el4lh_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 4)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 5 , v_add(el4lh_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 5)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 6 , v_add(el4hl_1, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 6)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 7 , v_add(el4hl_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 7)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 8 , v_add(el4hl_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 8)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 9 , v_add(el4hh_1, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 9)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 10, v_add(el4hh_2, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 10)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 11, v_add(el4hh_3, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 11)));
}
for (double v3 = sum_row[j - 1] - prev_sum_row[j - 1],
@@ -1043,7 +1043,7 @@ struct Integral_SIMD<uchar, double, double>
v_float64 prev_1 = vx_setzero_f64(), prev_2 = vx_setzero_f64();
int j = 0;
for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes)
for ( ; j + VTraits<v_uint16>::vlanes() <= width; j += VTraits<v_uint16>::vlanes())
{
v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j));
v_float64 el4ll, el4lh, el4hl, el4hh;
@@ -1065,10 +1065,10 @@ struct Integral_SIMD<uchar, double, double>
#endif
v_int32 el4li, el4hi;
v_expand(el8, el4li, el4hi);
el4ll = v_cvt_f64(el4li) + prev_1;
el4lh = v_cvt_f64_high(el4li) + prev_2;
el4hl = v_cvt_f64(el4hi) + el4ll;
el4hh = v_cvt_f64_high(el4hi) + el4lh;
el4ll = v_add(v_cvt_f64(el4li), prev_1);
el4lh = v_add(v_cvt_f64_high(el4li), prev_2);
el4hl = v_add(v_cvt_f64(el4hi), el4ll);
el4hh = v_add(v_cvt_f64_high(el4hi), el4lh);
#if CV_SIMD_WIDTH == 16
prev_1 = el4hl;
prev_2 = el4hh;
@@ -1078,10 +1078,10 @@ struct Integral_SIMD<uchar, double, double>
prev_1 = prev_2 = v_combine_high(el4hh, el4hh);
#endif
#endif
v_store(sum_row + j , el4ll + vx_load(prev_sum_row + j ));
v_store(sum_row + j + v_float64::nlanes , el4lh + vx_load(prev_sum_row + j + v_float64::nlanes ));
v_store(sum_row + j + v_float64::nlanes * 2, el4hl + vx_load(prev_sum_row + j + v_float64::nlanes * 2));
v_store(sum_row + j + v_float64::nlanes * 3, el4hh + vx_load(prev_sum_row + j + v_float64::nlanes * 3));
v_store(sum_row + j , v_add(el4ll, vx_load(prev_sum_row + j)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() , v_add(el4lh, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes())));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 2, v_add(el4hl, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 2)));
v_store(sum_row + j + VTraits<v_float64>::vlanes() * 3, v_add(el4hh, vx_load(prev_sum_row + j + VTraits<v_float64>::vlanes() * 3)));
}
for (double v4 = sum_row[j - 1] - prev_sum_row[j - 1],
+1 -1
View File
@@ -343,11 +343,11 @@ BarcodeDetector::BarcodeDetector(const string &prototxt_path, const string &mode
{
Ptr<BarcodeImpl> p_ = new BarcodeImpl();
p = p_;
p_->sr = make_shared<SuperScale>();
if (!prototxt_path.empty() && !model_path.empty())
{
CV_Assert(utils::fs::exists(prototxt_path));
CV_Assert(utils::fs::exists(model_path));
p_->sr = make_shared<SuperScale>();
int res = p_->sr->init(prototxt_path, model_path);
CV_Assert(res == 0);
p_->use_nn_sr = true;
+15 -8
View File
@@ -102,12 +102,21 @@ public:
return 0;
}
CV_CheckEQ(input_image.size(), Size(inputW, inputH), "Size does not match. Call setInputSize(size) if input size does not match the preset size");
// Pad input_image with divisor 32
Mat pad_image = padWithDivisor(input_image);
// Build blob from input image
Mat input_blob = dnn::blobFromImage(pad_image);
Mat input_blob;
if(input_image.kind() == _InputArray::UMAT) {
// Pad input_image with divisor 32
UMat pad_image;
padWithDivisor(input_image, pad_image);
// Build blob from input image
input_blob = dnn::blobFromImage(pad_image);
} else {
// Pad input_image with divisor 32
Mat pad_image;
padWithDivisor(input_image, pad_image);
// Build blob from input image
input_blob = dnn::blobFromImage(pad_image);
}
// Forward
std::vector<String> output_names = { "cls_8", "cls_16", "cls_32", "obj_8", "obj_16", "obj_32", "bbox_8", "bbox_16", "bbox_32", "kps_8", "kps_16", "kps_32" };
std::vector<Mat> output_blobs;
@@ -217,13 +226,11 @@ private:
}
}
Mat padWithDivisor(InputArray& input_image)
void padWithDivisor(InputArray input_image, OutputArray pad_image)
{
int bottom = padH - inputH;
int right = padW - inputW;
Mat pad_image;
copyMakeBorder(input_image, pad_image, 0, bottom, 0, right, BORDER_CONSTANT, 0);
return pad_image;
}
private:
dnn::Net net;
+46 -46
View File
@@ -268,13 +268,13 @@ void HOGDescriptor::computeGradient(InputArray _img, InputOutputArray _grad, Inp
for ( i = 0; i < 256; i += 4)
{
v_store(_data + i, v_sqrt(idx));
idx += ifour;
idx = v_add(idx, ifour);
}
else
for ( i = 0; i < 256; i += 4)
{
v_store(_data + i, idx);
idx += ifour;
idx = v_add(idx, ifour);
}
#else
if( gammaCorrection )
@@ -320,7 +320,7 @@ void HOGDescriptor::computeGradient(InputArray _img, InputOutputArray _grad, Inp
for ( ; x <= end - 4; x += 4)
{
v_int32x4 mul_res = v_load(xmap + x);
mul_res += mul_res + mul_res;
mul_res = v_add(mul_res, v_add(mul_res, mul_res));
v_store(xmap + x, mul_res);
}
#endif
@@ -444,34 +444,34 @@ void HOGDescriptor::computeGradient(InputArray _img, InputOutputArray _grad, Inp
{
int x0 = xmap[x], x1 = xmap[x+1], x2 = xmap[x+2], x3 = xmap[x+3];
v_float32x4 _dx0 = v_load(lutCurr+x+widthP2*0+2) - v_load(lutCurr+x+widthP2*0);
v_float32x4 _dx1 = v_load(lutCurr+x+widthP2*1+2) - v_load(lutCurr+x+widthP2*1);
v_float32x4 _dx2 = v_load(lutCurr+x+widthP2*2+2) - v_load(lutCurr+x+widthP2*2);
v_float32x4 _dx0 = v_sub(v_load(lutCurr + x + widthP2 * 0 + 2), v_load(lutCurr + x + widthP2 * 0));
v_float32x4 _dx1 = v_sub(v_load(lutCurr + x + widthP2 * 1 + 2), v_load(lutCurr + x + widthP2 * 1));
v_float32x4 _dx2 = v_sub(v_load(lutCurr + x + widthP2 * 2 + 2), v_load(lutCurr + x + widthP2 * 2));
v_float32x4 _dy00 = v_float32x4(lut[nextPtr[x0+0]], lut[nextPtr[x1+0]], lut[nextPtr[x2+0]], lut[nextPtr[x3+0]]);
v_float32x4 _dy0 = _dy00 - v_load(lutPrev+x+widthP2*0+1);
v_float32x4 _dy0 = v_sub(_dy00, v_load(lutPrev + x + widthP2 * 0 + 1));
v_store(lutNext+x+widthP2*0+1, _dy00);
v_float32x4 _dy10 = v_float32x4(lut[nextPtr[x0+1]], lut[nextPtr[x1+1]], lut[nextPtr[x2+1]], lut[nextPtr[x3+1]]);
v_float32x4 _dy1 = _dy10 - v_load(lutPrev+x+widthP2*1+1);
v_float32x4 _dy1 = v_sub(_dy10, v_load(lutPrev + x + widthP2 * 1 + 1));
v_store(lutNext+x+widthP2*1+1, _dy10);
v_float32x4 _dy20 = v_float32x4(lut[nextPtr[x0+2]], lut[nextPtr[x1+2]], lut[nextPtr[x2+2]], lut[nextPtr[x3+2]]);
v_float32x4 _dy2 = _dy20 - v_load(lutPrev+x+widthP2*2+1);
v_float32x4 _dy2 = v_sub(_dy20, v_load(lutPrev + x + widthP2 * 2 + 1));
v_store(lutNext+x+widthP2*2+1, _dy20);
v_float32x4 _mag0 = (_dx0 * _dx0) + (_dy0 * _dy0);
v_float32x4 _mag1 = (_dx1 * _dx1) + (_dy1 * _dy1);
v_float32x4 _mag2 = (_dx2 * _dx2) + (_dy2 * _dy2);
v_float32x4 _mag0 = v_add(v_mul(_dx0, _dx0), v_mul(_dy0, _dy0));
v_float32x4 _mag1 = v_add(v_mul(_dx1, _dx1), v_mul(_dy1, _dy1));
v_float32x4 _mag2 = v_add(v_mul(_dx2, _dx2), v_mul(_dy2, _dy2));
v_float32x4 mask = v_reinterpret_as_f32(_mag2 > _mag1);
v_float32x4 mask = v_reinterpret_as_f32(v_gt(_mag2, _mag1));
_dx2 = v_select(mask, _dx2, _dx1);
_dy2 = v_select(mask, _dy2, _dy1);
mask = v_reinterpret_as_f32(v_max(_mag2, _mag1) > _mag0);
mask = v_reinterpret_as_f32(v_gt(v_max(_mag2, _mag1), _mag0));
_dx2 = v_select(mask, _dx2, _dx0);
_dy2 = v_select(mask, _dy2, _dy0);
@@ -537,25 +537,25 @@ void HOGDescriptor::computeGradient(InputArray _img, InputOutputArray _grad, Inp
int x2 = x << 1;
v_float32x4 _mag = v_load(dbuf + x + (width << 1));
v_float32x4 _angle = v_load(dbuf + x + width * 3);
_angle = (_angleScale * _angle) - fhalf;
_angle = v_sub(v_mul(_angleScale, _angle), fhalf);
v_int32x4 _hidx = v_floor(_angle);
_angle -= v_cvt_f32(_hidx);
_angle = v_sub(_angle, v_cvt_f32(_hidx));
v_float32x4 ft0 = _mag * (fone - _angle);
v_float32x4 ft1 = _mag * _angle;
v_float32x4 ft0 = v_mul(_mag, v_sub(fone, _angle));
v_float32x4 ft1 = v_mul(_mag, _angle);
v_store_interleave(gradPtr + x2, ft0, ft1);
v_int32x4 mask0 = _hidx >> 31;
v_int32x4 it0 = mask0 & _nbins;
mask0 = (_hidx >= _nbins);
v_int32x4 it1 = mask0 & _nbins;
_hidx += (it0 - it1);
v_int32x4 mask0 = v_shr<31>(_hidx);
v_int32x4 it0 = v_and(mask0, _nbins);
mask0 = (v_ge(_hidx, _nbins));
v_int32x4 it1 = v_and(mask0, _nbins);
_hidx = v_add(_hidx, v_sub(it0, it1));
it0 = v_reinterpret_as_s32(v_pack(v_pack(_hidx, izero), v_reinterpret_as_s16(izero)));
_hidx += ione;
_hidx &= (_hidx < _nbins);
_hidx = v_add(_hidx, ione);
_hidx = v_and(_hidx, v_lt(_hidx, _nbins));
it1 = v_reinterpret_as_s32(v_pack(v_pack(_hidx, izero), v_reinterpret_as_s16(izero)));
v_uint8x16 it2, it3;
v_zip(v_reinterpret_as_u8(it0), v_reinterpret_as_u8(it1), it2, it3);
@@ -707,9 +707,9 @@ void HOGCache::init(const HOGDescriptor* _descriptor,
for (; i <= blockSize.height - 4; i += 4)
{
v_float32x4 t = idx - _bh;
t *= t;
idx += ifour;
v_float32x4 t = v_sub(idx, _bh);
t = v_mul(t, t);
idx = v_add(idx, ifour);
v_store(_di + i, t);
}
#endif
@@ -725,9 +725,9 @@ void HOGCache::init(const HOGDescriptor* _descriptor,
for (; j <= blockSize.height - 4; j += 4)
{
v_float32x4 t = idx - _bw;
t *= t;
idx += ifour;
v_float32x4 t = v_sub(idx, _bw);
t = v_mul(t, t);
idx = v_add(idx, ifour);
v_store(_dj + j, t);
}
#endif
@@ -936,8 +936,8 @@ const float* HOGCache::getBlock(Point pt, float* buf)
int h0 = h[0], h1 = h[1];
v_float32x4 _a0 = v_setall_f32(a[0]), _a1 = v_setall_f32(a[1]);
v_float32x4 w = v_setall_f32(pk.gradWeight) * v_load(pk.histWeights);
v_float32x4 _t0 = _a0 * w, _t1 = _a1 * w;
v_float32x4 w = v_mul(v_setall_f32(pk.gradWeight), v_load(pk.histWeights));
v_float32x4 _t0 = v_mul(_a0, w), _t1 = v_mul(_a1, w);
v_store(hist0, _t0);
v_store(hist1, _t1);
@@ -984,8 +984,8 @@ const float* HOGCache::getBlock(Point pt, float* buf)
int h0 = h[0], h1 = h[1];
v_float32x4 _a0 = v_setall_f32(a[0]), _a1 = v_setall_f32(a[1]);
v_float32x4 w = v_setall_f32(pk.gradWeight) * v_load(pk.histWeights);
v_float32x4 _t0 = _a0 * w, _t1 = _a1 * w;
v_float32x4 w = v_mul(v_setall_f32(pk.gradWeight), v_load(pk.histWeights));
v_float32x4 _t0 = v_mul(_a0, w), _t1 = v_mul(_a1, w);
v_store(hist0, _t0);
v_store(hist1, _t1);
@@ -1057,12 +1057,12 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const
#if CV_SIMD128
v_float32x4 p0 = v_load(hist);
v_float32x4 s = p0 * p0;
v_float32x4 s = v_mul(p0, p0);
for (i = 4; i <= sz - 4; i += 4)
{
p0 = v_load(hist + i);
s += p0 * p0;
s = v_add(s, v_mul(p0, p0));
}
v_store(partSum, s);
#else
@@ -1091,17 +1091,17 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const
v_float32x4 _scale = v_setall_f32(scale);
static v_float32x4 _threshold = v_setall_f32(thresh);
v_float32x4 p = _scale * v_load(hist);
v_float32x4 p = v_mul(_scale, v_load(hist));
p = v_min(p, _threshold);
s = p * p;
s = v_mul(p, p);
v_store(hist, p);
for(i = 4 ; i <= sz - 4; i += 4)
{
p = v_load(hist + i);
p *= _scale;
p = v_mul(p, _scale);
p = v_min(p, _threshold);
s += p * p;
s = v_add(s, v_mul(p, p));
v_store(hist + i, p);
}
@@ -1137,7 +1137,7 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const
v_float32x4 _scale2 = v_setall_f32(scale);
for ( ; i <= sz - 4; i += 4)
{
v_float32x4 t = _scale2 * v_load(hist + i);
v_float32x4 t = v_mul(_scale2, v_load(hist + i));
v_store(hist + i, t);
}
#endif
@@ -1593,14 +1593,14 @@ void HOGDescriptor::detect(InputArray _img,
#if CV_SIMD128
v_float32x4 _vec = v_load(vec);
v_float32x4 _svmVec = v_load(svmVec);
v_float32x4 sum = _svmVec * _vec;
v_float32x4 sum = v_mul(_svmVec, _vec);
for( k = 4; k <= blockHistogramSize - 4; k += 4 )
{
_vec = v_load(vec + k);
_svmVec = v_load(svmVec + k);
sum += _vec * _svmVec;
sum = v_add(sum, v_mul(_vec, _svmVec));
}
v_store(partSum, sum);
@@ -3392,14 +3392,14 @@ void HOGDescriptor::detectROI(InputArray _img, const std::vector<cv::Point> &loc
#if CV_SIMD128
v_float32x4 _vec = v_load(vec);
v_float32x4 _svmVec = v_load(svmVec);
v_float32x4 sum = _svmVec * _vec;
v_float32x4 sum = v_mul(_svmVec, _vec);
for( k = 4; k <= blockHistogramSize - 4; k += 4 )
{
_vec = v_load(vec + k);
_svmVec = v_load(svmVec + k);
sum += _vec * _svmVec;
sum = v_add(sum, v_mul(_vec, _svmVec));
}
v_store(partSum, sum);
+1 -1
View File
@@ -1060,7 +1060,7 @@ class FuncInfo(object):
else:
py_name = classinfo.full_export_name + "." + self.variants[0].wname
if not self.is_static:
if not self.is_static and not self.isconstructor:
cname = classinfo.cname + '::' + cname
else:
py_name = '.'.join([self.namespace, self.variants[0].wname])
+1 -1
View File
@@ -66,7 +66,7 @@ Mat CameraParams::K() const
Mat_<double> k = Mat::eye(3, 3, CV_64F);
k(0,0) = focal; k(0,2) = ppx;
k(1,1) = focal * aspect; k(1,2) = ppy;
return std::move(k);
return Mat(k);
}
} // namespace detail
+20 -20
View File
@@ -520,16 +520,16 @@ DISOpticalFlowImpl::PatchInverseSearch_ParBody::PatchInverseSearch_ParBody(DISOp
v_expand(I0_row_8, I0_row_4_left, I0_row_4_right); \
\
/* Compute diffs between I0 and bilinearly interpolated I1: */ \
I_diff_left = w00v * v_cvt_f32(v_reinterpret_as_s32(I1_row_4_left)) + \
w01v * v_cvt_f32(v_reinterpret_as_s32(I1_row_shifted_4_left)) + \
w10v * v_cvt_f32(v_reinterpret_as_s32(I1_row_next_4_left)) + \
w11v * v_cvt_f32(v_reinterpret_as_s32(I1_row_next_shifted_4_left)) - \
v_cvt_f32(v_reinterpret_as_s32(I0_row_4_left)); \
I_diff_right = w00v * v_cvt_f32(v_reinterpret_as_s32(I1_row_4_right)) + \
w01v * v_cvt_f32(v_reinterpret_as_s32(I1_row_shifted_4_right)) + \
w10v * v_cvt_f32(v_reinterpret_as_s32(I1_row_next_4_right)) + \
w11v * v_cvt_f32(v_reinterpret_as_s32(I1_row_next_shifted_4_right)) - \
v_cvt_f32(v_reinterpret_as_s32(I0_row_4_right));
I_diff_left = v_sub(v_add(v_mul(w00v, v_cvt_f32(v_reinterpret_as_s32(I1_row_4_left))), \
v_mul(w01v, v_cvt_f32(v_reinterpret_as_s32(I1_row_shifted_4_left))), \
v_mul(w10v, v_cvt_f32(v_reinterpret_as_s32(I1_row_next_4_left))), \
v_mul(w11v, v_cvt_f32(v_reinterpret_as_s32(I1_row_next_shifted_4_left)))), \
v_cvt_f32(v_reinterpret_as_s32(I0_row_4_left))); \
I_diff_right = v_sub(v_add(v_mul(w00v, v_cvt_f32(v_reinterpret_as_s32(I1_row_4_right))), \
v_mul(w01v, v_cvt_f32(v_reinterpret_as_s32(I1_row_shifted_4_right))), \
v_mul(w10v, v_cvt_f32(v_reinterpret_as_s32(I1_row_next_4_right))), \
v_mul(w11v, v_cvt_f32(v_reinterpret_as_s32(I1_row_next_shifted_4_right)))), \
v_cvt_f32(v_reinterpret_as_s32(I0_row_4_right)));
#define HAL_BILINEAR_8x8_PATCH_EXTRACTION_NEXT_ROW \
I0_ptr += I0_stride; \
@@ -572,9 +572,9 @@ inline float processPatch(float &dst_dUx, float &dst_dUy, uchar *I0_ptr, uchar *
v_expand(I0y_row, I0y_row_4_left, I0y_row_4_right);
/* Update the sums: */
Ux_vec += I_diff_left * v_cvt_f32(I0x_row_4_left) + I_diff_right * v_cvt_f32(I0x_row_4_right);
Uy_vec += I_diff_left * v_cvt_f32(I0y_row_4_left) + I_diff_right * v_cvt_f32(I0y_row_4_right);
SSD_vec += I_diff_left * I_diff_left + I_diff_right * I_diff_right;
Ux_vec = v_add(Ux_vec, v_add(v_mul(I_diff_left, v_cvt_f32(I0x_row_4_left)), v_mul(I_diff_right, v_cvt_f32(I0x_row_4_right))));
Uy_vec = v_add(Uy_vec, v_add(v_mul(I_diff_left, v_cvt_f32(I0y_row_4_left)), v_mul(I_diff_right, v_cvt_f32(I0y_row_4_right))));
SSD_vec = v_add(SSD_vec, v_add(v_mul(I_diff_left, I_diff_left), v_mul(I_diff_right, I_diff_right)));
I0x_ptr += I0_stride;
I0y_ptr += I0_stride;
@@ -640,10 +640,10 @@ inline float processPatchMeanNorm(float &dst_dUx, float &dst_dUy, uchar *I0_ptr,
v_expand(I0y_row, I0y_row_4_left, I0y_row_4_right);
/* Update the sums: */
sum_I0x_mul_vec += I_diff_left * v_cvt_f32(I0x_row_4_left) + I_diff_right * v_cvt_f32(I0x_row_4_right);
sum_I0y_mul_vec += I_diff_left * v_cvt_f32(I0y_row_4_left) + I_diff_right * v_cvt_f32(I0y_row_4_right);
sum_diff_sq_vec += I_diff_left * I_diff_left + I_diff_right * I_diff_right;
sum_diff_vec += I_diff_left + I_diff_right;
sum_I0x_mul_vec = v_add(sum_I0x_mul_vec, v_add(v_mul(I_diff_left, v_cvt_f32(I0x_row_4_left)), v_mul(I_diff_right, v_cvt_f32(I0x_row_4_right))));
sum_I0y_mul_vec = v_add(sum_I0y_mul_vec, v_add(v_mul(I_diff_left, v_cvt_f32(I0y_row_4_left)), v_mul(I_diff_right, v_cvt_f32(I0y_row_4_right))));
sum_diff_sq_vec = v_add(sum_diff_sq_vec, v_add(v_mul(I_diff_left, I_diff_left), v_mul(I_diff_right, I_diff_right)));
sum_diff_vec = v_add(sum_diff_vec, v_add(I_diff_left, I_diff_right));
I0x_ptr += I0_stride;
I0y_ptr += I0_stride;
@@ -692,7 +692,7 @@ inline float computeSSD(uchar *I0_ptr, uchar *I1_ptr, int I0_stride, int I1_stri
for (int row = 0; row < 8; row++)
{
HAL_PROCESS_BILINEAR_8x8_PATCH_EXTRACTION;
SSD_vec += I_diff_left * I_diff_left + I_diff_right * I_diff_right;
SSD_vec = v_add(SSD_vec, v_add(v_mul(I_diff_left, I_diff_left), v_mul(I_diff_right, I_diff_right)));
HAL_BILINEAR_8x8_PATCH_EXTRACTION_NEXT_ROW;
}
SSD = v_reduce_sum(SSD_vec);
@@ -728,8 +728,8 @@ inline float computeSSDMeanNorm(uchar *I0_ptr, uchar *I1_ptr, int I0_stride, int
for (int row = 0; row < 8; row++)
{
HAL_PROCESS_BILINEAR_8x8_PATCH_EXTRACTION;
sum_diff_sq_vec += I_diff_left * I_diff_left + I_diff_right * I_diff_right;
sum_diff_vec += I_diff_left + I_diff_right;
sum_diff_sq_vec = v_add(sum_diff_sq_vec, v_add(v_mul(I_diff_left, I_diff_left), v_mul(I_diff_right, I_diff_right)));
sum_diff_vec = v_add(sum_diff_vec, v_add(I_diff_left, I_diff_right));
HAL_BILINEAR_8x8_PATCH_EXTRACTION_NEXT_ROW;
}
sum_diff = v_reduce_sum(sum_diff_vec);
+24 -24
View File
@@ -97,8 +97,8 @@ void cv::detail::ScharrDerivInvoker::operator()(const Range& range) const
v_int16x8 s1 = v_reinterpret_as_s16(v_load_expand(srow1 + x));
v_int16x8 s2 = v_reinterpret_as_s16(v_load_expand(srow2 + x));
v_int16x8 t1 = s2 - s0;
v_int16x8 t0 = v_mul_wrap(s0 + s2, c3) + v_mul_wrap(s1, c10);
v_int16x8 t1 = v_sub(s2, s0);
v_int16x8 t0 = v_add(v_mul_wrap(v_add(s0, s2), c3), v_mul_wrap(s1, c10));
v_store(trow0 + x, t0);
v_store(trow1 + x, t1);
@@ -134,8 +134,8 @@ void cv::detail::ScharrDerivInvoker::operator()(const Range& range) const
v_int16x8 s3 = v_load(trow1 + x);
v_int16x8 s4 = v_load(trow1 + x + cn);
v_int16x8 t0 = s1 - s0;
v_int16x8 t1 = v_mul_wrap(s2 + s4, c3) + v_mul_wrap(s3, c10);
v_int16x8 t0 = v_sub(s1, s0);
v_int16x8 t1 = v_add(v_mul_wrap(v_add(s2, s4), c3), v_mul_wrap(s3, c10));
v_store_interleave((drow + x*2), t0, t1);
}
@@ -293,10 +293,10 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_dotprod(t00, qw0, qdelta) + v_dotprod(t10, qw1);
t1 = v_dotprod(t01, qw0, qdelta) + v_dotprod(t11, qw1);
t0 = t0 >> (W_BITS1-5);
t1 = t1 >> (W_BITS1-5);
t0 = v_add(v_dotprod(t00, qw0, qdelta), v_dotprod(t10, qw1));
t1 = v_add(v_dotprod(t01, qw0, qdelta), v_dotprod(t11, qw1));
t0 = v_shr<W_BITS1 - 5>(t0);
t1 = v_shr<W_BITS1 - 5>(t1);
v_store(Iptr + x, v_pack(t0, t1));
v00 = v_reinterpret_as_s16(v_load(dsrc));
@@ -307,10 +307,10 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_dotprod(t00, qw0, qdelta_d) + v_dotprod(t10, qw1);
t1 = v_dotprod(t01, qw0, qdelta_d) + v_dotprod(t11, qw1);
t0 = t0 >> W_BITS1;
t1 = t1 >> W_BITS1;
t0 = v_add(v_dotprod(t00, qw0, qdelta_d), v_dotprod(t10, qw1));
t1 = v_add(v_dotprod(t01, qw0, qdelta_d), v_dotprod(t11, qw1));
t0 = v_shr<W_BITS1>(t0);
t1 = v_shr<W_BITS1>(t1);
v00 = v_pack(t0, t1); // Ix0 Iy0 Ix1 Iy1 ...
v_store(dIptr, v00);
@@ -332,10 +332,10 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_dotprod(t00, qw0, qdelta_d) + v_dotprod(t10, qw1);
t1 = v_dotprod(t01, qw0, qdelta_d) + v_dotprod(t11, qw1);
t0 = t0 >> W_BITS1;
t1 = t1 >> W_BITS1;
t0 = v_add(v_dotprod(t00, qw0, qdelta_d), v_dotprod(t10, qw1));
t1 = v_add(v_dotprod(t01, qw0, qdelta_d), v_dotprod(t11, qw1));
t0 = v_shr<W_BITS1>(t0);
t1 = v_shr<W_BITS1>(t1);
v00 = v_pack(t0, t1); // Ix0 Iy0 Ix1 Iy1 ...
v_store(dIptr + 4*2, v00);
@@ -548,18 +548,18 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const
v_zip(v00, v01, t00, t01);
v_zip(v10, v11, t10, t11);
t0 = v_dotprod(t00, qw0, qdelta) + v_dotprod(t10, qw1);
t1 = v_dotprod(t01, qw0, qdelta) + v_dotprod(t11, qw1);
t0 = t0 >> (W_BITS1-5);
t1 = t1 >> (W_BITS1-5);
diff0 = v_pack(t0, t1) - diff0;
t0 = v_add(v_dotprod(t00, qw0, qdelta), v_dotprod(t10, qw1));
t1 = v_add(v_dotprod(t01, qw0, qdelta), v_dotprod(t11, qw1));
t0 = v_shr<W_BITS1 - 5>(t0);
t1 = v_shr<W_BITS1 - 5>(t1);
diff0 = v_sub(v_pack(t0, t1), diff0);
v_zip(diff0, diff0, diff2, diff1); // It0 It0 It1 It1 ...
v00 = v_reinterpret_as_s16(v_load(dIptr)); // Ix0 Iy0 Ix1 Iy1 ...
v01 = v_reinterpret_as_s16(v_load(dIptr + 8));
v_zip(v00, v01, v10, v11);
v_zip(diff2, diff1, v00, v01);
qb0 += v_cvt_f32(v_dotprod(v00, v10));
qb1 += v_cvt_f32(v_dotprod(v01, v11));
qb0 = v_add(qb0, v_cvt_f32(v_dotprod(v00, v10)));
qb1 = v_add(qb1, v_cvt_f32(v_dotprod(v01, v11)));
}
#endif
@@ -647,7 +647,7 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const
#if CV_SIMD128 && !CV_NEON
v_float32x4 qf0, qf1;
v_recombine(v_interleave_pairs(qb0 + qb1), v_setzero_f32(), qf0, qf1);
v_recombine(v_interleave_pairs(v_add(qb0, qb1)), v_setzero_f32(), qf0, qf1);
ib1 += v_reduce_sum(qf0);
ib2 += v_reduce_sum(qf1);
#endif

Some files were not shown because too many files have changed in this diff Show More