diff --git a/3rdparty/ippicv/ippicv.cmake b/3rdparty/ippicv/ippicv.cmake index 08cf091db3..380f65f8c6 100644 --- a/3rdparty/ippicv/ippicv.cmake +++ b/3rdparty/ippicv/ippicv.cmake @@ -2,32 +2,32 @@ function(download_ippicv root_var) set(${root_var} "" PARENT_SCOPE) # Commit SHA in the opencv_3rdparty repo - set(IPPICV_COMMIT "1224f78da6684df04397ac0f40c961ed37f79ccb") + set(IPPICV_COMMIT "0cc4aa06bf2bef4b05d237c69a5a96b9cd0cb85a") # Define actual ICV versions if(APPLE) set(OPENCV_ICV_PLATFORM "macosx") set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_mac") - set(OPENCV_ICV_NAME "ippicv_2021.8_mac_intel64_20230330_general.tgz") - set(OPENCV_ICV_HASH "d2b234a86af1b616958619a4560356d9") + set(OPENCV_ICV_NAME "ippicv_2021.9.1_mac_intel64_20230919_general.tgz") + set(OPENCV_ICV_HASH "14f01c5a4780bfae9dde9b0aaf5e56fc") elseif((UNIX AND NOT ANDROID) OR (UNIX AND ANDROID_ABI MATCHES "x86")) set(OPENCV_ICV_PLATFORM "linux") set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_lnx") if(X86_64) - set(OPENCV_ICV_NAME "ippicv_2021.8_lnx_intel64_20230330_general.tgz") - set(OPENCV_ICV_HASH "43219bdc7e3805adcbe3a1e2f1f3ef3b") + set(OPENCV_ICV_NAME "ippicv_2021.10.0_lnx_intel64_20230919_general.tgz") + set(OPENCV_ICV_HASH "606a19b207ebedfe42d59fd916cc4850") else() - set(OPENCV_ICV_NAME "ippicv_2021.8_lnx_ia32_20230330_general.tgz") - set(OPENCV_ICV_HASH "165875443d72faa3fd2146869da90d07") + set(OPENCV_ICV_NAME "ippicv_2021.10.0_lnx_ia32_20230919_general.tgz") + set(OPENCV_ICV_HASH "ea08487b810baad2f68aca87b74a2db9") endif() elseif(WIN32 AND NOT ARM) set(OPENCV_ICV_PLATFORM "windows") set(OPENCV_ICV_PACKAGE_SUBDIR "ippicv_win") if(X86_64) - set(OPENCV_ICV_NAME "ippicv_2021.8_win_intel64_20230330_general.zip") - set(OPENCV_ICV_HASH "71e4f58de939f0348ec7fb58ffb17dbf") + set(OPENCV_ICV_NAME "ippicv_2021.10.0_win_intel64_20230919_general.zip") + set(OPENCV_ICV_HASH "538a819ec84193a9c9f3c0f8df0be8b7") else() - set(OPENCV_ICV_NAME "ippicv_2021.8_win_ia32_20230330_general.zip") - set(OPENCV_ICV_HASH "57fd4648cfe64eae9e2ad9d50173a553") + set(OPENCV_ICV_NAME "ippicv_2021.10.0_win_ia32_20230919_general.zip") + set(OPENCV_ICV_HASH "8ff93c69415ab0835cc1e94dc5660f5d") endif() else() return() diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e1482478a..e10a04cd63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -467,7 +467,7 @@ OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF VISIBLE_IF TRUE VERIFY HAVE_TIMVX) OCV_OPTION(WITH_OBSENSOR "Include obsensor support (Orbbec RGB-D modules: Astra+/Femto)" ON - VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT) OR ( UNIX AND NOT APPLE AND NOT ANDROID) + VISIBLE_IF (WIN32 AND NOT ARM AND NOT WINRT AND NOT MINGW) OR ( UNIX AND NOT APPLE AND NOT ANDROID) VERIFY HAVE_OBSENSOR) OCV_OPTION(WITH_CANN "Include CANN support" OFF VISIBLE_IF TRUE diff --git a/cmake/OpenCVCompilerOptimizations.cmake b/cmake/OpenCVCompilerOptimizations.cmake index 90ad7d783c..d45f327beb 100644 --- a/cmake/OpenCVCompilerOptimizations.cmake +++ b/cmake/OpenCVCompilerOptimizations.cmake @@ -53,6 +53,7 @@ list(APPEND CPU_ALL_OPTIMIZATIONS NEON VFPV3 FP16 NEON_DOTPROD) list(APPEND CPU_ALL_OPTIMIZATIONS MSA) list(APPEND CPU_ALL_OPTIMIZATIONS VSX VSX3) list(APPEND CPU_ALL_OPTIMIZATIONS RVV) +list(APPEND CPU_ALL_OPTIMIZATIONS LSX) list(APPEND CPU_ALL_OPTIMIZATIONS LASX) list(REMOVE_DUPLICATES CPU_ALL_OPTIMIZATIONS) @@ -331,7 +332,9 @@ if(X86 OR X86_64) elseif(ARM OR AARCH64) ocv_update(CPU_NEON_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_neon.cpp") ocv_update(CPU_FP16_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_fp16.cpp") - ocv_update(CPU_NEON_DOTPROD_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_dotprod.cpp") + ocv_update(CPU_NEON_FP16_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_neon_fp16.cpp") + ocv_update(CPU_NEON_BF16_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_neon_bf16.cpp") + ocv_update(CPU_NEON_DOTPROD_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_neon_dotprod.cpp") if(NOT AARCH64) ocv_update(CPU_KNOWN_OPTIMIZATIONS "VFPV3;NEON;FP16") if(NOT MSVC) @@ -343,12 +346,17 @@ elseif(ARM OR AARCH64) endif() ocv_update(CPU_FP16_IMPLIES "NEON") else() - ocv_update(CPU_KNOWN_OPTIMIZATIONS "NEON;FP16;NEON_DOTPROD") + ocv_update(CPU_KNOWN_OPTIMIZATIONS "NEON;FP16;NEON_DOTPROD;NEON_FP16;NEON_BF16") ocv_update(CPU_NEON_FLAGS_ON "") ocv_update(CPU_FP16_IMPLIES "NEON") ocv_update(CPU_NEON_DOTPROD_FLAGS_ON "-march=armv8.2-a+dotprod") ocv_update(CPU_NEON_DOTPROD_IMPLIES "NEON") + ocv_update(CPU_NEON_FP16_FLAGS_ON "-march=armv8.2-a+fp16") + ocv_update(CPU_NEON_FP16_IMPLIES "NEON") + ocv_update(CPU_NEON_BF16_FLAGS_ON "-march=armv8.2-a+fp16+bf16") + ocv_update(CPU_NEON_BF16_IMPLIES "NEON") set(CPU_BASELINE "NEON;FP16" CACHE STRING "${HELP_CPU_BASELINE}") + set(CPU_DISPATCH "NEON_FP16;NEON_BF16;NEON_DOTPROD" CACHE STRING "${HELP_CPU_DISPATCH}") endif() elseif(MIPS) ocv_update(CPU_MSA_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_msa.cpp") @@ -390,10 +398,16 @@ elseif(RISCV) set(CPU_BASELINE "DETECT" CACHE STRING "${HELP_CPU_BASELINE}") elseif(LOONGARCH64) + ocv_update(CPU_LSX_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_lsx.cpp") ocv_update(CPU_LASX_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_lasx.cpp") - ocv_update(CPU_KNOWN_OPTIMIZATIONS "LASX") + ocv_update(CPU_KNOWN_OPTIMIZATIONS "LSX;LASX") + ocv_update(CPU_LSX_FLAGS_ON "-mlsx") ocv_update(CPU_LASX_FLAGS_ON "-mlasx") - set(CPU_BASELINE "LASX" CACHE STRING "${HELP_CPU_BASELINE}") + if("${CPU_BASELINE_DISABLE}" STREQUAL "LASX") + set(CPU_BASELINE "LSX" CACHE STRING "${HELP_CPU_BASELINE}") + else() + set(CPU_BASELINE "LASX" CACHE STRING "${HELP_CPU_BASELINE}") + endif() endif() diff --git a/cmake/OpenCVDetectCUDA.cmake b/cmake/OpenCVDetectCUDA.cmake index 752ae0edcf..7887c798e4 100644 --- a/cmake/OpenCVDetectCUDA.cmake +++ b/cmake/OpenCVDetectCUDA.cmake @@ -355,6 +355,14 @@ if(CUDA_FOUND) set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xptxas;-dlcm=ca") endif() + # Tell NVCC the maximum number of threads to be used to execute the compilation steps in parallel + # (option --threads was introduced in version 11.2) + if(NOT CUDA_VERSION VERSION_LESS "11.2") + if(CMAKE_GENERATOR MATCHES "Visual Studio" AND NOT $ENV{CMAKE_BUILD_PARALLEL_LEVEL} STREQUAL "") + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "--threads=$ENV{CMAKE_BUILD_PARALLEL_LEVEL}") + endif() + endif() + message(STATUS "CUDA NVCC target flags: ${CUDA_NVCC_FLAGS}") OCV_OPTION(CUDA_FAST_MATH "Enable --use_fast_math for CUDA compiler " OFF) diff --git a/cmake/OpenCVFindProtobuf.cmake b/cmake/OpenCVFindProtobuf.cmake index 8835347d1d..5b1e17529f 100644 --- a/cmake/OpenCVFindProtobuf.cmake +++ b/cmake/OpenCVFindProtobuf.cmake @@ -30,8 +30,14 @@ if(BUILD_PROTOBUF) set(Protobuf_LIBRARIES "libprotobuf") set(HAVE_PROTOBUF TRUE) else() + # we still need this for command PROTOBUF_GENERATE_CPP. + set(protobuf_MODULE_COMPATIBLE ON) + unset(Protobuf_VERSION CACHE) - find_package(Protobuf QUIET) + find_package(Protobuf QUIET CONFIG) + if(NOT Protobuf_FOUND) + find_package(Protobuf QUIET) + endif() # Backwards compatibility # Define camel case versions of input variables @@ -67,6 +73,20 @@ else() endif() endif() +# See https://github.com/opencv/opencv/issues/24369 +# In Protocol Buffers v22.0 and later drops C++11 support and depends abseil-cpp. +# Details: https://protobuf.dev/news/2022-08-03/ +# And if std::text_view is in abseil-cpp requests C++17 and later. + +if(HAVE_PROTOBUF) + if(NOT (Protobuf_VERSION VERSION_LESS 22)) + if((CMAKE_CXX_STANDARD EQUAL 98) OR (CMAKE_CXX_STANDARD LESS 17)) + message(STATUS "CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} is too old to support protobuf(${Protobuf_VERSION}) and/or abseil-cpp. Use C++17 or later. Turning HAVE_PROTOBUF off") + set(HAVE_PROTOBUF FALSE) + endif() + endif() +endif() + if(HAVE_PROTOBUF AND PROTOBUF_UPDATE_FILES AND NOT COMMAND PROTOBUF_GENERATE_CPP) message(FATAL_ERROR "Can't configure protobuf dependency (BUILD_PROTOBUF=${BUILD_PROTOBUF} PROTOBUF_UPDATE_FILES=${PROTOBUF_UPDATE_FILES})") endif() @@ -74,15 +94,20 @@ endif() if(HAVE_PROTOBUF) list(APPEND CUSTOM_STATUS protobuf) if(NOT BUILD_PROTOBUF) + unset( __location) if(TARGET "${Protobuf_LIBRARIES}") get_target_property(__location "${Protobuf_LIBRARIES}" IMPORTED_LOCATION_RELEASE) if(NOT __location) get_target_property(__location "${Protobuf_LIBRARIES}" IMPORTED_LOCATION) endif() - elseif(Protobuf_LIBRARY) - set(__location "${Protobuf_LIBRARY}") - else() - set(__location "${Protobuf_LIBRARIES}") + endif() + + if(NOT __location) + if(Protobuf_LIBRARY) + set(__location "${Protobuf_LIBRARY}") + else() + set(__location "${Protobuf_LIBRARIES}") + endif() endif() endif() list(APPEND CUSTOM_STATUS_protobuf " Protobuf:" diff --git a/cmake/checks/cpu_lsx.cpp b/cmake/checks/cpu_lsx.cpp new file mode 100644 index 0000000000..86c2e5c7a6 --- /dev/null +++ b/cmake/checks/cpu_lsx.cpp @@ -0,0 +1,15 @@ +#include +#include + +int test() +{ + const float src[] = { 0.0f, 1.0f, 2.0f, 3.0f}; + v4f32 val = (v4f32)__lsx_vld((const float*)(src), 0); + return __lsx_vpickve2gr_w(__lsx_vftint_w_s(val), 3); +} + +int main() +{ + printf("%d\n", test()); + return 0; +} diff --git a/cmake/checks/cpu_neon_bf16.cpp b/cmake/checks/cpu_neon_bf16.cpp new file mode 100644 index 0000000000..c18d63c579 --- /dev/null +++ b/cmake/checks/cpu_neon_bf16.cpp @@ -0,0 +1,46 @@ +#if defined __GNUC__ && (defined __arm__ || defined __aarch64__) +#include +#include "arm_neon.h" + +/*#if defined __clang__ +#pragma clang attribute push (__attribute__((target("bf16"))), apply_to=function) +#elif defined GCC +#pragma GCC push_options +#pragma GCC target("armv8.2-a", "bf16") +#endif*/ +bfloat16x8_t vld1q_as_bf16(const float* src) +{ + float32x4_t s0 = vld1q_f32(src), s1 = vld1q_f32(src + 4); + return vcombine_bf16(vcvt_bf16_f32(s0), vcvt_bf16_f32(s1)); +} + +void vprintreg(const char* name, const float32x4_t& r) +{ + float data[4]; + vst1q_f32(data, r); + printf("%s: (%.2f, %.2f, %.2f, %.2f)\n", + name, data[0], data[1], data[2], data[3]); +} + +void test() +{ + const float src1[] = { 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f }; + const float src2[] = { 1.f, 3.f, 6.f, 10.f, 15.f, 21.f, 28.f, 36.f }; + bfloat16x8_t s1 = vld1q_as_bf16(src1), s2 = vld1q_as_bf16(src2); + float32x4_t d = vbfdotq_f32(vdupq_n_f32(0.f), s1, s2); + vprintreg("(s1[0]*s2[0] + s1[1]*s2[1], ... s1[6]*s2[6] + s1[7]*s2[7])", d); +} +/*#if defined __clang__ +#pragma clang attribute pop +#elif defined GCC +#pragma GCC pop_options +#endif*/ +#else +#error "BF16 is not supported" +#endif + +int main() +{ + test(); + return 0; +} diff --git a/cmake/checks/cpu_dotprod.cpp b/cmake/checks/cpu_neon_dotprod.cpp similarity index 100% rename from cmake/checks/cpu_dotprod.cpp rename to cmake/checks/cpu_neon_dotprod.cpp diff --git a/cmake/checks/cpu_neon_fp16.cpp b/cmake/checks/cpu_neon_fp16.cpp new file mode 100644 index 0000000000..20fbab25d5 --- /dev/null +++ b/cmake/checks/cpu_neon_fp16.cpp @@ -0,0 +1,46 @@ +#include + +#if defined __GNUC__ && (defined __arm__ || defined __aarch64__) +#include "arm_neon.h" + +float16x8_t vld1q_as_f16(const float* src) +{ + float32x4_t s0 = vld1q_f32(src), s1 = vld1q_f32(src + 4); + return vcombine_f16(vcvt_f16_f32(s0), vcvt_f16_f32(s1)); +} + +void vprintreg(const char* name, const float16x8_t& r) +{ + float data[8]; + vst1q_f32(data, vcvt_f32_f16(vget_low_f16(r))); + vst1q_f32(data + 4, vcvt_f32_f16(vget_high_f16(r))); + printf("%s: (%.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f, %.2f)\n", + name, data[0], data[1], data[2], data[3], + data[4], data[5], data[6], data[7]); +} + +void test() +{ + const float src1[] = { 1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 7.f, 8.f }; + const float src2[] = { 1.f, 3.f, 6.f, 10.f, 15.f, 21.f, 28.f, 36.f }; + float16x8_t s1 = vld1q_as_f16(src1), s2 = vld1q_as_f16(src2); + float16x8_t d = vsubq_f16(s1, s1); + d = vfmaq_laneq_f16(d, s1, s2, 0); + d = vfmaq_laneq_f16(d, s1, s2, 1); + d = vfmaq_laneq_f16(d, s1, s2, 2); + d = vfmaq_laneq_f16(d, s1, s2, 3); + d = vfmaq_laneq_f16(d, s1, s2, 4); + d = vfmaq_laneq_f16(d, s1, s2, 5); + d = vfmaq_laneq_f16(d, s1, s2, 6); + d = vfmaq_laneq_f16(d, s1, s2, 7); + vprintreg("s1*s2[0]+s1*s2[1] + ... + s1*s2[7]", d); +} +#else +#error "FP16 is not supported" +#endif + +int main() +{ + test(); + return 0; +} diff --git a/doc/py_tutorials/py_imgproc/py_transforms/py_fourier_transform/py_fourier_transform.markdown b/doc/py_tutorials/py_imgproc/py_transforms/py_fourier_transform/py_fourier_transform.markdown index 59337b1355..df12efd45c 100644 --- a/doc/py_tutorials/py_imgproc/py_transforms/py_fourier_transform/py_fourier_transform.markdown +++ b/doc/py_tutorials/py_imgproc/py_transforms/py_fourier_transform/py_fourier_transform.markdown @@ -80,7 +80,7 @@ using **np.ifft2()** function. The result, again, will be a complex number. You absolute value. @code{.py} rows, cols = img.shape -crow,ccol = rows//2 , cols//2 +crow, ccol = rows//2, cols//2 fshift[crow-30:crow+31, ccol-30:ccol+31] = 0 f_ishift = np.fft.ifftshift(fshift) img_back = np.fft.ifft2(f_ishift) @@ -146,7 +146,7 @@ content, and 0 at HF region. @code{.py} rows, cols = img.shape -crow,ccol = rows/2 , cols/2 +crow, ccol = rows//2, cols//2 # create a mask first, center square is 1, remaining all zeros mask = np.zeros((rows,cols,2),np.uint8) diff --git a/modules/3d/src/ptcloud/sampling.cpp b/modules/3d/src/ptcloud/sampling.cpp index 06ee2201b7..6920301a2c 100644 --- a/modules/3d/src/ptcloud/sampling.cpp +++ b/modules/3d/src/ptcloud/sampling.cpp @@ -277,14 +277,14 @@ int farthestPointSampling(OutputArray sampled_point_flags, InputArray input_pts, v_float32 v_last_p_y = vx_setall_f32(last_pt_y); v_float32 v_last_p_z = vx_setall_f32(last_pt_z); - for (; i <= ori_pts_size - v_float32::nlanes; i += v_float32::nlanes) + for (; i <= ori_pts_size - VTraits::vlanes(); i += VTraits::vlanes()) { - v_float32 vx_diff = v_last_p_x - vx_load(ori_pts_ptr_x + i); - v_float32 vy_diff = v_last_p_y - vx_load(ori_pts_ptr_y + i); - v_float32 vz_diff = v_last_p_z - vx_load(ori_pts_ptr_z + i); + v_float32 vx_diff = v_sub(v_last_p_x, vx_load(ori_pts_ptr_x + i)); + v_float32 vy_diff = v_sub(v_last_p_y, vx_load(ori_pts_ptr_y + i)); + v_float32 vz_diff = v_sub(v_last_p_z, vx_load(ori_pts_ptr_z + i)); v_float32 v_next_dist_square = - vx_diff * vx_diff + vy_diff * vy_diff + vz_diff * vz_diff; + v_add(v_add(v_mul(vx_diff, vx_diff), v_mul(vy_diff, vy_diff)), v_mul(vz_diff, vz_diff)); // Update the distance from the points(in C) to S float *dist_square_ptr = dist_square + i; @@ -293,9 +293,9 @@ int farthestPointSampling(OutputArray sampled_point_flags, InputArray input_pts, vx_store(dist_square_ptr, v_dist_square); // Find a point in C that is the farthest away from S and take it from C to S - if (v_check_any(v_dist_square > vx_setall_f32(max_dist_square))) + if (v_check_any(v_gt(v_dist_square, vx_setall_f32(max_dist_square)))) { - for (int m = 0; m < v_float32::nlanes; ++m) + for (int m = 0; m < VTraits::vlanes(); ++m) { if (dist_square_ptr[m] > max_dist_square) { diff --git a/modules/3d/src/rgbd/color_tsdf_functions.cpp b/modules/3d/src/rgbd/color_tsdf_functions.cpp index 44111d1880..747a813717 100644 --- a/modules/3d/src/rgbd/color_tsdf_functions.cpp +++ b/modules/3d/src/rgbd/color_tsdf_functions.cpp @@ -112,35 +112,35 @@ void integrateColorTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& // optimization of the following: //Point3f volPt = Point3f(x, y, z)*voxelSize; //Point3f camSpacePt = vol2cam * volPt; - camSpacePt += zStep; + camSpacePt = v_add(camSpacePt, zStep); - float zCamSpace = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))).get0(); + float zCamSpace = v_get0(v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt)))); if (zCamSpace <= 0.f) continue; - v_float32x4 camPixVec = camSpacePt / v_setall_f32(zCamSpace); + v_float32x4 camPixVec = v_div(camSpacePt, v_setall_f32(zCamSpace)); v_float32x4 projected = v_muladd(camPixVec, vfxy, vcxy); // leave only first 2 lanes - projected = v_reinterpret_as_f32(v_reinterpret_as_u32(projected) & - v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0)); + projected = v_reinterpret_as_f32(v_and(v_reinterpret_as_u32(projected), + v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0))); depthType v; // bilinearly interpolate depth at projected { const v_float32x4& pt = projected; // check coords >= 0 and < imgSize - v_uint32x4 limits = v_reinterpret_as_u32(pt < v_setzero_f32()) | - v_reinterpret_as_u32(pt >= upLimits); - limits = limits | v_rotate_right<1>(limits); - if (limits.get0()) + v_uint32x4 limits = v_or(v_reinterpret_as_u32(v_lt(pt, v_setzero_f32())), + v_reinterpret_as_u32(v_ge(pt, upLimits))); + limits = v_or(limits, v_rotate_right<1>(limits)); + if (v_get0(limits)) continue; // xi, yi = floor(pt) v_int32x4 ip = v_floor(pt); v_int32x4 ipshift = ip; - int xi = ipshift.get0(); + int xi = v_get0(ipshift); ipshift = v_rotate_right<1>(ipshift); - int yi = ipshift.get0(); + int yi = v_get0(ipshift); const depthType* row0 = depth[yi + 0]; const depthType* row1 = depth[yi + 1]; @@ -154,17 +154,17 @@ void integrateColorTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& // assume correct depth is positive // don't fix missing data - if (v_check_all(vall > v_setzero_f32())) + if (v_check_all( v_gt(vall, v_setzero_f32()))) { - v_float32x4 t = pt - v_cvt_f32(ip); - float tx = t.get0(); + v_float32x4 t = v_sub(pt, v_cvt_f32(ip)); + float tx = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - v_float32x4 ty = v_setall_f32(t.get0()); + v_float32x4 ty = v_setall_f32(v_get0(t)); // vx is y-interpolated between rows 0 and 1 - v_float32x4 vx = v001 + ty * (v101 - v001); - float v0 = vx.get0(); + v_float32x4 vx = v_add(v001, v_mul(ty, v_sub(v101, v001))); + float v0 = v_get0(vx); vx = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vx))); - float v1 = vx.get0(); + float v1 = v_get0(vx); v = v0 + tx * (v1 - v0); } else @@ -172,8 +172,8 @@ void integrateColorTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& } // norm(camPixVec) produces double which is too slow - int _u = (int)projected.get0(); - int _v = (int)v_rotate_right<1>(projected).get0(); + int _u = (int)v_get0(projected); + int _v = (int)v_get0(v_rotate_right<1>(projected)); if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows)) continue; @@ -330,21 +330,21 @@ inline float interpolateColorVoxel(const Mat& volume, { // tx, ty, tz = floor(p) v_int32x4 ip = v_floor(p); - v_float32x4 t = p - v_cvt_f32(ip); - float tx = t.get0(); + v_float32x4 t = v_sub(p, v_cvt_f32(ip)); + float tx = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - float ty = t.get0(); + float ty = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - float tz = t.get0(); + float tz = v_get0(t); int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; const RGBTsdfVoxel* volData = volume.ptr(); - int ix = ip.get0(); + int ix = v_get0(ip); ip = v_rotate_right<1>(ip); - int iy = ip.get0(); + int iy = v_get0(ip); ip = v_rotate_right<1>(ip); - int iz = ip.get0(); + int iz = v_get0(ip); int coordBase = ix * xdim + iy * ydim + iz * zdim; @@ -354,15 +354,15 @@ inline float interpolateColorVoxel(const Mat& volume, v_float32x4 v0246 = tsdfToFloat_INTR(v_int32x4(vx[0], vx[2], vx[4], vx[6])); v_float32x4 v1357 = tsdfToFloat_INTR(v_int32x4(vx[1], vx[3], vx[5], vx[7])); - v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246))); v_float32x4 v00_10 = vxx; v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); - v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); - float v0 = v0_1.get0(); + v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10))); + float v0 = v_get0(v0_1); v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); - float v1 = v0_1.get0(); + float v1 = v_get0(v0_1); return v0 + tx * (v1 - v0); } @@ -419,27 +419,27 @@ inline v_float32x4 getNormalColorVoxel(const Mat& volume, const Vec4i& volDims, const Vec8i& neighbourCoords, const Point3i volResolution, const v_float32x4& p) { - if (v_check_any(p < v_float32x4(1.f, 1.f, 1.f, 0.f)) || - v_check_any(p >= v_float32x4((float)(volResolution.x - 2), + if (v_check_any(v_lt(p, v_float32x4(1.f, 1.f, 1.f, 0.f))) || + v_check_any(v_ge(p, v_float32x4((float)(volResolution.x - 2), (float)(volResolution.y - 2), - (float)(volResolution.z - 2), 1.f)) + (float)(volResolution.z - 2), 1.f))) ) return nanv; v_int32x4 ip = v_floor(p); - v_float32x4 t = p - v_cvt_f32(ip); - float tx = t.get0(); + v_float32x4 t = v_sub(p, v_cvt_f32(ip)); + float tx = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - float ty = t.get0(); + float ty = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - float tz = t.get0(); + float tz = v_get0(t); const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; const RGBTsdfVoxel* volData = volume.ptr(); - int ix = ip.get0(); ip = v_rotate_right<1>(ip); - int iy = ip.get0(); ip = v_rotate_right<1>(ip); - int iz = ip.get0(); + int ix = v_get0(ip); ip = v_rotate_right<1>(ip); + int iy = v_get0(ip); ip = v_rotate_right<1>(ip); + int iz = v_get0(ip); int coordBase = ix * xdim + iy * ydim + iz * zdim; @@ -457,23 +457,23 @@ inline v_float32x4 getNormalColorVoxel(const Mat& volume, v_float32x4 v0246(vx[0], vx[2], vx[4], vx[6]); v_float32x4 v1357(vx[1], vx[3], vx[5], vx[7]); - v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246))); v_float32x4 v00_10 = vxx; v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); - v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); - float v0 = v0_1.get0(); + v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10))); + float v0 = v_get0(v0_1); v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); - float v1 = v0_1.get0(); + float v1 = v_get0(v0_1); nv = v0 + tx * (v1 - v0); } v_float32x4 n = v_load_aligned(an); - v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(n * n))); + v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(v_mul(n, n)))); - return Norm.get0() < 0.0001f ? nanv : n / Norm; + return v_get0(Norm) < 0.0001f ? nanv : v_div(n, Norm); } inline Point3f getNormalColorVoxel(const Mat& volume, @@ -544,15 +544,15 @@ inline float interpolateColor(float tx, float ty, float tz, float vx[8]) v_float32x4 v0246, v1357; v_load_deinterleave(vx, v0246, v1357); - v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246))); v_float32x4 v00_10 = vxx; v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); - v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); - float v0 = v0_1.get0(); + v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10))); + float v0 = v_get0(v0_1); v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); - float v1 = v0_1.get0(); + float v1 = v_get0(v0_1); return v0 + tx * (v1 - v0); } @@ -579,10 +579,10 @@ inline v_float32x4 getColorVoxel(const Mat& volume, const Vec4i& volDims, const Vec8i& neighbourCoords, const Point3i volResolution, const float voxelSizeInv, const v_float32x4& p) { - if (v_check_any(p < v_float32x4(1.f, 1.f, 1.f, 0.f)) || - v_check_any(p >= v_float32x4((float)(volResolution.x - 2), + if (v_check_any(v_lt(p, v_float32x4(1.f, 1.f, 1.f, 0.f))) || + v_check_any(v_ge(p, v_float32x4((float)(volResolution.x - 2), (float)(volResolution.y - 2), - (float)(volResolution.z - 2), 1.f)) + (float)(volResolution.z - 2), 1.f))) ) return nanv; @@ -591,9 +591,9 @@ inline v_float32x4 getColorVoxel(const Mat& volume, const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; const RGBTsdfVoxel* volData = volume.ptr(); - int ix = ip.get0(); ip = v_rotate_right<1>(ip); - int iy = ip.get0(); ip = v_rotate_right<1>(ip); - int iz = ip.get0(); + int ix = v_get0(ip); ip = v_rotate_right<1>(ip); + int iy = v_get0(ip); ip = v_rotate_right<1>(ip); + int iz = v_get0(ip); int coordBase = ix * xdim + iy * ydim + iz * zdim; float CV_DECL_ALIGNED(16) rgb[4]; @@ -608,12 +608,12 @@ inline v_float32x4 getColorVoxel(const Mat& volume, } v_float32x4 vsi(voxelSizeInv, voxelSizeInv, voxelSizeInv, voxelSizeInv); - v_float32x4 ptVox = p * vsi; + v_float32x4 ptVox = v_mul(p, vsi); v_int32x4 iptVox = v_floor(ptVox); - v_float32x4 t = ptVox - v_cvt_f32(iptVox); - float tx = t.get0(); t = v_rotate_right<1>(t); - float ty = t.get0(); t = v_rotate_right<1>(t); - float tz = t.get0(); + v_float32x4 t = v_sub(ptVox, v_cvt_f32(iptVox)); + float tx = v_get0(t); t = v_rotate_right<1>(t); + float ty = v_get0(t); t = v_rotate_right<1>(t); + float tz = v_get0(t); rgb[0] = interpolateColor(tx, ty, tz, r); rgb[1] = interpolateColor(tx, ty, tz, g); rgb[2] = interpolateColor(tx, ty, tz, b); @@ -783,21 +783,21 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c // get direction through pixel in volume space: // 1. reproject (x, y) on projecting plane where z = 1.f - v_float32x4 planed = (v_float32x4((float)x, (float)y, 0.f, 0.f) - vcxy) * vfxy; + v_float32x4 planed = v_mul(v_sub(v_float32x4((float)x, (float)y, 0.f, 0.f), vcxy), vfxy); planed = v_combine_low(planed, v_float32x4(1.f, 0.f, 0.f, 0.f)); // 2. rotate to volume space planed = v_matmuladd(planed, camRot0, camRot1, camRot2, v_setzero_f32()); // 3. normalize - v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(planed * planed))); - v_float32x4 dir = planed * invNorm; + v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(v_mul(planed, planed)))); + v_float32x4 dir = v_mul(planed, invNorm); // compute intersection of ray with all six bbox planes - v_float32x4 rayinv = v_setall_f32(1.f) / dir; + v_float32x4 rayinv = v_div(v_setall_f32(1.f), dir); // div by zero should be eliminated by these products - v_float32x4 tbottom = rayinv * (boxDown - orig); - v_float32x4 ttop = rayinv * (boxUp - orig); + v_float32x4 tbottom = v_mul(rayinv, v_sub(boxDown, orig)); + v_float32x4 ttop = v_mul(rayinv, v_sub(boxUp, orig)); // re-order intersections to find smallest and largest on each axis v_float32x4 minAx = v_min(ttop, tbottom); @@ -818,14 +818,14 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c if (tmin < tmax) { // interpolation optimized a little - orig *= invVoxelSize; - dir *= invVoxelSize; + orig = v_mul(orig, invVoxelSize); + dir = v_mul(dir, invVoxelSize); int xdim = volDims[0]; int ydim = volDims[1]; int zdim = volDims[2]; - v_float32x4 rayStep = dir * v_setall_f32(tstep); - v_float32x4 next = (orig + dir * v_setall_f32(tmin)); + v_float32x4 rayStep = v_mul(dir, v_setall_f32(tstep)); + v_float32x4 next = v_add(orig, v_mul(dir, v_setall_f32(tmin))); float f = interpolateColorVoxel(volume, volDims, neighbourCoords, next); float fnext = f; @@ -834,11 +834,11 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c int nSteps = cvFloor((tmax - tmin) / tstep); for (; steps < nSteps; steps++) { - next += rayStep; + next = v_add(next, rayStep); v_int32x4 ip = v_round(next); - int ix = ip.get0(); ip = v_rotate_right<1>(ip); - int iy = ip.get0(); ip = v_rotate_right<1>(ip); - int iz = ip.get0(); + int ix = v_get0(ip); ip = v_rotate_right<1>(ip); + int iy = v_get0(ip); ip = v_rotate_right<1>(ip); + int iz = v_get0(ip); int coord = ix * xdim + iy * ydim + iz * zdim; fnext = tsdfToFloat(volume.at(coord).tsdf); @@ -858,7 +858,7 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c // linearly interpolate t between two f values if (f > 0.f && fnext < 0.f) { - v_float32x4 tp = next - rayStep; + v_float32x4 tp = v_sub(next, rayStep); float ft = interpolateColorVoxel(volume, volDims, neighbourCoords, tp); float ftdt = interpolateColorVoxel(volume, volDims, neighbourCoords, next); float ts = tmin + tstep * (steps - ft / (ftdt - ft)); @@ -866,7 +866,7 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c // avoid division by zero if (!cvIsNaN(ts) && !cvIsInf(ts)) { - v_float32x4 pv = (orig + dir * v_setall_f32(ts)); + v_float32x4 pv = v_add(orig, v_mul(dir, v_setall_f32(ts))); v_float32x4 nv = getNormalColorVoxel(volume, volDims, neighbourCoords, volResolution, pv); v_float32x4 cv = getColorVoxel(volume, volDims, neighbourCoords, volResolution, voxelSizeInv, pv); @@ -876,7 +876,7 @@ void raycastColorTsdfVolumeUnit(const VolumeSettings &settings, const Matx44f &c //convert pv and nv to camera space normal = v_matmuladd(nv, volRot0, volRot1, volRot2, v_setzero_f32()); // interpolation optimized a little - point = v_matmuladd(pv * v_float32x4(voxelSize, voxelSize, voxelSize, 1.f), + point = v_matmuladd(v_mul(pv, v_float32x4(voxelSize, voxelSize, voxelSize, 1.f)), volRot0, volRot1, volRot2, volTrans); } } diff --git a/modules/3d/src/rgbd/hash_tsdf_functions.cpp b/modules/3d/src/rgbd/hash_tsdf_functions.cpp index 5d30a79622..d364d64a6a 100644 --- a/modules/3d/src/rgbd/hash_tsdf_functions.cpp +++ b/modules/3d/src/rgbd/hash_tsdf_functions.cpp @@ -630,15 +630,15 @@ inline float interpolate(float tx, float ty, float tz, float vx[8]) v_float32x4 v0246, v1357; v_load_deinterleave(vx, v0246, v1357); - v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246))); v_float32x4 v00_10 = vxx; v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); - v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); - float v0 = v0_1.get0(); + v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10))); + float v0 = v_get0(v0_1); v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); - float v1 = v0_1.get0(); + float v1 = v_get0(v0_1); return v0 + tx * (v1 - v0); } @@ -816,9 +816,9 @@ Point3f getNormalVoxel( v_float32x4 czp0 = v_lut(vals, idxzp + 0); v_float32x4 czp1 = v_lut(vals, idxzp + 4); v_float32x4 czn0 = v_lut(vals, idxzn + 0); v_float32x4 czn1 = v_lut(vals, idxzn + 4); - v_float32x4 cxv0 = cxn0 - cxp0; v_float32x4 cxv1 = cxn1 - cxp1; - v_float32x4 cyv0 = cyn0 - cyp0; v_float32x4 cyv1 = cyn1 - cyp1; - v_float32x4 czv0 = czn0 - czp0; v_float32x4 czv1 = czn1 - czp1; + v_float32x4 cxv0 = v_sub(cxn0, cxp0); v_float32x4 cxv1 = v_sub(cxn1, cxp1); + v_float32x4 cyv0 = v_sub(cyn0, cyp0); v_float32x4 cyv1 = v_sub(cyn1, cyp1); + v_float32x4 czv0 = v_sub(czn0, czp0); v_float32x4 czv1 = v_sub(czn1, czp1); v_store(cxv + 0, cxv0); v_store(cxv + 4, cxv1); v_store(cyv + 0, cyv0); v_store(cyv + 4, cyv1); @@ -959,9 +959,9 @@ Point3f ocl_getNormalVoxel( v_float32x4 czp0 = v_lut(vals, idxzp + 0); v_float32x4 czp1 = v_lut(vals, idxzp + 4); v_float32x4 czn0 = v_lut(vals, idxzn + 0); v_float32x4 czn1 = v_lut(vals, idxzn + 4); - v_float32x4 cxv0 = cxn0 - cxp0; v_float32x4 cxv1 = cxn1 - cxp1; - v_float32x4 cyv0 = cyn0 - cyp0; v_float32x4 cyv1 = cyn1 - cyp1; - v_float32x4 czv0 = czn0 - czp0; v_float32x4 czv1 = czn1 - czp1; + v_float32x4 cxv0 = v_sub(cxn0, cxp0); v_float32x4 cxv1 = v_sub(cxn1, cxp1); + v_float32x4 cyv0 = v_sub(cyn0, cyp0); v_float32x4 cyv1 = v_sub(cyn1, cyp1); + v_float32x4 czv0 = v_sub(czn0, czp0); v_float32x4 czv1 = v_sub(czn1, czp1); v_store(cxv + 0, cxv0); v_store(cxv + 4, cxv1); v_store(cyv + 0, cyv0); v_store(cyv + 4, cyv1); diff --git a/modules/3d/src/rgbd/odometry_functions.cpp b/modules/3d/src/rgbd/odometry_functions.cpp index 687aafe2cf..96d1c2fe80 100644 --- a/modules/3d/src/rgbd/odometry_functions.cpp +++ b/modules/3d/src/rgbd/odometry_functions.cpp @@ -1007,7 +1007,7 @@ static inline v_float32x4 crossProduct(const v_float32x4& a, const v_float32x4& v_float32x4 ayzx, azxy, byzx, bzxy; getCrossPerm(a, ayzx, azxy); getCrossPerm(b, byzx, bzxy); - return ayzx * bzxy - azxy * byzx; + return v_sub(v_mul(ayzx, bzxy), v_mul(azxy, byzx)); } #else static inline bool fastCheck(const Point3f& p) @@ -1082,11 +1082,11 @@ struct GetAbInvoker : ParallelLoopBody //find correspondence by projecting the point v_float32x4 oldCoords; - float pz = (v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(newP))).get0()); + float pz = v_get0(v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(newP)))); // x, y, 0, 0 - oldCoords = v_muladd(newP / v_setall_f32(pz), vfxy, vcxy); + oldCoords = v_muladd(v_div(newP, v_setall_f32(pz)), vfxy, vcxy); - if (!v_check_all((oldCoords >= v_setzero_f32()) & (oldCoords < vframe))) + if (!v_check_all(v_and(v_ge(oldCoords, v_setzero_f32()), v_lt(oldCoords, vframe)))) continue; // bilinearly interpolate oldPts and oldNrm under oldCoords point @@ -1094,12 +1094,12 @@ struct GetAbInvoker : ParallelLoopBody v_float32x4 oldN; { v_int32x4 ixy = v_floor(oldCoords); - v_float32x4 txy = oldCoords - v_cvt_f32(ixy); - int xi = ixy.get0(); - int yi = v_rotate_right<1>(ixy).get0(); - v_float32x4 tx = v_setall_f32(txy.get0()); + v_float32x4 txy = v_sub(oldCoords, v_cvt_f32(ixy)); + int xi = v_get0(ixy); + int yi = v_get0(v_rotate_right<1>(ixy)); + v_float32x4 tx = v_setall_f32(v_get0(txy)); txy = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(txy))); - v_float32x4 ty = v_setall_f32(txy.get0()); + v_float32x4 ty = v_setall_f32(v_get0(txy)); const float* prow0 = (const float*)oldPts[yi + 0]; const float* prow1 = (const float*)oldPts[yi + 1]; @@ -1122,23 +1122,23 @@ struct GetAbInvoker : ParallelLoopBody // NaN check is done later - v_float32x4 p0 = p00 + tx * (p01 - p00); - v_float32x4 p1 = p10 + tx * (p11 - p10); - oldP = p0 + ty * (p1 - p0); + v_float32x4 p0 = v_add(p00, v_mul(tx, v_sub(p01, p00))); + v_float32x4 p1 = v_add(p10, v_mul(tx, v_sub(p11, p10))); + oldP = v_add(p0, v_mul(ty, v_sub(p1, p0))); - v_float32x4 n0 = n00 + tx * (n01 - n00); - v_float32x4 n1 = n10 + tx * (n11 - n10); - oldN = n0 + ty * (n1 - n0); + v_float32x4 n0 = v_add(n00, v_mul(tx, v_sub(n01, n00))); + v_float32x4 n1 = v_add(n10, v_mul(tx, v_sub(n11, n10))); + oldN = v_add(n0, v_mul(ty, v_sub(n1, n0))); } bool oldPNcheck = fastCheck(oldP, oldN); //filter by distance - v_float32x4 diff = newP - oldP; - bool distCheck = !(v_reduce_sum(diff * diff) > sqThresh); + v_float32x4 diff = v_sub(newP, oldP); + bool distCheck = !(v_reduce_sum(v_mul(diff, diff)) > sqThresh); //filter by angle - bool angleCheck = !(abs(v_reduce_sum(newN * oldN)) < cosThresh); + bool angleCheck = !(abs(v_reduce_sum(v_mul(newN, oldN))) < cosThresh); if (!(oldPNcheck && distCheck && angleCheck)) continue; @@ -1146,17 +1146,17 @@ struct GetAbInvoker : ParallelLoopBody // build point-wise vector ab = [ A | b ] v_float32x4 VxNv = crossProduct(newP, oldN); Point3f VxN; - VxN.x = VxNv.get0(); - VxN.y = v_reinterpret_as_f32(v_extract<1>(v_reinterpret_as_u32(VxNv), v_setzero_u32())).get0(); - VxN.z = v_reinterpret_as_f32(v_extract<2>(v_reinterpret_as_u32(VxNv), v_setzero_u32())).get0(); + VxN.x = v_get0(VxNv); + VxN.y = v_get0(v_reinterpret_as_f32(v_extract<1>(v_reinterpret_as_u32(VxNv), v_setzero_u32()))); + VxN.z = v_get0(v_reinterpret_as_f32(v_extract<2>(v_reinterpret_as_u32(VxNv), v_setzero_u32()))); - float dotp = -v_reduce_sum(oldN * diff); + float dotp = -v_reduce_sum(v_mul(oldN, diff)); // build point-wise upper-triangle matrix [ab^T * ab] w/o last row // which is [A^T*A | A^T*b] // and gather sum - v_float32x4 vd = VxNv | v_float32x4(0, 0, 0, dotp); + v_float32x4 vd = v_or(VxNv, v_float32x4(0, 0, 0, dotp)); v_float32x4 n = oldN; v_float32x4 nyzx; { diff --git a/modules/3d/src/rgbd/tsdf_functions.cpp b/modules/3d/src/rgbd/tsdf_functions.cpp index 4aa1e37f93..3fdb481dda 100644 --- a/modules/3d/src/rgbd/tsdf_functions.cpp +++ b/modules/3d/src/rgbd/tsdf_functions.cpp @@ -144,35 +144,35 @@ void integrateTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& volu // optimization of the following: //Point3f volPt = Point3f(x, y, z)*voxelSize; //Point3f camSpacePt = vol2cam * volPt; - camSpacePt += zStep; + camSpacePt = v_add(camSpacePt, zStep); - float zCamSpace = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt))).get0(); + float zCamSpace = v_get0(v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(camSpacePt)))); if (zCamSpace <= 0.f) continue; - v_float32x4 camPixVec = camSpacePt / v_setall_f32(zCamSpace); + v_float32x4 camPixVec = v_div(camSpacePt, v_setall_f32(zCamSpace)); v_float32x4 projected = v_muladd(camPixVec, vfxy, vcxy); // leave only first 2 lanes - projected = v_reinterpret_as_f32(v_reinterpret_as_u32(projected) & - v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0)); + projected = v_reinterpret_as_f32(v_and(v_reinterpret_as_u32(projected), + v_uint32x4(0xFFFFFFFF, 0xFFFFFFFF, 0, 0))); depthType v; // bilinearly interpolate depth at projected { const v_float32x4& pt = projected; // check coords >= 0 and < imgSize - v_uint32x4 limits = v_reinterpret_as_u32(pt < v_setzero_f32()) | - v_reinterpret_as_u32(pt >= upLimits); - limits = limits | v_rotate_right<1>(limits); - if (limits.get0()) + v_uint32x4 limits = v_or(v_reinterpret_as_u32(v_lt(pt, v_setzero_f32())), + v_reinterpret_as_u32(v_ge(pt, upLimits))); + limits = v_or(limits, v_rotate_right<1>(limits)); + if (v_get0(limits)) continue; // xi, yi = floor(pt) v_int32x4 ip = v_floor(pt); v_int32x4 ipshift = ip; - int xi = ipshift.get0(); + int xi = v_get0(ipshift); ipshift = v_rotate_right<1>(ipshift); - int yi = ipshift.get0(); + int yi = v_get0(ipshift); const depthType* row0 = depth[yi + 0]; const depthType* row1 = depth[yi + 1]; @@ -186,17 +186,17 @@ void integrateTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& volu // assume correct depth is positive // don't fix missing data - if (v_check_all(vall > v_setzero_f32())) + if (v_check_all(v_gt(vall, v_setzero_f32()))) { - v_float32x4 t = pt - v_cvt_f32(ip); - float tx = t.get0(); + v_float32x4 t = v_sub(pt, v_cvt_f32(ip)); + float tx = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - v_float32x4 ty = v_setall_f32(t.get0()); + v_float32x4 ty = v_setall_f32(v_get0(t)); // vx is y-interpolated between rows 0 and 1 - v_float32x4 vx = v001 + ty * (v101 - v001); - float v0 = vx.get0(); + v_float32x4 vx = v_add(v001, v_mul(ty, v_sub(v101, v001))); + float v0 = v_get0(vx); vx = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vx))); - float v1 = vx.get0(); + float v1 = v_get0(vx); v = v0 + tx * (v1 - v0); } else @@ -204,8 +204,8 @@ void integrateTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& volu } // norm(camPixVec) produces double which is too slow - int _u = (int)projected.get0(); - int _v = (int)v_rotate_right<1>(projected).get0(); + int _u = (int)v_get0(projected); + int _v = (int)v_get0(v_rotate_right<1>(projected)); if (!(_u >= 0 && _u < depth.cols && _v >= 0 && _v < depth.rows)) continue; float pixNorm = pixNorms.at(_v, _u); @@ -408,21 +408,21 @@ inline float interpolateTsdfVoxel(const Mat& volume, { // tx, ty, tz = floor(p) v_int32x4 ip = v_floor(p); - v_float32x4 t = p - v_cvt_f32(ip); - float tx = t.get0(); + v_float32x4 t = v_sub(p, v_cvt_f32(ip)); + float tx = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - float ty = t.get0(); + float ty = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - float tz = t.get0(); + float tz = v_get0(t); int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; const TsdfVoxel* volData = volume.ptr(); - int ix = ip.get0(); + int ix = v_get0(ip); ip = v_rotate_right<1>(ip); - int iy = ip.get0(); + int iy = v_get0(ip); ip = v_rotate_right<1>(ip); - int iz = ip.get0(); + int iz = v_get0(ip); int coordBase = ix * xdim + iy * ydim + iz * zdim; @@ -432,15 +432,15 @@ inline float interpolateTsdfVoxel(const Mat& volume, v_float32x4 v0246 = tsdfToFloat_INTR(v_int32x4(vx[0], vx[2], vx[4], vx[6])); v_float32x4 v1357 = tsdfToFloat_INTR(v_int32x4(vx[1], vx[3], vx[5], vx[7])); - v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246))); v_float32x4 v00_10 = vxx; v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); - v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); - float v0 = v0_1.get0(); + v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10))); + float v0 = v_get0(v0_1); v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); - float v1 = v0_1.get0(); + float v1 = v_get0(v0_1); return v0 + tx * (v1 - v0); } @@ -495,27 +495,27 @@ inline v_float32x4 getNormalVoxel( const Mat& volume, const Vec4i& volDims, const Vec8i& neighbourCoords, const Point3i volResolution, const v_float32x4& p) { - if (v_check_any(p < v_float32x4(1.f, 1.f, 1.f, 0.f)) || - v_check_any(p >= v_float32x4((float)(volResolution.x - 2), + if (v_check_any(v_lt(p, v_float32x4(1.f, 1.f, 1.f, 0.f))) || + v_check_any(v_ge(p, v_float32x4((float)(volResolution.x - 2), (float)(volResolution.y - 2), - (float)(volResolution.z - 2), 1.f)) + (float)(volResolution.z - 2), 1.f))) ) return nanv; v_int32x4 ip = v_floor(p); - v_float32x4 t = p - v_cvt_f32(ip); - float tx = t.get0(); + v_float32x4 t = v_sub(p, v_cvt_f32(ip)); + float tx = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - float ty = t.get0(); + float ty = v_get0(t); t = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(t))); - float tz = t.get0(); + float tz = v_get0(t); const int xdim = volDims[0], ydim = volDims[1], zdim = volDims[2]; const TsdfVoxel* volData = volume.ptr(); - int ix = ip.get0(); ip = v_rotate_right<1>(ip); - int iy = ip.get0(); ip = v_rotate_right<1>(ip); - int iz = ip.get0(); + int ix = v_get0(ip); ip = v_rotate_right<1>(ip); + int iy = v_get0(ip); ip = v_rotate_right<1>(ip); + int iz = v_get0(ip); int coordBase = ix * xdim + iy * ydim + iz * zdim; @@ -533,23 +533,23 @@ inline v_float32x4 getNormalVoxel( const Mat& volume, v_float32x4 v0246(vx[0], vx[2], vx[4], vx[6]); v_float32x4 v1357(vx[1], vx[3], vx[5], vx[7]); - v_float32x4 vxx = v0246 + v_setall_f32(tz) * (v1357 - v0246); + v_float32x4 vxx = v_add(v0246, v_mul(v_setall_f32(tz), v_sub(v1357, v0246))); v_float32x4 v00_10 = vxx; v_float32x4 v01_11 = v_reinterpret_as_f32(v_rotate_right<1>(v_reinterpret_as_u32(vxx))); - v_float32x4 v0_1 = v00_10 + v_setall_f32(ty) * (v01_11 - v00_10); - float v0 = v0_1.get0(); + v_float32x4 v0_1 = v_add(v00_10, v_mul(v_setall_f32(ty), v_sub(v01_11, v00_10))); + float v0 = v_get0(v0_1); v0_1 = v_reinterpret_as_f32(v_rotate_right<2>(v_reinterpret_as_u32(v0_1))); - float v1 = v0_1.get0(); + float v1 = v_get0(v0_1); nv = v0 + tx * (v1 - v0); } v_float32x4 n = v_load_aligned(an); - v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(n * n))); + v_float32x4 Norm = v_sqrt(v_setall_f32(v_reduce_sum(v_mul(n, n)))); - return Norm.get0() < 0.0001f ? nanv : n / Norm; + return v_get0(Norm) < 0.0001f ? nanv : v_div(n, Norm); } inline Point3f getNormalVoxel( const Mat& volume, @@ -703,21 +703,21 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera // get direction through pixel in volume space: // 1. reproject (x, y) on projecting plane where z = 1.f - v_float32x4 planed = (v_float32x4((float)x, (float)y, 0.f, 0.f) - vcxy) * vfxy; + v_float32x4 planed = v_mul(v_sub(v_float32x4((float)x, (float)y, 0.f, 0.f), vcxy), vfxy); planed = v_combine_low(planed, v_float32x4(1.f, 0.f, 0.f, 0.f)); // 2. rotate to volume space planed = v_matmuladd(planed, camRot0, camRot1, camRot2, v_setzero_f32()); // 3. normalize - v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(planed * planed))); - v_float32x4 dir = planed * invNorm; + v_float32x4 invNorm = v_invsqrt(v_setall_f32(v_reduce_sum(v_mul(planed, planed)))); + v_float32x4 dir = v_mul(planed, invNorm); // compute intersection of ray with all six bbox planes - v_float32x4 rayinv = v_setall_f32(1.f) / dir; + v_float32x4 rayinv = v_div(v_setall_f32(1.f), dir); // div by zero should be eliminated by these products - v_float32x4 tbottom = rayinv * (boxDown - orig); - v_float32x4 ttop = rayinv * (boxUp - orig); + v_float32x4 tbottom = v_mul(rayinv, v_sub(boxDown, orig)); + v_float32x4 ttop = v_mul(rayinv, v_sub(boxUp, orig)); // re-order intersections to find smallest and largest on each axis v_float32x4 minAx = v_min(ttop, tbottom); @@ -738,14 +738,14 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera if (tmin < tmax) { // interpolation optimized a little - orig *= invVoxelSize; - dir *= invVoxelSize; + orig = v_mul(orig, invVoxelSize); + dir = v_mul(dir, invVoxelSize); int xdim = volDims[0]; int ydim = volDims[1]; int zdim = volDims[2]; - v_float32x4 rayStep = dir * v_setall_f32(tstep); - v_float32x4 next = (orig + dir * v_setall_f32(tmin)); + v_float32x4 rayStep = v_mul(dir, v_setall_f32(tstep)); + v_float32x4 next = v_add(orig, v_mul(dir, v_setall_f32(tmin))); float f = interpolateTsdfVoxel(volume, volDims, neighbourCoords, next); float fnext = f; @@ -754,11 +754,11 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera int nSteps = cvFloor((tmax - tmin) / tstep); for (; steps < nSteps; steps++) { - next += rayStep; + next = v_add(next, rayStep); v_int32x4 ip = v_round(next); - int ix = ip.get0(); ip = v_rotate_right<1>(ip); - int iy = ip.get0(); ip = v_rotate_right<1>(ip); - int iz = ip.get0(); + int ix = v_get0(ip); ip = v_rotate_right<1>(ip); + int iy = v_get0(ip); ip = v_rotate_right<1>(ip); + int iz = v_get0(ip); int coord = ix * xdim + iy * ydim + iz * zdim; fnext = tsdfToFloat(volume.at(coord).tsdf); @@ -778,7 +778,7 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera // linearly interpolate t between two f values if (f > 0.f && fnext < 0.f) { - v_float32x4 tp = next - rayStep; + v_float32x4 tp = v_sub(next, rayStep); float ft = interpolateTsdfVoxel(volume, volDims, neighbourCoords, tp); float ftdt = interpolateTsdfVoxel(volume, volDims, neighbourCoords, next); float ts = tmin + tstep * (steps - ft / (ftdt - ft)); @@ -786,7 +786,7 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera // avoid division by zero if (!cvIsNaN(ts) && !cvIsInf(ts)) { - v_float32x4 pv = (orig + dir * v_setall_f32(ts)); + v_float32x4 pv = v_add(orig, v_mul(dir, v_setall_f32(ts))); v_float32x4 nv = getNormalVoxel(volume, volDims, neighbourCoords, volResolution, pv); if (!isNaN(nv)) @@ -794,7 +794,7 @@ void raycastTsdfVolumeUnit(const VolumeSettings& settings, const Matx44f& camera //convert pv and nv to camera space normal = v_matmuladd(nv, volRot0, volRot1, volRot2, v_setzero_f32()); // interpolation optimized a little - point = v_matmuladd(pv * v_float32x4(voxelSize, voxelSize, voxelSize, 1.f), + point = v_matmuladd(v_mul(pv, v_float32x4(voxelSize, voxelSize, voxelSize, 1.f)), volRot0, volRot1, volRot2, volTrans); } } diff --git a/modules/3d/src/rgbd/tsdf_functions.hpp b/modules/3d/src/rgbd/tsdf_functions.hpp index 4f9e2f7676..c2ef7010e9 100644 --- a/modules/3d/src/rgbd/tsdf_functions.hpp +++ b/modules/3d/src/rgbd/tsdf_functions.hpp @@ -45,7 +45,7 @@ typedef Vec VecRGBTsdfVoxel; inline v_float32x4 tsdfToFloat_INTR(const v_int32x4& num) { v_float32x4 num128 = v_setall_f32(-1.f / 128.f); - return v_cvt_f32(num) * num128; + return v_mul(v_cvt_f32(num), num128); } #endif diff --git a/modules/3d/src/rgbd/utils.hpp b/modules/3d/src/rgbd/utils.hpp index 334ef73db8..fb9b2b1022 100644 --- a/modules/3d/src/rgbd/utils.hpp +++ b/modules/3d/src/rgbd/utils.hpp @@ -71,7 +71,7 @@ inline bool isNaN(cv::Point3f p) #if USE_INTRINSICS static inline bool isNaN(const cv::v_float32x4& p) { - return cv::v_check_any(p != p); + return cv::v_check_any(v_ne(p, p)); } #endif diff --git a/modules/3d/src/usac/estimator.cpp b/modules/3d/src/usac/estimator.cpp index 493f5d4c3b..eb0fe4a461 100644 --- a/modules/3d/src/usac/estimator.cpp +++ b/modules/3d/src/usac/estimator.cpp @@ -645,11 +645,11 @@ public: v_float32 v_d = vx_setall_f32(d); float* errors_cache_ptr = errors_cache.data(); - for (; i <= pts_cnt - v_float32::nlanes; i += v_float32::nlanes) + for (; i <= pts_cnt - VTraits::vlanes(); i += VTraits::vlanes()) { - v_float32 v_error = v_a * vx_load(pts_ptr_x + i) + v_b * vx_load(pts_ptr_y + i) - + v_c * vx_load(pts_ptr_z + i) + v_d; - v_store(errors_cache_ptr + i, v_error * v_error); + v_float32 v_error = v_add(v_add(v_mul(v_a, vx_load(pts_ptr_x + i)), v_mul(v_b, vx_load(pts_ptr_y + i))), + v_add(v_mul(v_c, vx_load(pts_ptr_z + i)), v_d)); + v_store(errors_cache_ptr + i, v_mul(v_error, v_error)); } #endif for (; i < pts_cnt; ++i) { @@ -725,16 +725,16 @@ public: v_float32 v_radius = vx_setall_f32(radius); float* errors_cache_ptr = errors_cache.data(); - for (; i <= pts_cnt - v_float32::nlanes; i += v_float32::nlanes) + for (; i <= pts_cnt - VTraits::vlanes(); i += VTraits::vlanes()) { - v_float32 v_diff_x = v_center_x - vx_load(pts_ptr_x + i); - v_float32 v_diff_y = v_center_y - vx_load(pts_ptr_y + i); - v_float32 v_diff_z = v_center_z - vx_load(pts_ptr_z + i); + v_float32 v_diff_x = v_sub(v_center_x, vx_load(pts_ptr_x + i)); + v_float32 v_diff_y = v_sub(v_center_y, vx_load(pts_ptr_y + i)); + v_float32 v_diff_z = v_sub(v_center_z, vx_load(pts_ptr_z + i)); - v_float32 v_distance_from_center = v_sqrt(v_diff_x * v_diff_x + - v_diff_y * v_diff_y + v_diff_z * v_diff_z); - v_float32 v_diff_dist = v_distance_from_center - v_radius; - v_store(errors_cache_ptr + i, v_diff_dist * v_diff_dist); + v_float32 v_distance_from_center = v_sqrt(v_add(v_add(v_mul(v_diff_x, v_diff_x), + v_mul(v_diff_y, v_diff_y)), v_mul(v_diff_z, v_diff_z))); + v_float32 v_diff_dist = v_sub(v_distance_from_center, v_radius); + v_store(errors_cache_ptr + i, v_mul(v_diff_dist, v_diff_dist)); } #endif for (; i < pts_cnt; ++i) diff --git a/modules/core/include/opencv2/core/cv_cpu_dispatch.h b/modules/core/include/opencv2/core/cv_cpu_dispatch.h index 3235b6317e..de7b84b82a 100644 --- a/modules/core/include/opencv2/core/cv_cpu_dispatch.h +++ b/modules/core/include/opencv2/core/cv_cpu_dispatch.h @@ -172,6 +172,11 @@ # define CV_MSA 1 #endif +#ifdef CV_CPU_COMPILE_LSX +# include +# define CV_LSX 1 +#endif + #ifdef CV_CPU_COMPILE_LASX # include # define CV_LASX 1 @@ -376,6 +381,10 @@ struct VZeroUpperGuard { # define CV_RVV 0 #endif +#ifndef CV_LSX +# define CV_LSX 0 +#endif + #ifndef CV_LASX # define CV_LASX 0 #endif diff --git a/modules/core/include/opencv2/core/cv_cpu_helper.h b/modules/core/include/opencv2/core/cv_cpu_helper.h index 41fc9d50fa..daca592e65 100644 --- a/modules/core/include/opencv2/core/cv_cpu_helper.h +++ b/modules/core/include/opencv2/core/cv_cpu_helper.h @@ -525,6 +525,27 @@ #endif #define __CV_CPU_DISPATCH_CHAIN_RVV(fn, args, mode, ...) CV_CPU_CALL_RVV(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__)) +#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_LSX +# define CV_TRY_LSX 1 +# define CV_CPU_FORCE_LSX 1 +# define CV_CPU_HAS_SUPPORT_LSX 1 +# define CV_CPU_CALL_LSX(fn, args) return (cpu_baseline::fn args) +# define CV_CPU_CALL_LSX_(fn, args) return (opt_LSX::fn args) +#elif !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_DISPATCH_COMPILE_LSX +# define CV_TRY_LSX 1 +# define CV_CPU_FORCE_LSX 0 +# define CV_CPU_HAS_SUPPORT_LSX (cv::checkHardwareSupport(CV_CPU_LSX)) +# define CV_CPU_CALL_LSX(fn, args) if (CV_CPU_HAS_SUPPORT_LSX) return (opt_LSX::fn args) +# define CV_CPU_CALL_LSX_(fn, args) if (CV_CPU_HAS_SUPPORT_LSX) return (opt_LSX::fn args) +#else +# define CV_TRY_LSX 0 +# define CV_CPU_FORCE_LSX 0 +# define CV_CPU_HAS_SUPPORT_LSX 0 +# define CV_CPU_CALL_LSX(fn, args) +# define CV_CPU_CALL_LSX_(fn, args) +#endif +#define __CV_CPU_DISPATCH_CHAIN_LSX(fn, args, mode, ...) CV_CPU_CALL_LSX(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__)) + #if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_LASX # define CV_TRY_LASX 1 # define CV_CPU_FORCE_LASX 1 diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index 8a2d7d3935..11457451ef 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -269,6 +269,8 @@ namespace cv { #define CV_CPU_NEON 100 #define CV_CPU_NEON_DOTPROD 101 +#define CV_CPU_NEON_FP16 102 +#define CV_CPU_NEON_BF16 103 #define CV_CPU_MSA 150 @@ -279,7 +281,8 @@ namespace cv { #define CV_CPU_RVV 210 -#define CV_CPU_LASX 230 +#define CV_CPU_LSX 230 +#define CV_CPU_LASX 231 // CPU features groups #define CV_CPU_AVX512_SKX 256 @@ -328,6 +331,8 @@ enum CpuFeatures { CPU_NEON = 100, CPU_NEON_DOTPROD = 101, + CPU_NEON_FP16 = 102, + CPU_NEON_BF16 = 103, CPU_MSA = 150, @@ -338,7 +343,8 @@ enum CpuFeatures { CPU_RVV = 210, - CPU_LASX = 230, + CPU_LSX = 230, + CPU_LASX = 231, CPU_AVX512_SKX = 256, //!< Skylake-X with AVX-512F/CD/BW/DQ/VL CPU_AVX512_COMMON = 257, //!< Common instructions AVX-512F/CD for all CPUs that support AVX-512 diff --git a/modules/core/include/opencv2/core/hal/intrin.hpp b/modules/core/include/opencv2/core/hal/intrin.hpp index b607b8b3b0..cf0711fa61 100644 --- a/modules/core/include/opencv2/core/hal/intrin.hpp +++ b/modules/core/include/opencv2/core/hal/intrin.hpp @@ -206,7 +206,7 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE; # undef CV_RVV #endif -#if (CV_SSE2 || CV_NEON || CV_VSX || CV_MSA || CV_WASM_SIMD || CV_RVV071) && !defined(CV_FORCE_SIMD128_CPP) +#if (CV_SSE2 || CV_NEON || CV_VSX || CV_MSA || CV_WASM_SIMD || CV_RVV071 || CV_LSX) && !defined(CV_FORCE_SIMD128_CPP) #define CV__SIMD_FORWARD 128 #include "opencv2/core/hal/intrin_forward.hpp" #endif @@ -242,6 +242,10 @@ using namespace CV_CPU_OPTIMIZATION_HAL_NAMESPACE; #include "opencv2/core/hal/intrin_rvv.hpp" #endif +#elif CV_LSX && !defined(CV_FORCE_SIMD128_CPP) + +#include "opencv2/core/hal/intrin_lsx.hpp" + #elif CV_LASX #if !defined(CV_FORCE_SIMD128_CPP) #define CV_FORCE_SIMD128_CPP 1 @@ -738,7 +742,7 @@ namespace CV__SIMD_NAMESPACE { /** @brief SIMD processing state cleanup call */ inline void vx_cleanup() { VXPREFIX(_cleanup)(); } -#if !CV_SIMD_SCALABLE +#if !CV_SIMD_SCALABLE && !(CV_NEON && !defined(CV_FORCE_SIMD128_CPP)) // Compatibility layer template struct VTraits { @@ -1166,6 +1170,74 @@ namespace CV__SIMD_NAMESPACE { #endif //!CV_SIMD_SCALABLE +#if (CV_NEON /* || CV_others */) && !defined(CV_FORCE_SIMD128_CPP) +// Compatibility layer for the backend that cleaned up. + #define OPENCV_HAL_WRAP_BIN_OP_ADDSUB(_Tpvec) \ + template \ + inline _Tpvec v_add(const _Tpvec& f1, const _Tpvec& f2, const Args&... vf) { \ + return v_add(v_add(f1, f2), vf...); \ + } + + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint8) + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint16) + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint32) + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint64) + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int8) + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int16) + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int32) + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int64) + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float32) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float64) + #endif + + #define OPENCV_HAL_WRAP_BIN_OP_MUL(_Tpvec) \ + template \ + inline _Tpvec v_mul(const _Tpvec& f1, const _Tpvec& f2, const Args&... vf) { \ + return v_mul(v_mul(f1, f2), vf...); \ + } + OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint8) + OPENCV_HAL_WRAP_BIN_OP_MUL(v_int8) + OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint16) + OPENCV_HAL_WRAP_BIN_OP_MUL(v_uint32) + OPENCV_HAL_WRAP_BIN_OP_MUL(v_int16) + OPENCV_HAL_WRAP_BIN_OP_MUL(v_int32) + OPENCV_HAL_WRAP_BIN_OP_MUL(v_float32) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_BIN_OP_MUL(v_float64) + #endif + + #define OPENCV_HAL_WRAP_EXTRACT(_Tpvec) \ + inline typename VTraits<_Tpvec>::lane_type v_extract_highest(const _Tpvec& v) \ + { \ + return v_extract_n::nlanes-1>(v); \ + } + + OPENCV_HAL_WRAP_EXTRACT(v_uint8) + OPENCV_HAL_WRAP_EXTRACT(v_int8) + OPENCV_HAL_WRAP_EXTRACT(v_uint16) + OPENCV_HAL_WRAP_EXTRACT(v_int16) + OPENCV_HAL_WRAP_EXTRACT(v_uint32) + OPENCV_HAL_WRAP_EXTRACT(v_int32) + OPENCV_HAL_WRAP_EXTRACT(v_uint64) + OPENCV_HAL_WRAP_EXTRACT(v_int64) + OPENCV_HAL_WRAP_EXTRACT(v_float32) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_EXTRACT(v_float64) + #endif + + #define OPENCV_HAL_WRAP_BROADCAST(_Tpvec) \ + inline _Tpvec v_broadcast_highest(const _Tpvec& v) \ + { \ + return v_broadcast_element::nlanes-1>(v); \ + } + + OPENCV_HAL_WRAP_BROADCAST(v_uint32) + OPENCV_HAL_WRAP_BROADCAST(v_int32) + OPENCV_HAL_WRAP_BROADCAST(v_float32) + +#endif //CV_NEON + //! @cond IGNORED // backward compatibility diff --git a/modules/core/include/opencv2/core/hal/intrin_lasx.hpp b/modules/core/include/opencv2/core/hal/intrin_lasx.hpp index 37f2e3f81d..5214e80743 100644 --- a/modules/core/include/opencv2/core/hal/intrin_lasx.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_lasx.hpp @@ -96,54 +96,22 @@ inline __m256d _v256_setall_pd(double f64) inline __m256i _lasx_packus_h(const __m256i& a, const __m256i& b) { - __m256i u8min = __lasx_xvreplgr2vr_h(0); - __m256i u8max = __lasx_xvreplgr2vr_h(255); - __m256i sat_a = __lasx_xvmax_h(a, u8min); - sat_a = __lasx_xvmin_h(sat_a, u8max); - __m256i sat_b = __lasx_xvmax_h(b, u8min); - sat_b = __lasx_xvmin_h(sat_b, u8max); - __m256i byteIndex = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, - 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); - return __lasx_xvshuf_b(sat_b, sat_a, byteIndex); + return __lasx_xvssrarni_bu_h(b, a, 0); } inline __m256i _lasx_packs_h(const __m256i& a, const __m256i& b) { - __m256i s8min = __lasx_xvreplgr2vr_h(-128); - __m256i s8max = __lasx_xvreplgr2vr_h(127); - __m256i sat_a = __lasx_xvmax_h(a, s8min); - sat_a = __lasx_xvmin_h(sat_a, s8max); - __m256i sat_b = __lasx_xvmax_h(b, s8min); - sat_b = __lasx_xvmin_h(sat_b, s8max); - __m256i byteIndex = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, - 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30); - return __lasx_xvshuf_b(sat_b, sat_a, byteIndex); + return __lasx_xvssrarni_b_h(b, a, 0); } inline __m256i _lasx_packus_w(const __m256i& a, const __m256i& b) { - __m256i u16min = __lasx_xvreplgr2vr_w(0); - __m256i u16max = __lasx_xvreplgr2vr_w(0xffff); - __m256i sat_a = __lasx_xvmax_w(a, u16min); - sat_a = __lasx_xvmin_w(sat_a, u16max); - __m256i sat_b = __lasx_xvmax_w(b, u16min); - sat_b = __lasx_xvmin_w(sat_b, u16max); - __m256i hwordIndex = _v256_setr_h(0, 2, 4, 6, 8, 10, 12, 14, - 0, 2, 4, 6, 8, 10, 12, 14); - return __lasx_xvshuf_h(hwordIndex, sat_b, sat_a); + return __lasx_xvssrarni_hu_w(b, a, 0); } inline __m256i _lasx_packs_w(const __m256i& a, const __m256i& b) { - __m256i s16min = __lasx_xvreplgr2vr_w(-0x8000); - __m256i s16max = __lasx_xvreplgr2vr_w(0x7fff); - __m256i sat_a = __lasx_xvmax_w(a, s16min); - sat_a = __lasx_xvmin_w(sat_a, s16max); - __m256i sat_b = __lasx_xvmax_w(b, s16min); - sat_b = __lasx_xvmin_w(sat_b, s16max); - __m256i hwordIndex = _v256_setr_h(0, 2, 4, 6, 8, 10, 12, 14, - 0, 2, 4, 6, 8, 10, 12, 14); - return __lasx_xvshuf_h(hwordIndex, sat_b, sat_a); + return __lasx_xvssrarni_h_w(b, a, 0); } inline __m256i _v256_combine(const __m128i& lo, const __m128i& hi) @@ -191,7 +159,7 @@ inline _Tpvec v256_permute4x64(const _Tpvec& a) { return _Tpvec(_v256_permute4x64(a.val)); } inline __m128i _v256_extract_high(const __m256i& v) -{ __m256i temp256i = __lasx_xvpermi_q(v, v, 0x31); +{ __m256i temp256i = __lasx_xvpermi_d(v, 0x4E); return *((__m128i*)&temp256i); } inline __m128 _v256_extract_high(const __m256& v) @@ -211,10 +179,7 @@ inline __m128d _v256_extract_low(const __m256d& v) inline __m256i _v256_packs_epu32(const __m256i& a, const __m256i& b) { - const __m256i maxv = __lasx_xvreplgr2vr_w(65535); - __m256i am = __lasx_xvmin_wu(a, maxv); - __m256i bm = __lasx_xvmin_wu(b, maxv); - return _lasx_packus_w(am, bm); + return __lasx_xvssrlrni_hu_w(b, a, 0); } template @@ -869,14 +834,11 @@ OPENCV_HAL_IMPL_LASX_BIN_FUNC(v_mul_wrap, v_int16x16, __lasx_xvmul_h) inline v_uint8x32 v_mul_wrap(const v_uint8x32& a, const v_uint8x32& b) { - __m256i ad = __lasx_xvsrai_h(a.val, 8); - __m256i bd = __lasx_xvsrai_h(b.val, 8); - __m256i p0 = __lasx_xvmul_h(a.val, b.val); - __m256i p1 = __lasx_xvslli_h(__lasx_xvmul_h(ad, bd), 8); - - const __m256i b01 = __lasx_xvreplgr2vr_w(0xFF00FF00); - return v_uint8x32(__lasx_xvbitsel_v(p0, p1, b01)); + __m256i p0 = __lasx_xvmulwev_h_bu(a.val, b.val); + __m256i p1 = __lasx_xvmulwod_h_bu(a.val, b.val); + return v_uint8x32(__lasx_xvpackev_b(p1, p0)); } + inline v_int8x32 v_mul_wrap(const v_int8x32& a, const v_int8x32& b) { return v_reinterpret_as_s8(v_mul_wrap(v_reinterpret_as_u8(a), v_reinterpret_as_u8(b))); @@ -963,14 +925,7 @@ inline v_uint16x16 v_mul_hi(const v_uint16x16& a, const v_uint16x16& b) { return OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint16x16, v_int16x16, h, __lasx_xvsra_h) OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint32x8, v_int32x8, w, __lasx_xvsra_w) - -inline __m256i _v256_srai_dx(const __m256i a, const __m256i shift) -{ - __m256i d = __lasx_xvreplgr2vr_d((int64)1 << 63); - __m256i r = __lasx_xvsrl_d(__lasx_xvadd_d(a, d), shift); - return __lasx_xvsub_d(r, __lasx_xvsrl_d(d, shift)); -} -OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, _v256_srai_dx) +OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, __lasx_xvsra_d) /** Bitwise logic **/ @@ -979,7 +934,7 @@ OPENCV_HAL_IMPL_LASX_SHIFT_OP(v_uint64x4, v_int64x4, d, _v256_srai_dx) OPENCV_HAL_IMPL_LASX_BIN_OP(|, _Tpvec, __lasx_xvor_##suffix) \ OPENCV_HAL_IMPL_LASX_BIN_OP(^, _Tpvec, __lasx_xvxor_##suffix) \ inline _Tpvec operator ~ (const _Tpvec& a) \ - { return _Tpvec(__lasx_xvxor_##suffix(a.val, not_const)); } + { return _Tpvec(__lasx_xvnori_b(a.val, 0)); } OPENCV_HAL_IMPL_LASX_LOGIC_OP(v_uint8x32, v, __lasx_xvreplgr2vr_w(-1)) OPENCV_HAL_IMPL_LASX_LOGIC_OP(v_int8x32, v, __lasx_xvreplgr2vr_w(-1)) @@ -1224,11 +1179,9 @@ inline v_int8x32 v_reverse(const v_int8x32 &a) inline v_uint16x16 v_reverse(const v_uint16x16 &a) { - static const __m256i perm = _v256_setr_b( - 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, - 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); - __m256i vec = __lasx_xvshuf_b(a.val, a.val, perm); - return v_uint16x16(__lasx_xvpermi_q(vec, vec, 1)); + __m256i vec = __lasx_xvshuf4i_h(a.val, 0x1B); + vec = __lasx_xvshuf4i_w(vec, 0x4E); + return v_uint16x16(__lasx_xvpermi_d(vec, 0x4E)); } inline v_int16x16 v_reverse(const v_int16x16 &a) @@ -1236,8 +1189,8 @@ inline v_int16x16 v_reverse(const v_int16x16 &a) inline v_uint32x8 v_reverse(const v_uint32x8 &a) { - static const __m256i perm = _v256_setr_w(7, 6, 5, 4, 3, 2, 1, 0); - return v_uint32x8(__lasx_xvperm_w(a.val, perm)); + __m256i vec = __lasx_xvshuf4i_w(a.val, 0x1B); + return v_uint32x8(__lasx_xvpermi_d(vec, 0x4E)); } inline v_int32x8 v_reverse(const v_int32x8 &a) @@ -1266,17 +1219,19 @@ inline unsigned v_reduce_sum(const v_uint8x32& a) __m256i t1 = __lasx_xvhaddw_hu_bu(a.val, a.val); __m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1); __m256i t3 = __lasx_xvhaddw_du_wu(t2, t2); - return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]); + __m256i t4 = __lasx_xvhaddw_qu_du(t3, t3); + return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]); } + inline int v_reduce_sum(const v_int8x32& a) { __m256i t1 = __lasx_xvhaddw_h_b(a.val, a.val); __m256i t2 = __lasx_xvhaddw_w_h(t1, t1); __m256i t3 = __lasx_xvhaddw_d_w(t2, t2); - return (int)(((v4i64)t3)[0]+((v4i64)t3)[1]+((v4i64)t3)[2]+((v4i64)t3)[3]); + __m256i t4 = __lasx_xvhaddw_q_d(t3, t3); + return (int)(((v8i32)t4)[0]+((v8i32)t4)[4]); } - #define OPENCV_HAL_IMPL_LASX_REDUCE_32(_Tpvec, sctype, func, intrin) \ inline sctype v_reduce_##func(const _Tpvec& a) \ { \ @@ -1344,7 +1299,8 @@ OPENCV_HAL_IMPL_LASX_REDUCE_FLT(max, __lsx_vfmax_s) inline int v_reduce_sum(const v_int32x8& a) { __m256i t1 = __lasx_xvhaddw_d_w(a.val, a.val); - return (int)(((v4i64)t1)[0]+((v4i64)t1)[1]+((v4i64)t1)[2]+((v4i64)t1)[3]); + __m256i t2 = __lasx_xvhaddw_q_d(t1, t1); + return (int)(((v8i32)t2)[0]+((v8i32)t2)[4]); } inline unsigned v_reduce_sum(const v_uint32x8& a) @@ -1367,13 +1323,13 @@ inline float v_reduce_sum(const v_float32x8& a) inline uint64 v_reduce_sum(const v_uint64x4& a) { - uint64 *pa = (uint64*)&a; - return pa[0] + pa[1] + pa[2] + pa[3]; + __m256i t0 = __lasx_xvhaddw_qu_du(a.val, a.val); + return (uint64)(((v4u64)t0)[0] + ((v4u64)t0)[2]); } inline int64 v_reduce_sum(const v_int64x4& a) { - int64 *pa = (int64*)&a; - return pa[0] + pa[1] + pa[2] + pa[3]; + __m256i t0 = __lasx_xvhaddw_q_d(a.val, a.val); + return (int64)(((v4i64)t0)[0] + ((v4i64)t0)[2]); } inline double v_reduce_sum(const v_float64x4& a) { @@ -1406,7 +1362,8 @@ inline unsigned v_reduce_sad(const v_uint8x32& a, const v_uint8x32& b) __m256i t1 = __lasx_xvhaddw_hu_bu(t0, t0); __m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1); __m256i t3 = __lasx_xvhaddw_du_wu(t2, t2); - return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]); + __m256i t4 = __lasx_xvhaddw_qu_du(t3, t3); + return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]); } inline unsigned v_reduce_sad(const v_int8x32& a, const v_int8x32& b) { @@ -1414,7 +1371,8 @@ inline unsigned v_reduce_sad(const v_int8x32& a, const v_int8x32& b) __m256i t1 = __lasx_xvhaddw_hu_bu(t0, t0); __m256i t2 = __lasx_xvhaddw_wu_hu(t1, t1); __m256i t3 = __lasx_xvhaddw_du_wu(t2, t2); - return (unsigned)(((v4u64)t3)[0]+((v4u64)t3)[1]+((v4u64)t3)[2]+((v4u64)t3)[3]); + __m256i t4 = __lasx_xvhaddw_qu_du(t3, t3); + return (unsigned)(((v8u32)t4)[0]+((v8u32)t4)[4]); } inline unsigned v_reduce_sad(const v_uint16x16& a, const v_uint16x16& b) { @@ -1445,36 +1403,13 @@ inline float v_reduce_sad(const v_float32x8& a, const v_float32x8& b) /** Popcount **/ inline v_uint8x32 v_popcount(const v_uint8x32& a) -{ - __m256i _popcnt_table = _v256_setr_b(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4); - __m256i _popcnt_mask = __lasx_xvreplgr2vr_b(0x0F); - return v_uint8x32(__lasx_xvadd_b(__lasx_xvshuf_b(_popcnt_table, _popcnt_table, __lasx_xvand_v(a.val, _popcnt_mask)), - __lasx_xvshuf_b(_popcnt_table, _popcnt_table, __lasx_xvand_v(__lasx_xvsrli_h(a.val, 4), _popcnt_mask)))); -} +{ return v_uint8x32(__lasx_xvpcnt_b(a.val)); } inline v_uint16x16 v_popcount(const v_uint16x16& a) -{ - v_uint8x32 p = v_popcount(v_reinterpret_as_u8(a)); - p += v_rotate_right<1>(p); - return v_reinterpret_as_u16(p) & v_uint16x16(__lasx_xvreplgr2vr_h(0x00ff)); -} +{ return v_uint16x16(__lasx_xvpcnt_h(a.val)); } inline v_uint32x8 v_popcount(const v_uint32x8& a) -{ - v_uint8x32 p = v_popcount(v_reinterpret_as_u8(a)); - p += v_rotate_right<1>(p); - p += v_rotate_right<2>(p); - return v_reinterpret_as_u32(p) & v_uint32x8(__lasx_xvreplgr2vr_w(0x000000ff)); -} +{ return v_uint32x8(__lasx_xvpcnt_w(a.val)); } inline v_uint64x4 v_popcount(const v_uint64x4& a) -{ - v_uint8x32 atemp = v_popcount(v_reinterpret_as_u8(a)); - uint8_t *pa = (uint8_t*)&atemp; - uint64 v[4]; - for (int i = 0; i < 4; ++i) { - v[i] = pa[i*8] + pa[i*8+1] + pa[i*8+2] + pa[i*8+3] + pa[i*8+4] + pa[i*8+5] + pa[i*8+6] + pa[i*8+7]; - } - return v_uint64x4(v[0], v[1], v[2], v[3]); -} +{ return v_uint64x4(__lasx_xvpcnt_d(a.val)); } inline v_uint8x32 v_popcount(const v_int8x32& a) { return v_popcount(v_reinterpret_as_u8(a)); } inline v_uint16x16 v_popcount(const v_int16x16& a) @@ -1500,10 +1435,9 @@ OPENCV_HAL_IMPL_REINTERPRET_INT(double, int64) inline int v_signmask(const v_int8x32& a) { - int mask = 0; - int8_t *pa = (int8_t*)&a; - for( int i = 0; i < 32; i++ ) - mask |= (reinterpret_int(pa[i]) < 0) << i; + __m256i result = __lasx_xvmskltz_b(a.val); + int mask = __lasx_xvpickve2gr_w(result, 0); + mask |= (__lasx_xvpickve2gr_w(result, 4) << 16); return mask; } inline int v_signmask(const v_uint8x32& a) @@ -1516,10 +1450,9 @@ inline int v_signmask(const v_uint16x16& a) inline int v_signmask(const v_int32x8& a) { - int mask = 0; - int *pa = (int*)&a; - for( int i = 0; i < 8; i++ ) - mask |= (pa[i] < 0) << i; + __m256i result = __lasx_xvmskltz_w(a.val); + int mask = __lasx_xvpickve2gr_w(result, 0); + mask |= (__lasx_xvpickve2gr_w(result, 4) << 4); return mask; } inline int v_signmask(const v_uint32x8& a) @@ -1527,10 +1460,9 @@ inline int v_signmask(const v_uint32x8& a) inline int v_signmask(const v_int64x4& a) { - int mask = 0; - int64 *pa = (int64*)&a; - for( int i = 0; i < 4; i++ ) - mask |= (pa[i] < 0) << i; + __m256i result = __lasx_xvmskltz_d(a.val); + int mask = __lasx_xvpickve2gr_d(result, 0); + mask |= (__lasx_xvpickve2gr_w(result, 4) << 2); return mask; } inline int v_signmask(const v_uint64x4& a) @@ -1592,7 +1524,7 @@ OPENCV_HAL_IMPL_LASX_MULADD(v_float64x4, d) inline v_int32x8 v_fma(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c) { - return a * b + c; + return v_int32x8(__lasx_xvmadd_w(c.val, a.val, b.val)); } inline v_int32x8 v_muladd(const v_int32x8& a, const v_int32x8& b, const v_int32x8& c) @@ -1601,17 +1533,10 @@ inline v_int32x8 v_muladd(const v_int32x8& a, const v_int32x8& b, const v_int32x } inline v_float32x8 v_invsqrt(const v_float32x8& x) -{ - v_float32x8 half = x * v_float32x8(0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5); - v_float32x8 t = v_float32x8(__lasx_xvfrsqrt_s(x.val)); - t *= v_float32x8(1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5) - ((t * t) * half); - return t; -} +{ return v_float32x8(__lasx_xvfrsqrt_s(x.val)); } inline v_float64x4 v_invsqrt(const v_float64x4& x) -{ - return v_float64x4(1., 1., 1., 1.) / v_sqrt(x); -} +{ return v_float64x4(__lasx_xvfrsqrt_d(x.val)); } /** Absolute values **/ #define OPENCV_HAL_IMPL_LASX_ABS(_Tpvec, suffix) \ @@ -1629,28 +1554,18 @@ inline v_float64x4 v_abs(const v_float64x4& x) /** Absolute difference **/ inline v_uint8x32 v_absdiff(const v_uint8x32& a, const v_uint8x32& b) -{ return v_add_wrap(a - b, b - a); } +{ return (v_uint8x32)__lasx_xvabsd_bu(a.val, b.val); } inline v_uint16x16 v_absdiff(const v_uint16x16& a, const v_uint16x16& b) -{ return v_add_wrap(a - b, b - a); } +{ return (v_uint16x16)__lasx_xvabsd_hu(a.val, b.val); } inline v_uint32x8 v_absdiff(const v_uint32x8& a, const v_uint32x8& b) -{ return v_max(a, b) - v_min(a, b); } +{ return (v_uint32x8)__lasx_xvabsd_wu(a.val, b.val); } inline v_uint8x32 v_absdiff(const v_int8x32& a, const v_int8x32& b) -{ - v_int8x32 d = v_sub_wrap(a, b); - v_int8x32 m = a < b; - return v_reinterpret_as_u8(v_sub_wrap(d ^ m, m)); -} - +{ return (v_uint8x32)__lasx_xvabsd_b(a.val, b.val); } inline v_uint16x16 v_absdiff(const v_int16x16& a, const v_int16x16& b) -{ return v_reinterpret_as_u16(v_sub_wrap(v_max(a, b), v_min(a, b))); } - +{ return (v_uint16x16)__lasx_xvabsd_h(a.val, b.val); } inline v_uint32x8 v_absdiff(const v_int32x8& a, const v_int32x8& b) -{ - v_int32x8 d = a - b; - v_int32x8 m = a < b; - return v_reinterpret_as_u32((d ^ m) - m); -} +{ return (v_uint32x8)__lasx_xvabsd_w(a.val, b.val); } inline v_float32x8 v_absdiff(const v_float32x8& a, const v_float32x8& b) { return v_abs(a - b); } @@ -1740,28 +1655,8 @@ inline v_float64x4 v_cvt_f64_high(const v_float32x8& a) return v_float64x4(__lasx_xvfcvtl_d_s((__m256)ahigh)); } -// from (Mysticial and wim) https://stackoverflow.com/q/41144668 inline v_float64x4 v_cvt_f64(const v_int64x4& v) -{ - // constants encoded as floating-point - __m256i magic_i_lo = __lasx_xvreplgr2vr_d(0x4330000000000000); - __m256i magic_i_hi32 = __lasx_xvreplgr2vr_d(0x4530000080000000); - __m256i magic_i_all = __lasx_xvreplgr2vr_d(0x4530000080100000); - __m256d magic_d_all = _lasx_256_castsi256_pd(magic_i_all); - - // Blend the 32 lowest significant bits of v with magic_int_lo - __m256i mask = _v256_set_w(0, -1, 0, -1, 0, -1, 0, -1); - __m256i v_lo = __lasx_xvbitsel_v(magic_i_lo, v.val, mask); - // Extract the 32 most significant bits of v - __m256i v_hi = __lasx_xvsrli_d(v.val, 32); - // Flip the msb of v_hi and blend with 0x45300000 - v_hi = __lasx_xvxor_v(v_hi, magic_i_hi32); - // Compute in double precision - __m256d v_hi_dbl = __lasx_xvfsub_d(_lasx_256_castsi256_pd(v_hi), magic_d_all); - // (v_hi - magic_d_all) + v_lo Do not assume associativity of floating point addition - __m256d result = __lasx_xvfadd_d(v_hi_dbl, _lasx_256_castsi256_pd(v_lo)); - return v_float64x4(result); -} +{ return v_float64x4(__lasx_xvffint_d_l(v.val)); } ////////////// Lookup table access //////////////////// @@ -1967,11 +1862,9 @@ inline v_float32x8 v_interleave_pairs(const v_float32x8& vec) inline v_int8x32 v_pack_triplets(const v_int8x32& vec) { __m256i vzero = __lasx_xvreplgr2vr_w(0); - __m256i t1 = __lasx_xvshuf_b(vec.val, vec.val, - _v256_set_d(0xffffff0f0e0d0c0a, 0x0908060504020100, 0xffffff0f0e0d0c0a, 0x0908060504020100)); - __m256i t2 = __lasx_xvshuf_b(vzero, t1, - _v256_set_d(0x1211100c0b0a0908, 0x0706050403020100, 0x1211100c0b0a0908, 0x0706050403020100)); - return v_int8x32(__lasx_xvperm_w(t2, + __m256i t1 = __lasx_xvshuf_b(vzero, vec.val, + _v256_set_d(0x1211100f0e0d0c0a, 0x0908060504020100, 0x1211100f0e0d0c0a, 0x0908060504020100)); + return v_int8x32(__lasx_xvperm_w(t1, _v256_set_d(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000))); } inline v_uint8x32 v_pack_triplets(const v_uint8x32& vec) @@ -1980,11 +1873,9 @@ inline v_uint8x32 v_pack_triplets(const v_uint8x32& vec) inline v_int16x16 v_pack_triplets(const v_int16x16& vec) { __m256i vzero = __lasx_xvreplgr2vr_w(0); - __m256i t1 = __lasx_xvshuf_b(vec.val, vec.val, - _v256_set_d(0xffff0f0e0d0c0b0a, 0x0908050403020100, 0xffff0f0e0d0c0b0a, 0x0908050403020100)); - __m256i t2 = __lasx_xvshuf_b(vzero, t1, - _v256_set_d(0x11100d0c0b0a0908, 0x0706050403020100, 0x11100d0c0b0a0908, 0x0706050403020100)); - return v_int16x16(__lasx_xvperm_w(t2, + __m256i t1 = __lasx_xvshuf_b(vzero, vec.val, + _v256_set_d(0x11100f0e0d0c0b0a, 0x0908050403020100, 0x11100f0e0d0c0b0a, 0x0908050403020100)); + return v_int16x16(__lasx_xvperm_w(t1, _v256_set_d(0x0000000700000007, 0x0000000600000005, 0x0000000400000002, 0x0000000100000000))); } inline v_uint16x16 v_pack_triplets(const v_uint16x16& vec) @@ -2018,24 +1909,21 @@ inline v_int32x8 v_dotprod(const v_int16x16& a, const v_int16x16& b, const v_int inline v_int64x4 v_dotprod(const v_int32x8& a, const v_int32x8& b) { __m256i even = __lasx_xvmulwev_d_w(a.val, b.val); - __m256i odd = __lasx_xvmulwod_d_w(a.val, b.val); - return v_int64x4(__lasx_xvadd_d(even, odd)); + return v_int64x4(__lasx_xvmaddwod_d_w(even, a.val, b.val)); } inline v_int64x4 v_dotprod(const v_int32x8& a, const v_int32x8& b, const v_int64x4& c) -{ return v_dotprod(a, b) + c; } +{ + __m256i even = __lasx_xvmaddwev_d_w(c.val, a.val, b.val); + return v_int64x4(__lasx_xvmaddwod_d_w(even, a.val, b.val)); +} // 8 >> 32 inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b) { - __m256i even_m = __lasx_xvreplgr2vr_w(0xFF00FF00); - __m256i even_a = __lasx_xvbitsel_v(a.val, __lasx_xvreplgr2vr_d(0), even_m); - __m256i odd_a = __lasx_xvsrli_h(a.val, 8); - - __m256i even_b = __lasx_xvbitsel_v(b.val, __lasx_xvreplgr2vr_d(0), even_m); - __m256i odd_b = __lasx_xvsrli_h(b.val, 8); - - __m256i prod0 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(even_a, even_b), __lasx_xvmulwod_w_h(even_a, even_b)); - __m256i prod1 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(odd_a, odd_b),__lasx_xvmulwod_w_h(odd_a, odd_b)); + __m256i even = __lasx_xvmulwev_h_bu(a.val, b.val); + __m256i odd = __lasx_xvmulwod_h_bu(a.val, b.val); + __m256i prod0 = __lasx_xvhaddw_wu_hu(even, even); + __m256i prod1 = __lasx_xvhaddw_wu_hu(odd, odd); return v_uint32x8(__lasx_xvadd_w(prod0, prod1)); } inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b, const v_uint32x8& c) @@ -2043,14 +1931,10 @@ inline v_uint32x8 v_dotprod_expand(const v_uint8x32& a, const v_uint8x32& b, con inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b) { - __m256i even_a = __lasx_xvsrai_h(__lasx_xvbsll_v(a.val, 1), 8); - __m256i odd_a = __lasx_xvsrai_h(a.val, 8); - - __m256i even_b = __lasx_xvsrai_h(__lasx_xvbsll_v(b.val, 1), 8); - __m256i odd_b = __lasx_xvsrai_h(b.val, 8); - - __m256i prod0 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(even_a, even_b), __lasx_xvmulwod_w_h(even_a, even_b)); - __m256i prod1 = __lasx_xvadd_w(__lasx_xvmulwev_w_h(odd_a, odd_b),__lasx_xvmulwod_w_h(odd_a, odd_b)); + __m256i even = __lasx_xvmulwev_h_b(a.val, b.val); + __m256i odd = __lasx_xvmulwod_h_b(a.val, b.val); + __m256i prod0 = __lasx_xvhaddw_w_h(even, even); + __m256i prod1 = __lasx_xvhaddw_w_h(odd, odd); return v_int32x8(__lasx_xvadd_w(prod0, prod1)); } inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b, const v_int32x8& c) @@ -2059,36 +1943,24 @@ inline v_int32x8 v_dotprod_expand(const v_int8x32& a, const v_int8x32& b, const // 16 >> 64 inline v_uint64x4 v_dotprod_expand(const v_uint16x16& a, const v_uint16x16& b) { - __m256i mullo = __lasx_xvmul_h(a.val, b.val); - __m256i mulhi = __lasx_xvmuh_hu(a.val, b.val); - __m256i mul0 = __lasx_xvilvl_h(mulhi, mullo); - __m256i mul1 = __lasx_xvilvh_h(mulhi, mullo); - - __m256i p02 = __lasx_xvbitsel_v(mul0, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0)); - __m256i p13 = __lasx_xvsrli_d(mul0, 32); - __m256i p46 = __lasx_xvbitsel_v(mul1, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0)); - __m256i p57 = __lasx_xvsrli_d(mul1, 32); - - __m256i p15_ = __lasx_xvadd_d(p02, p13); - __m256i p9d_ = __lasx_xvadd_d(p46, p57); - - return v_uint64x4(__lasx_xvadd_d( - __lasx_xvilvl_d(p9d_, p15_), - __lasx_xvilvh_d(p9d_, p15_))); + __m256i even = __lasx_xvmulwev_w_hu(a.val, b.val); + __m256i odd = __lasx_xvmulwod_w_hu(a.val, b.val); + __m256i prod0 = __lasx_xvhaddw_du_wu(even, even); + __m256i prod1 = __lasx_xvhaddw_du_wu(odd, odd); + return v_uint64x4(__lasx_xvadd_d(prod0, prod1)); } inline v_uint64x4 v_dotprod_expand(const v_uint16x16& a, const v_uint16x16& b, const v_uint64x4& c) { return v_dotprod_expand(a, b) + c; } inline v_int64x4 v_dotprod_expand(const v_int16x16& a, const v_int16x16& b) { - __m256i prod = __lasx_xvadd_w(__lasx_xvmulwev_w_h(a.val, b.val), __lasx_xvmulwod_w_h(a.val, b.val)); - __m256i sign = __lasx_xvsrai_w(prod, 31); - - __m256i lo = __lasx_xvilvl_w(sign, prod); - __m256i hi = __lasx_xvilvh_w(sign, prod); - - return v_int64x4(__lasx_xvadd_d(__lasx_xvilvl_d(hi, lo), __lasx_xvilvh_d(hi, lo))); + __m256i even = __lasx_xvmulwev_w_h(a.val, b.val); + __m256i odd = __lasx_xvmulwod_w_h(a.val, b.val); + __m256i prod0 = __lasx_xvhaddw_d_w(even, even); + __m256i prod1 = __lasx_xvhaddw_d_w(odd, odd); + return v_int64x4(__lasx_xvadd_d(prod0, prod1)); } + inline v_int64x4 v_dotprod_expand(const v_int16x16& a, const v_int16x16& b, const v_int64x4& c) { return v_dotprod_expand(a, b) + c; } @@ -2126,20 +1998,11 @@ inline v_int32x8 v_dotprod_expand_fast(const v_int8x32& a, const v_int8x32& b, c // 16 >> 64 inline v_uint64x4 v_dotprod_expand_fast(const v_uint16x16& a, const v_uint16x16& b) { - __m256i mullo = __lasx_xvmul_h(a.val, b.val); - __m256i mulhi = __lasx_xvmuh_hu(a.val, b.val); - __m256i mul0 = __lasx_xvilvl_h(mulhi, mullo); - __m256i mul1 = __lasx_xvilvh_h(mulhi, mullo); - - __m256i p02 = __lasx_xvbitsel_v(mul0, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0)); - __m256i p13 = __lasx_xvsrli_d(mul0, 32); - __m256i p46 = __lasx_xvbitsel_v(mul1, __lasx_xvreplgr2vr_d(0), _v256_set_w(-1, 0, -1, 0, -1, 0, -1, 0)); - __m256i p57 = __lasx_xvsrli_d(mul1, 32); - - __m256i p15_ = __lasx_xvadd_d(p02, p13); - __m256i p9d_ = __lasx_xvadd_d(p46, p57); - - return v_uint64x4(__lasx_xvadd_d(p15_, p9d_)); + __m256i even = __lasx_xvmulwev_w_hu(a.val, b.val); + __m256i odd = __lasx_xvmulwod_w_hu(a.val, b.val); + __m256i prod0 = __lasx_xvhaddw_du_wu(even, even); + __m256i prod1 = __lasx_xvhaddw_du_wu(odd, odd); + return v_uint64x4(__lasx_xvadd_d(__lasx_xvilvl_d(prod1, prod0), __lasx_xvilvh_d(prod1, prod0))); } inline v_uint64x4 v_dotprod_expand_fast(const v_uint16x16& a, const v_uint16x16& b, const v_uint64x4& c) { return v_dotprod_expand_fast(a, b) + c; } @@ -2261,12 +2124,7 @@ inline v_int8x32 v_pack(const v_int16x16& a, const v_int16x16& b) { return v_int8x32(_v256_shuffle_odd_64(_lasx_packs_h(a.val, b.val))); } inline v_uint8x32 v_pack(const v_uint16x16& a, const v_uint16x16& b) -{ - __m256i t = __lasx_xvreplgr2vr_h(255); - __m256i a1 = __lasx_xvmin_hu(a.val, t); - __m256i b1 = __lasx_xvmin_hu(b.val, t); - return v_uint8x32(_v256_shuffle_odd_64(_lasx_packus_h(a1, b1))); -} +{ return v_uint8x32(_v256_shuffle_odd_64(__lasx_xvssrlrni_bu_h(b.val, a.val, 0))); } inline v_uint8x32 v_pack_u(const v_int16x16& a, const v_int16x16& b) { @@ -2276,13 +2134,8 @@ inline v_uint8x32 v_pack_u(const v_int16x16& a, const v_int16x16& b) inline void v_pack_store(schar* ptr, const v_int16x16& a) { v_store_low(ptr, v_pack(a, a)); } -inline void v_pack_store(uchar* ptr, const v_uint16x16& a) -{ - const __m256i m = __lasx_xvreplgr2vr_h(255); - __m256i am = __lasx_xvmin_hu(a.val, m); - am = _v256_shuffle_odd_64(_lasx_packus_h(am, am)); - v_store_low(ptr, v_uint8x32(am)); -} +inline void v_pack_store(uchar *ptr, const v_uint16x16& a) +{ v_store_low(ptr, v_pack(a, a)); } inline void v_pack_u_store(uchar* ptr, const v_int16x16& a) { v_store_low(ptr, v_pack_u(a, a)); } @@ -2290,45 +2143,43 @@ inline void v_pack_u_store(uchar* ptr, const v_int16x16& a) template inline v_uint8x32 v_rshr_pack(const v_uint16x16& a, const v_uint16x16& b) { - // we assume that n > 0, and so the shifted 16-bit values can be treated as signed numbers. - v_uint16x16 delta = v256_setall_u16((short)(1 << (n-1))); - return v_pack_u(v_reinterpret_as_s16((a + delta) >> n), - v_reinterpret_as_s16((b + delta) >> n)); + __m256i res = __lasx_xvssrlrni_bu_h(b.val, a.val, n); + return v_uint8x32(_v256_shuffle_odd_64(res)); } template inline void v_rshr_pack_store(uchar* ptr, const v_uint16x16& a) { - v_uint16x16 delta = v256_setall_u16((short)(1 << (n-1))); - v_pack_u_store(ptr, v_reinterpret_as_s16((a + delta) >> n)); + __m256i res = __lasx_xvssrlrni_bu_h(a.val, a.val, n); + __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); } template inline v_uint8x32 v_rshr_pack_u(const v_int16x16& a, const v_int16x16& b) { - v_int16x16 delta = v256_setall_s16((short)(1 << (n-1))); - return v_pack_u((a + delta) >> n, (b + delta) >> n); + __m256i res = __lasx_xvssrarni_bu_h(b.val, a.val, n); + return v_uint8x32(_v256_shuffle_odd_64(res)); } template inline void v_rshr_pack_u_store(uchar* ptr, const v_int16x16& a) { - v_int16x16 delta = v256_setall_s16((short)(1 << (n-1))); - v_pack_u_store(ptr, (a + delta) >> n); + __m256i res = __lasx_xvssrarni_bu_h(a.val, a.val, n); + __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); } template inline v_int8x32 v_rshr_pack(const v_int16x16& a, const v_int16x16& b) { - v_int16x16 delta = v256_setall_s16((short)(1 << (n-1))); - return v_pack((a + delta) >> n, (b + delta) >> n); + __m256i res = __lasx_xvssrarni_b_h(b.val, a.val, n); + return v_int8x32(_v256_shuffle_odd_64(res)); } template inline void v_rshr_pack_store(schar* ptr, const v_int16x16& a) { - v_int16x16 delta = v256_setall_s16((short)(1 << (n-1))); - v_pack_store(ptr, (a + delta) >> n); + __m256i res = __lasx_xvssrarni_b_h(a.val, a.val, n); + __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); } // 32 @@ -2346,67 +2197,51 @@ inline void v_pack_store(short* ptr, const v_int32x8& a) inline void v_pack_store(ushort* ptr, const v_uint32x8& a) { - const __m256i m = __lasx_xvreplgr2vr_w(65535); - __m256i am = __lasx_xvmin_wu(a.val, m); - am = _v256_shuffle_odd_64(_lasx_packus_w(am, am)); - v_store_low(ptr, v_uint16x16(am)); + __m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, 0); + __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); } inline void v_pack_u_store(ushort* ptr, const v_int32x8& a) { v_store_low(ptr, v_pack_u(a, a)); } - template inline v_uint16x16 v_rshr_pack(const v_uint32x8& a, const v_uint32x8& b) -{ - // we assume that n > 0, and so the shifted 32-bit values can be treated as signed numbers. - v_uint32x8 delta = v256_setall_u32(1 << (n-1)); - return v_pack_u(v_reinterpret_as_s32((a + delta) >> n), - v_reinterpret_as_s32((b + delta) >> n)); -} +{ return v_uint16x16(_v256_shuffle_odd_64(__lasx_xvssrlrni_hu_w(b.val, a.val, n))); } template inline void v_rshr_pack_store(ushort* ptr, const v_uint32x8& a) { - v_uint32x8 delta = v256_setall_u32(1 << (n-1)); - v_pack_u_store(ptr, v_reinterpret_as_s32((a + delta) >> n)); + __m256i res = __lasx_xvssrlrni_hu_w(a.val, a.val, n); + __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); } template inline v_uint16x16 v_rshr_pack_u(const v_int32x8& a, const v_int32x8& b) -{ - v_int32x8 delta = v256_setall_s32(1 << (n-1)); - return v_pack_u((a + delta) >> n, (b + delta) >> n); -} +{ return v_uint16x16(_v256_shuffle_odd_64(__lasx_xvssrarni_hu_w(b.val, a.val, n))); } template inline void v_rshr_pack_u_store(ushort* ptr, const v_int32x8& a) { - v_int32x8 delta = v256_setall_s32(1 << (n-1)); - v_pack_u_store(ptr, (a + delta) >> n); + __m256i res = __lasx_xvssrarni_hu_w(a.val, a.val, n); + __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); } template inline v_int16x16 v_rshr_pack(const v_int32x8& a, const v_int32x8& b) -{ - v_int32x8 delta = v256_setall_s32(1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} +{ return v_int16x16(_v256_shuffle_odd_64(__lasx_xvssrarni_h_w(b.val, a.val, n))); } template inline void v_rshr_pack_store(short* ptr, const v_int32x8& a) { - v_int32x8 delta = v256_setall_s32(1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); + __m256i res = __lasx_xvssrarni_h_w(a.val, a.val, n); + __lsx_vst(_v256_extract_low(_v256_shuffle_odd_64(res)), ptr, 0); } // 64 // Non-saturating pack inline v_uint32x8 v_pack(const v_uint64x4& a, const v_uint64x4& b) { - __m256i a0 = __lasx_xvshuf4i_w(a.val, 0x08); - __m256i b0 = __lasx_xvshuf4i_w(b.val, 0x08); - __m256i ab = __lasx_xvilvl_d(b0, a0); + __m256i ab = __lasx_xvpickev_w(b.val, a.val); return v_uint32x8(_v256_shuffle_odd_64(ab)); } @@ -2424,31 +2259,19 @@ inline void v_pack_store(int* ptr, const v_int64x4& b) template inline v_uint32x8 v_rshr_pack(const v_uint64x4& a, const v_uint64x4& b) -{ - v_uint64x4 delta = v256_setall_u64((uint64)1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} +{ return v_uint32x8(_v256_shuffle_odd_64(__lasx_xvsrlrni_w_d(b.val, a.val, n))); } template inline void v_rshr_pack_store(unsigned* ptr, const v_uint64x4& a) -{ - v_uint64x4 delta = v256_setall_u64((uint64)1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); -} +{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrlrni_w_d(a.val, a.val, n)), ptr, 0); } template inline v_int32x8 v_rshr_pack(const v_int64x4& a, const v_int64x4& b) -{ - v_int64x4 delta = v256_setall_s64((int64)1 << (n-1)); - return v_pack((a + delta) >> n, (b + delta) >> n); -} +{ return v_int32x8(_v256_shuffle_odd_64(__lasx_xvsrarni_w_d(b.val, a.val, n))); } template inline void v_rshr_pack_store(int* ptr, const v_int64x4& a) -{ - v_int64x4 delta = v256_setall_s64((int64)1 << (n-1)); - v_pack_store(ptr, (a + delta) >> n); -} +{ __lsx_vst(_v256_shuffle_odd_64(__lasx_xvsrarni_w_d(a.val, a.val, n)), ptr, 0); } // pack boolean inline v_uint8x32 v_pack_b(const v_uint16x16& a, const v_uint16x16& b) @@ -2583,63 +2406,48 @@ template inline v_float32x8 v_broadcast_element(const v_float32x8 &a) { return v_reinterpret_as_f32(v_broadcast_element(v_reinterpret_as_u32(a))); } - ///////////////////// load deinterleave ///////////////////////////// -inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b ) +inline void v_load_deinterleave(const uchar* ptr, v_uint8x32& a, v_uint8x32& b) { - __m256i ab0 = __lasx_xvld(ptr, 0); - __m256i ab1 = __lasx_xvld(ptr + 32, 0); + __m256i t0 = __lasx_xvld(ptr, 0); + __m256i t1 = __lasx_xvld(ptr, 32); - const __m256i sh = _v256_setr_b(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, - 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); - __m256i p0 = __lasx_xvshuf_b(ab0, ab0, sh); - __m256i p1 = __lasx_xvshuf_b(ab1, ab1, sh); - __m256i pl = __lasx_xvpermi_q(p0, p1, 0x02); - __m256i ph = __lasx_xvpermi_q(p0, p1, 0x13); - __m256i a0 = __lasx_xvilvl_d(ph, pl); - __m256i b0 = __lasx_xvilvh_d(ph, pl); - a = v_uint8x32(a0); - b = v_uint8x32(b0); + __m256i p0 = __lasx_xvpickev_b(t1, t0); + __m256i p1 = __lasx_xvpickod_b(t1, t0); + + a.val = __lasx_xvpermi_d(p0, 0xd8); + b.val = __lasx_xvpermi_d(p1, 0xd8); } inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b ) { - __m256i ab0 = __lasx_xvld(ptr, 0); - __m256i ab1 = __lasx_xvld(ptr + 16, 0); + __m256i t0 = __lasx_xvld(ptr, 0); + __m256i t1 = __lasx_xvld(ptr, 32); - const __m256i sh = _v256_setr_b(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15, - 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15); - __m256i p0 = __lasx_xvshuf_b(ab0, ab0, sh); - __m256i p1 = __lasx_xvshuf_b(ab1, ab1, sh); - __m256i pl = __lasx_xvpermi_q(p0, p1, 0x02); - __m256i ph = __lasx_xvpermi_q(p0, p1, 0x13); - __m256i a0 = __lasx_xvilvl_d(ph, pl); - __m256i b0 = __lasx_xvilvh_d(ph, pl); - a = v_uint16x16(a0); - b = v_uint16x16(b0); + __m256i p0 = __lasx_xvpickev_h(t1, t0); + __m256i p1 = __lasx_xvpickod_h(t1, t0); + + a.val = __lasx_xvpermi_d(p0, 0xd8); + b.val = __lasx_xvpermi_d(p1, 0xd8); } inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b ) { - __m256i ab0 = __lasx_xvld(ptr, 0); - __m256i ab1 = __lasx_xvld(ptr + 8, 0); + __m256i t0 = __lasx_xvld(ptr, 0); + __m256i t1 = __lasx_xvld(ptr, 32); - //const int sh = 0+2*4+1*16+3*64; - __m256i p0 = __lasx_xvshuf4i_w(ab0, 0xD8); - __m256i p1 = __lasx_xvshuf4i_w(ab1, 0xD8); - __m256i pl = __lasx_xvpermi_q(p0, p1, 0x02); - __m256i ph = __lasx_xvpermi_q(p0, p1, 0x13); - __m256i a0 = __lasx_xvilvl_d(ph, pl); - __m256i b0 = __lasx_xvilvh_d(ph, pl); - a = v_uint32x8(a0); - b = v_uint32x8(b0); + __m256i p0 = __lasx_xvpickev_w(t1, t0); + __m256i p1 = __lasx_xvpickod_w(t1, t0); + + a.val = __lasx_xvpermi_d(p0, 0xd8); + b.val = __lasx_xvpermi_d(p1, 0xd8); } inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b ) { __m256i ab0 = __lasx_xvld(ptr, 0); - __m256i ab1 = __lasx_xvld(ptr + 4, 0); + __m256i ab1 = __lasx_xvld(ptr, 32); __m256i pl = __lasx_xvpermi_q(ab0, ab1, 0x02); __m256i ph = __lasx_xvpermi_q(ab0, ab1, 0x13); @@ -2652,8 +2460,8 @@ inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c ) { __m256i bgr0 = __lasx_xvld(ptr, 0); - __m256i bgr1 = __lasx_xvld(ptr + 32, 0); - __m256i bgr2 = __lasx_xvld(ptr + 64, 0); + __m256i bgr1 = __lasx_xvld(ptr, 32); + __m256i bgr2 = __lasx_xvld(ptr, 64); __m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02); __m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13); @@ -2686,8 +2494,8 @@ inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c ) { __m256i bgr0 = __lasx_xvld(ptr, 0); - __m256i bgr1 = __lasx_xvld(ptr + 16, 0); - __m256i bgr2 = __lasx_xvld(ptr + 32, 0); + __m256i bgr1 = __lasx_xvld(ptr, 32); + __m256i bgr2 = __lasx_xvld(ptr, 64); __m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02); __m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13); @@ -2717,8 +2525,8 @@ inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b, v_uint32x8& c ) { __m256i bgr0 = __lasx_xvld(ptr, 0); - __m256i bgr1 = __lasx_xvld(ptr + 8, 0); - __m256i bgr2 = __lasx_xvld(ptr + 16, 0); + __m256i bgr1 = __lasx_xvld(ptr, 32); + __m256i bgr2 = __lasx_xvld(ptr, 64); __m256i s02_low = __lasx_xvpermi_q(bgr0, bgr2, 0x02); __m256i s02_high = __lasx_xvpermi_q(bgr0, bgr2, 0x13); @@ -2741,8 +2549,8 @@ inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b, v_uint64x4& c ) { __m256i bgr0 = __lasx_xvld(ptr, 0); - __m256i bgr1 = __lasx_xvld(ptr + 4, 0); - __m256i bgr2 = __lasx_xvld(ptr + 8, 0); + __m256i bgr1 = __lasx_xvld(ptr, 32); + __m256i bgr2 = __lasx_xvld(ptr, 64); __m256i s01 = __lasx_xvpermi_q(bgr0, bgr1, 0x12); // get bgr0 low 128 and bgr1 high 128 __m256i s12 = __lasx_xvpermi_q(bgr1, bgr2, 0x12); @@ -2756,81 +2564,60 @@ inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b c = v_uint64x4(r0); } -inline void v_load_deinterleave( const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c, v_uint8x32& d ) +inline void v_load_deinterleave(const uchar* ptr, v_uint8x32& a, v_uint8x32& b, v_uint8x32& c, v_uint8x32& d) { - __m256i bgr0 = __lasx_xvld(ptr, 0); - __m256i bgr1 = __lasx_xvld(ptr + 32, 0); - __m256i bgr2 = __lasx_xvld(ptr + 64, 0); - __m256i bgr3 = __lasx_xvld(ptr + 96, 0); - const __m256i sh = _v256_setr_b(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, - 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + __m256i t0 = __lasx_xvld(ptr, 0); + __m256i t1 = __lasx_xvld(ptr, 32); + __m256i t2 = __lasx_xvld(ptr, 64); + __m256i t3 = __lasx_xvld(ptr, 96); - __m256i p0 = __lasx_xvshuf_b(bgr0, bgr0, sh); - __m256i p1 = __lasx_xvshuf_b(bgr1, bgr1, sh); - __m256i p2 = __lasx_xvshuf_b(bgr2, bgr2, sh); - __m256i p3 = __lasx_xvshuf_b(bgr3, bgr3, sh); + const __m256i sh = _v256_setr_w(0, 4, 1, 5, 2, 6, 3, 7); + __m256i ac_lo = __lasx_xvpickev_b(t1, t0); + __m256i bd_lo = __lasx_xvpickod_b(t1, t0); + __m256i ac_hi = __lasx_xvpickev_b(t3, t2); + __m256i bd_hi = __lasx_xvpickod_b(t3, t2); - __m256i p01l = __lasx_xvilvl_w(p1, p0); - __m256i p01h = __lasx_xvilvh_w(p1, p0); - __m256i p23l = __lasx_xvilvl_w(p3, p2); - __m256i p23h = __lasx_xvilvh_w(p3, p2); + __m256i a_pre = __lasx_xvpickev_b(ac_hi, ac_lo); + __m256i c_pre = __lasx_xvpickod_b(ac_hi, ac_lo); + __m256i b_pre = __lasx_xvpickev_b(bd_hi, bd_lo); + __m256i d_pre = __lasx_xvpickod_b(bd_hi, bd_lo); - __m256i pll = __lasx_xvpermi_q(p01l, p23l, 0x02); - __m256i plh = __lasx_xvpermi_q(p01l, p23l, 0x13); - __m256i phl = __lasx_xvpermi_q(p01h, p23h, 0x02); - __m256i phh = __lasx_xvpermi_q(p01h, p23h, 0x13); - - __m256i b0 = __lasx_xvilvl_w(plh, pll); - __m256i g0 = __lasx_xvilvh_w(plh, pll); - __m256i r0 = __lasx_xvilvl_w(phh, phl); - __m256i a0 = __lasx_xvilvh_w(phh, phl); - - a = v_uint8x32(b0); - b = v_uint8x32(g0); - c = v_uint8x32(r0); - d = v_uint8x32(a0); + a.val = __lasx_xvperm_w(a_pre, sh); + b.val = __lasx_xvperm_w(b_pre, sh); + c.val = __lasx_xvperm_w(c_pre, sh); + d.val = __lasx_xvperm_w(d_pre, sh); } -inline void v_load_deinterleave( const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c, v_uint16x16& d ) +inline void v_load_deinterleave(const ushort* ptr, v_uint16x16& a, v_uint16x16& b, v_uint16x16& c, v_uint16x16& d) { - __m256i bgr0 = __lasx_xvld(ptr, 0); - __m256i bgr1 = __lasx_xvld(ptr + 16, 0); - __m256i bgr2 = __lasx_xvld(ptr + 32, 0); - __m256i bgr3 = __lasx_xvld(ptr + 48, 0); - const __m256i sh = _v256_setr_b(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15, - 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15); - __m256i p0 = __lasx_xvshuf_b(bgr0, bgr0, sh); - __m256i p1 = __lasx_xvshuf_b(bgr1, bgr1, sh); - __m256i p2 = __lasx_xvshuf_b(bgr2, bgr2, sh); - __m256i p3 = __lasx_xvshuf_b(bgr3, bgr3, sh); + __m256i t0 = __lasx_xvld(ptr, 0); + __m256i t1 = __lasx_xvld(ptr, 32); + __m256i t2 = __lasx_xvld(ptr, 64); + __m256i t3 = __lasx_xvld(ptr, 96); - __m256i p01l = __lasx_xvilvl_w(p1, p0); - __m256i p01h = __lasx_xvilvh_w(p1, p0); - __m256i p23l = __lasx_xvilvl_w(p3, p2); - __m256i p23h = __lasx_xvilvh_w(p3, p2); + const __m256i sh = _v256_setr_w(0, 4, 1, 5, 2, 6, 3, 7); + __m256i ac_lo = __lasx_xvpickev_h(t1, t0); + __m256i bd_lo = __lasx_xvpickod_h(t1, t0); + __m256i ac_hi = __lasx_xvpickev_h(t3, t2); + __m256i bd_hi = __lasx_xvpickod_h(t3, t2); - __m256i pll = __lasx_xvpermi_q(p01l, p23l, 0x02); - __m256i plh = __lasx_xvpermi_q(p01l, p23l, 0x13); - __m256i phl = __lasx_xvpermi_q(p01h, p23h, 0x02); - __m256i phh = __lasx_xvpermi_q(p01h, p23h, 0x13); + __m256i a_pre = __lasx_xvpickev_h(ac_hi, ac_lo); + __m256i c_pre = __lasx_xvpickod_h(ac_hi, ac_lo); + __m256i b_pre = __lasx_xvpickev_h(bd_hi, bd_lo); + __m256i d_pre = __lasx_xvpickod_h(bd_hi, bd_lo); - __m256i b0 = __lasx_xvilvl_w(plh, pll); - __m256i g0 = __lasx_xvilvh_w(plh, pll); - __m256i r0 = __lasx_xvilvl_w(phh, phl); - __m256i a0 = __lasx_xvilvh_w(phh, phl); - - a = v_uint16x16(b0); - b = v_uint16x16(g0); - c = v_uint16x16(r0); - d = v_uint16x16(a0); + a.val = __lasx_xvperm_w(a_pre, sh); + b.val = __lasx_xvperm_w(b_pre, sh); + c.val = __lasx_xvperm_w(c_pre, sh); + d.val = __lasx_xvperm_w(d_pre, sh); } inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& b, v_uint32x8& c, v_uint32x8& d ) { __m256i p0 = __lasx_xvld(ptr, 0); - __m256i p1 = __lasx_xvld(ptr + 8, 0); - __m256i p2 = __lasx_xvld(ptr + 16, 0); - __m256i p3 = __lasx_xvld(ptr + 24, 0); + __m256i p1 = __lasx_xvld(ptr, 32); + __m256i p2 = __lasx_xvld(ptr, 64); + __m256i p3 = __lasx_xvld(ptr, 96); __m256i p01l = __lasx_xvilvl_w(p1, p0); __m256i p01h = __lasx_xvilvh_w(p1, p0); @@ -2856,9 +2643,9 @@ inline void v_load_deinterleave( const unsigned* ptr, v_uint32x8& a, v_uint32x8& inline void v_load_deinterleave( const uint64* ptr, v_uint64x4& a, v_uint64x4& b, v_uint64x4& c, v_uint64x4& d ) { __m256i bgra0 = __lasx_xvld(ptr, 0); - __m256i bgra1 = __lasx_xvld(ptr + 4, 0); - __m256i bgra2 = __lasx_xvld(ptr + 8, 0); - __m256i bgra3 = __lasx_xvld(ptr + 12, 0); + __m256i bgra1 = __lasx_xvld(ptr, 32); + __m256i bgra2 = __lasx_xvld(ptr, 64); + __m256i bgra3 = __lasx_xvld(ptr, 96); __m256i l02 = __lasx_xvpermi_q(bgra0, bgra2, 0x02); __m256i h02 = __lasx_xvpermi_q(bgra0, bgra2, 0x13); diff --git a/modules/core/include/opencv2/core/hal/intrin_lsx.hpp b/modules/core/include/opencv2/core/hal/intrin_lsx.hpp new file mode 100644 index 0000000000..4e3d2319ad --- /dev/null +++ b/modules/core/include/opencv2/core/hal/intrin_lsx.hpp @@ -0,0 +1,2536 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_HAL_INTRIN_LSX_HPP +#define OPENCV_HAL_INTRIN_LSX_HPP + +#include + +#define CV_SIMD128 1 +#define CV_SIMD128_64F 1 +#define CV_SIMD128_FP16 0 + +namespace cv +{ + +//! @cond IGNORED + +CV_CPU_OPTIMIZATION_HAL_NAMESPACE_BEGIN + +/////////// Utils //////// + +inline __m128i _v128_setr_b(char v0, char v1, char v2, char v3, char v4, char v5, char v6, + char v7, char v8, char v9, char v10, char v11, char v12, char v13, char v14, char v15) +{ + return (__m128i)v16i8{ v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 }; +} + +inline __m128i _v128_set_b(char v0, char v1, char v2, char v3, char v4, char v5, char v6, + char v7, char v8, char v9, char v10, char v11, char v12, char v13, char v14, char v15) +{ + return (__m128i)v16i8{ v15, v14, v13, v12, v11, v10, v9, v8, + v7, v6, v5, v4, v3, v2, v1, v0 }; +} + +inline __m128i _v128_setr_h(short v0, short v1, short v2, short v3, short v4, short v5, + short v6, short v7) +{ + return (__m128i)v8i16{ v0, v1, v2, v3, v4, v5, v6, v7 }; +} + +inline __m128i _v128_setr_w(int v0, int v1, int v2, int v3) +{ + return (__m128i)v4i32{ v0, v1, v2, v3 }; +} + +inline __m128i _v128_set_w(int v0, int v1, int v2, int v3) +{ + return (__m128i)v4i32{ v3, v2, v1, v0 }; +} + +inline __m128i _v128_setall_w(int v0) +{ + return __lsx_vreplgr2vr_w(v0); +} + +inline __m128i _v128_setr_d(int64 v0, int64 v1) +{ + return (__m128i)v2i64{ v0, v1 }; +} + +inline __m128i _v128_set_d(int64 v0, int64 v1) +{ + return (__m128i)v2i64{ v1, v0 }; +} + +inline __m128 _v128_setr_ps(float v0, float v1, float v2, float v3) +{ + return (__m128)v4f32{ v0, v1, v2, v3 }; +} + +inline __m128 _v128_setall_ps(float v0) +{ + return (__m128)v4f32{ v0, v0, v0, v0 }; +} + +inline __m128d _v128_setr_pd(double v0, double v1) +{ + return (__m128d)v2f64{ v0, v1 }; +} + +inline __m128d _v128_setall_pd(double v0) +{ + return (__m128d)v2f64{ v0, v0 }; +} + +inline __m128i _lsx_packus_h(const __m128i& a, const __m128i& b) +{ + return __lsx_vssrarni_bu_h(b, a, 0); +} + +inline __m128i _lsx_packs_h(const __m128i& a, const __m128i& b) +{ + return __lsx_vssrarni_b_h(b, a, 0); +} + +inline __m128i _lsx_packus_w(const __m128i& a, const __m128i& b) +{ + return __lsx_vssrarni_hu_w(b, a, 0); +} + +/////// Types /////// + +struct v_uint8x16 +{ + typedef uchar lane_type; + enum { nlanes = 16}; + + v_uint8x16() {} + explicit v_uint8x16(__m128i v): val(v) {} + v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7, + uchar v8, uchar v9, uchar v10, uchar v11, uchar v12, uchar v13, uchar v14, uchar v15) + { + val = _v128_setr_b(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15); + } + + uchar get0() const + { + return (uchar)__lsx_vpickve2gr_bu(val, 0); + } + + __m128i val; +}; + +struct v_int8x16 +{ + typedef schar lane_type; + enum { nlanes = 16 }; + + v_int8x16() {} + explicit v_int8x16(__m128i v) : val(v) {} + v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7, + schar v8, schar v9, schar v10, schar v11, schar v12, schar v13, schar v14, schar v15) + { + val = _v128_setr_b(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15); + } + + schar get0() const + { + return (schar)__lsx_vpickve2gr_b(val, 0); + } + + __m128i val; +}; + +struct v_uint16x8 +{ + typedef ushort lane_type; + enum { nlanes = 8 }; + + v_uint16x8() {} + explicit v_uint16x8(__m128i v) : val(v) {} + v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7) + { + val = _v128_setr_h(v0, v1, v2, v3, v4, v5, v6, v7); + } + + ushort get0() const + { + return (ushort)__lsx_vpickve2gr_hu(val, 0); + } + + __m128i val; +}; + +struct v_int16x8 +{ + typedef short lane_type; + enum { nlanes = 8 }; + + v_int16x8() {} + explicit v_int16x8(__m128i v) : val(v) {} + v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7) + { + val = _v128_setr_h(v0, v1, v2, v3, v4, v5, v6, v7); + } + + short get0() const + { + return (short)__lsx_vpickve2gr_h(val, 0); + } + + __m128i val; +}; + +struct v_uint32x4 +{ + typedef unsigned lane_type; + enum { nlanes = 4 }; + + v_uint32x4() {} + explicit v_uint32x4(__m128i v) : val(v) {} + v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3) + { + val = _v128_setr_w(v0, v1, v2, v3); + } + + unsigned get0() const + { + return (unsigned)__lsx_vpickve2gr_wu(val, 0); + } + + __m128i val; +}; + +struct v_int32x4 +{ + typedef int lane_type; + enum { nlanes = 4 }; + + v_int32x4() {} + explicit v_int32x4(__m128i v) : val(v) {} + v_int32x4(int v0, int v1, int v2, int v3) + { + val = _v128_setr_w(v0, v1, v2, v3); + } + + int get0() const + { + return (int)__lsx_vpickve2gr_w(val, 0); + } + + __m128i val; +}; + +struct v_float32x4 +{ + typedef float lane_type; + enum { nlanes = 4}; + + v_float32x4() {} + explicit v_float32x4(__m128 v) : val(v) {} + explicit v_float32x4(__m128i v) { val = *((__m128*)&v); } + v_float32x4(float v0, float v1, float v2, float v3) + { + val = _v128_setr_ps(v0, v1, v2, v3); + } + + float get0() const + { + union { int iv; float fv; } d; + d.iv = __lsx_vpickve2gr_w(val, 0); + return d.fv; + } + + int get0toint() const + { + __m128i result = __lsx_vftintrz_w_s(val); + return (int)__lsx_vpickve2gr_w(result, 0); + } + + __m128 val; +}; + +struct v_uint64x2 +{ + typedef uint64 lane_type; + enum { nlanes = 2}; + + v_uint64x2() {} + explicit v_uint64x2(__m128i v) : val(v) {} + v_uint64x2(uint64 v0, uint64 v1) + { + val = _v128_setr_d(v0, v1); + } + + uint64 get0() const + { + return __lsx_vpickve2gr_du(val, 0); + } + + __m128i val; +}; + +struct v_int64x2 +{ + typedef int64 lane_type; + enum { nlanes = 2}; + + v_int64x2() {} + explicit v_int64x2(__m128i v) : val(v) {} + v_int64x2(int64 v0, int64 v1) + { + val = _v128_setr_d(v0, v1); + } + + uint64 get0() const + { + return __lsx_vpickve2gr_d(val, 0); + } + + __m128i val; +}; + +struct v_float64x2 +{ + typedef double lane_type; + enum { nlanes = 2}; + + v_float64x2() {} + explicit v_float64x2(__m128d v) : val(v) {} + explicit v_float64x2(__m128i v) { val = *((__m128d*)&v); } + v_float64x2(double v0, double v1) + { + val = _v128_setr_pd(v0, v1); + } + + double get0() const + { + union { int64 iv; double fv; } d; + d.iv = __lsx_vpickve2gr_d(val, 0); + return d.fv; + } + + int64 get0toint64() const + { + __m128i result = __lsx_vftintrz_l_d(val); + return (int64)__lsx_vpickve2gr_d(result, 0); + } + + __m128d val; +}; + +////////////// Load and store operations ///////// + +#define OPENCV_HAL_IMPL_LSX_LOADSTORE(_Tpvec, _Tp) \ + inline _Tpvec v_load(const _Tp* ptr) \ + { return _Tpvec(__lsx_vld(ptr, 0)); } \ + inline _Tpvec v_load_aligned(const _Tp* ptr) \ + { return _Tpvec(__lsx_vld(ptr, 0)); } \ + inline _Tpvec v_load_low(const _Tp* ptr) \ + { return _Tpvec(__lsx_vldrepl_d(ptr, 0)); } \ + inline _Tpvec v_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ + { \ + __m128i vl = __lsx_vldrepl_d(ptr0, 0); \ + __m128i vh = __lsx_vldrepl_d(ptr1, 0); \ + return _Tpvec(__lsx_vilvl_d(vh, vl)); \ + } \ + inline void v_store(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vst(a.val, ptr, 0); } \ + inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vst(a.val, ptr, 0); } \ + inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vst(a.val, ptr, 0); } \ + inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode)\ + { \ + if ( mode == hal::STORE_UNALIGNED) \ + __lsx_vst(a.val, ptr, 0); \ + else if ( mode == hal::STORE_ALIGNED_NOCACHE) \ + __lsx_vst(a.val, ptr, 0); \ + else \ + __lsx_vst(a.val, ptr, 0); \ + } \ + inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vstelm_d(a.val, ptr, 0, 0); } \ + inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vstelm_d(a.val, ptr, 0, 1); } \ + +OPENCV_HAL_IMPL_LSX_LOADSTORE(v_uint8x16, uchar) +OPENCV_HAL_IMPL_LSX_LOADSTORE(v_int8x16, schar) +OPENCV_HAL_IMPL_LSX_LOADSTORE(v_uint16x8, ushort) +OPENCV_HAL_IMPL_LSX_LOADSTORE(v_int16x8, short) +OPENCV_HAL_IMPL_LSX_LOADSTORE(v_uint32x4, unsigned) +OPENCV_HAL_IMPL_LSX_LOADSTORE(v_int32x4, int) +OPENCV_HAL_IMPL_LSX_LOADSTORE(v_uint64x2, uint64) +OPENCV_HAL_IMPL_LSX_LOADSTORE(v_int64x2, int64) + +#define OPENCV_HAL_IMPL_LSX_LOADSTORE_FLT(_Tpvec, _Tp, halfreg) \ + inline _Tpvec v_load(const _Tp* ptr) \ + { return _Tpvec((halfreg)__lsx_vld(ptr, 0)); } \ + inline _Tpvec v_load_aligned(const _Tp* ptr) \ + { return _Tpvec((halfreg)__lsx_vld(ptr, 0)); } \ + inline _Tpvec v_load_low(const _Tp* ptr) \ + { return _Tpvec((halfreg)__lsx_vldrepl_d(ptr, 0)); } \ + inline _Tpvec v_load_halves(const _Tp* ptr0, const _Tp* ptr1) \ + { \ + __m128i vl = __lsx_vldrepl_d(ptr0, 0); \ + __m128i vh = __lsx_vldrepl_d(ptr1, 0); \ + return _Tpvec((halfreg)__lsx_vilvl_d(vh, vl)); \ + } \ + inline void v_store(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vst((__m128i)a.val, ptr, 0); } \ + inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vst((__m128i)a.val, ptr, 0); } \ + inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vst((__m128i)a.val, ptr, 0); } \ + inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode mode)\ + { \ + if( mode == hal::STORE_UNALIGNED) \ + __lsx_vst((__m128i)a.val, ptr, 0); \ + else if( mode == hal::STORE_ALIGNED_NOCACHE) \ + __lsx_vst((__m128i)a.val, ptr, 0); \ + else \ + __lsx_vst((__m128i)a.val, ptr, 0); \ + } \ + inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vstelm_d((__m128i)a.val, ptr, 0, 0); } \ + inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ + { __lsx_vstelm_d((__m128i)a.val, ptr, 0, 1); } \ + +OPENCV_HAL_IMPL_LSX_LOADSTORE_FLT(v_float32x4, float, __m128) +OPENCV_HAL_IMPL_LSX_LOADSTORE_FLT(v_float64x2, double, __m128d) + +inline __m128i _lsx_128_castps_si128(const __m128& v) +{ return __m128i(v); } + +inline __m128i _lsx_128_castpd_si128(const __m128d& v) +{ return __m128i(v); } + +#define OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, _Tpvecf, suffix, cast) \ + inline _Tpvec v_reinterpret_as_##suffix(const _Tpvecf& a) \ + { return _Tpvec(cast(a.val)); } + +#define OPENCV_HAL_IMPL_LSX_INIT(_Tpvec, _Tp, suffix, ssuffix, ctype_s) \ + inline _Tpvec v_setzero_##suffix() \ + { return _Tpvec(__lsx_vldi(0)); } \ + inline _Tpvec v_setall_##suffix(_Tp v) \ + { return _Tpvec(__lsx_vreplgr2vr_##ssuffix((ctype_s)v)); } \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_uint8x16, suffix, OPENCV_HAL_NOP) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_int8x16, suffix, OPENCV_HAL_NOP) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_uint16x8, suffix, OPENCV_HAL_NOP) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_int16x8, suffix, OPENCV_HAL_NOP) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_uint32x4, suffix, OPENCV_HAL_NOP) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_int32x4, suffix, OPENCV_HAL_NOP) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_uint64x2, suffix, OPENCV_HAL_NOP) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_int64x2, suffix, OPENCV_HAL_NOP) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_float32x4, suffix, _lsx_128_castps_si128) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_float64x2, suffix, _lsx_128_castpd_si128) \ + +OPENCV_HAL_IMPL_LSX_INIT(v_uint8x16, uchar, u8, b, int) +OPENCV_HAL_IMPL_LSX_INIT(v_int8x16, schar, s8, b, int) +OPENCV_HAL_IMPL_LSX_INIT(v_uint16x8, ushort, u16, h, int) +OPENCV_HAL_IMPL_LSX_INIT(v_int16x8, short, s16, h, int) +OPENCV_HAL_IMPL_LSX_INIT(v_uint32x4, unsigned, u32, w, int) +OPENCV_HAL_IMPL_LSX_INIT(v_int32x4, int, s32, w, int) +OPENCV_HAL_IMPL_LSX_INIT(v_uint64x2, uint64, u64, d, long int) +OPENCV_HAL_IMPL_LSX_INIT(v_int64x2, int64, s64, d, long int) + +inline __m128 _lsx_128_castsi128_ps(const __m128i &v) +{ return __m128(v); } + +inline __m128d _lsx_128_castsi128_pd(const __m128i &v) +{ return __m128d(v); } + +#define OPENCV_HAL_IMPL_LSX_INIT_FLT(_Tpvec, _Tp, suffix, zsuffix, cast) \ + inline _Tpvec v_setzero_##suffix() \ + { return _Tpvec(__lsx_vldi(0)); } \ + inline _Tpvec v_setall_##suffix(_Tp v) \ + { return _Tpvec(_v128_setall_##zsuffix(v)); } \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_uint8x16, suffix, cast) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_int8x16, suffix, cast) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_uint16x8, suffix, cast) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_int16x8, suffix, cast) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_uint32x4, suffix, cast) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_int32x4, suffix, cast) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_uint64x2, suffix, cast) \ + OPENCV_HAL_IMPL_LSX_CAST(_Tpvec, v_int64x2, suffix, cast) \ + +OPENCV_HAL_IMPL_LSX_INIT_FLT(v_float32x4, float, f32, ps, _lsx_128_castsi128_ps) +OPENCV_HAL_IMPL_LSX_INIT_FLT(v_float64x2, double, f64, pd, _lsx_128_castsi128_pd) + +inline v_float32x4 v_reinterpret_as_f32(const v_float32x4& a) +{ return a; } +inline v_float32x4 v_reinterpret_as_f32(const v_float64x2& a) +{ return v_float32x4(_lsx_128_castps_si128(__m128(a.val))); } + +inline v_float64x2 v_reinterpret_as_f64(const v_float64x2& a) +{ return a; } +inline v_float64x2 v_reinterpret_as_f64(const v_float32x4& a) +{ return v_float64x2(_lsx_128_castpd_si128(__m128d(a.val))); } + +//////////////// Variant Value reordering /////////////// + +// unpacks +#define OPENCV_HAL_IMPL_LSX_UNPACK(_Tpvec, suffix) \ + inline _Tpvec v128_unpacklo(const _Tpvec& a, const _Tpvec& b) \ + { return _Tpvec(__lsx_vilvl_##suffix(__m128i(b.val), __m128i(a.val))); } \ + inline _Tpvec v128_unpackhi(const _Tpvec& a, const _Tpvec& b) \ + { return _Tpvec(__lsx_vilvh_##suffix(__m128i(b.val), __m128i(a.val))); } \ + +OPENCV_HAL_IMPL_LSX_UNPACK(v_uint8x16, b) +OPENCV_HAL_IMPL_LSX_UNPACK(v_int8x16, b) +OPENCV_HAL_IMPL_LSX_UNPACK(v_uint16x8, h) +OPENCV_HAL_IMPL_LSX_UNPACK(v_int16x8, h) +OPENCV_HAL_IMPL_LSX_UNPACK(v_uint32x4, w) +OPENCV_HAL_IMPL_LSX_UNPACK(v_int32x4, w) +OPENCV_HAL_IMPL_LSX_UNPACK(v_uint64x2, d) +OPENCV_HAL_IMPL_LSX_UNPACK(v_int64x2, d) +OPENCV_HAL_IMPL_LSX_UNPACK(v_float32x4, w) +OPENCV_HAL_IMPL_LSX_UNPACK(v_float64x2, d) + +//ZIP +#define OPENCV_HAL_IMPL_LSX_ZIP(_Tpvec) \ + inline _Tpvec v_combine_low(const _Tpvec& a, const _Tpvec& b) \ + { return (_Tpvec)__lsx_vilvl_d((__m128i)b.val, (__m128i)a.val); } \ + inline _Tpvec v_combine_high(const _Tpvec& a, const _Tpvec& b) \ + { return (_Tpvec)__lsx_vilvh_d((__m128i)b.val, (__m128i)a.val); } \ + inline void v_recombine(const _Tpvec& a, const _Tpvec& b, \ + _Tpvec& c, _Tpvec& d) \ + { \ + __m128i a1 = (__m128i)a.val, b1 = (__m128i)b.val; \ + c = _Tpvec(__lsx_vilvl_d(b1, a1)); \ + d = _Tpvec(__lsx_vilvh_d(b1, a1)); \ + } \ + inline void v_zip(const _Tpvec& a, const _Tpvec& b, \ + _Tpvec& ab0, _Tpvec& ab1) \ + { \ + ab0 = v128_unpacklo(a, b); \ + ab1 = v128_unpackhi(a, b); \ + } + +OPENCV_HAL_IMPL_LSX_ZIP(v_uint8x16) +OPENCV_HAL_IMPL_LSX_ZIP(v_int8x16) +OPENCV_HAL_IMPL_LSX_ZIP(v_uint16x8) +OPENCV_HAL_IMPL_LSX_ZIP(v_int16x8) +OPENCV_HAL_IMPL_LSX_ZIP(v_uint32x4) +OPENCV_HAL_IMPL_LSX_ZIP(v_int32x4) +OPENCV_HAL_IMPL_LSX_ZIP(v_uint64x2) +OPENCV_HAL_IMPL_LSX_ZIP(v_int64x2) +OPENCV_HAL_IMPL_LSX_ZIP(v_float32x4) +OPENCV_HAL_IMPL_LSX_ZIP(v_float64x2) + +////////// Arithmetic, bitwise and comparison operations ///////// + +/** Arithmetics **/ +#define OPENCV_HAL_IMPL_LSX_BIN_OP(bin_op, _Tpvec, intrin) \ + inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ + { return _Tpvec(intrin(a.val, b.val)); } \ + inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ + { a.val = intrin(a.val, b.val); return a; } + +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_uint8x16, __lsx_vsadd_bu) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_uint8x16, __lsx_vssub_bu) +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_int8x16, __lsx_vsadd_b) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_int8x16, __lsx_vssub_b) +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_uint16x8, __lsx_vsadd_hu) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_uint16x8, __lsx_vssub_hu) +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_int16x8, __lsx_vsadd_h) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_int16x8, __lsx_vssub_h) +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_uint32x4, __lsx_vadd_w) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_uint32x4, __lsx_vsub_w) +OPENCV_HAL_IMPL_LSX_BIN_OP(*, v_uint32x4, __lsx_vmul_w) +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_int32x4, __lsx_vadd_w) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_int32x4, __lsx_vsub_w) +OPENCV_HAL_IMPL_LSX_BIN_OP(*, v_int32x4, __lsx_vmul_w) +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_uint64x2, __lsx_vadd_d) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_uint64x2, __lsx_vsub_d) +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_int64x2, __lsx_vadd_d) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_int64x2, __lsx_vsub_d) + +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_float32x4, __lsx_vfadd_s) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_float32x4, __lsx_vfsub_s) +OPENCV_HAL_IMPL_LSX_BIN_OP(*, v_float32x4, __lsx_vfmul_s) +OPENCV_HAL_IMPL_LSX_BIN_OP(/, v_float32x4, __lsx_vfdiv_s) +OPENCV_HAL_IMPL_LSX_BIN_OP(+, v_float64x2, __lsx_vfadd_d) +OPENCV_HAL_IMPL_LSX_BIN_OP(-, v_float64x2, __lsx_vfsub_d) +OPENCV_HAL_IMPL_LSX_BIN_OP(*, v_float64x2, __lsx_vfmul_d) +OPENCV_HAL_IMPL_LSX_BIN_OP(/, v_float64x2, __lsx_vfdiv_d) + +// saturating multiply 8-bit, 16-bit +inline v_uint8x16 operator * (const v_uint8x16& a, const v_uint8x16& b) +{ + v_uint16x8 c, d; + v_mul_expand(a, b, c, d); + return v_pack(c, d); +} +inline v_int8x16 operator * (const v_int8x16& a, const v_int8x16& b) +{ + v_int16x8 c, d; + v_mul_expand(a, b, c, d); + return v_pack(c, d); +} +inline v_uint16x8 operator * (const v_uint16x8& a, const v_uint16x8& b) +{ + __m128i a0 = a.val, b0 = b.val; + __m128i pev = __lsx_vmulwev_w_hu(a0, b0); + __m128i pod = __lsx_vmulwod_w_hu(a0, b0); + __m128i pl = __lsx_vilvl_w(pod, pev); + __m128i ph = __lsx_vilvh_w(pod, pev); + return (v_uint16x8)__lsx_vssrlrni_hu_w(ph, pl, 0); +} +inline v_int16x8 operator * (const v_int16x8& a, const v_int16x8& b) +{ + __m128i a0 = a.val, b0 = b.val; + __m128i pev = __lsx_vmulwev_w_h(a0, b0); + __m128i pod = __lsx_vmulwod_w_h(a0, b0); + __m128i pl = __lsx_vilvl_w(pod, pev); + __m128i ph = __lsx_vilvh_w(pod, pev); + return (v_int16x8)__lsx_vssrarni_h_w(ph, pl, 0); +} +inline v_uint8x16& operator *= (v_uint8x16& a, const v_uint8x16& b) +{ a = a * b; return a; } +inline v_int8x16& operator *= (v_int8x16& a, const v_int8x16& b) +{ a = a * b; return a; } +inline v_uint16x8& operator *= (v_uint16x8& a, const v_uint16x8& b) +{ a = a * b; return a; } +inline v_int16x8& operator *= (v_int16x8& a, const v_int16x8& b) +{ a = a * b; return a; } + +/** Non-saturating arithmetics **/ + +#define OPENCV_HAL_IMPL_LSX_BIN_FUNC(func, _Tpvec, intrin) \ + inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ + { return _Tpvec(intrin(a.val, b.val)); } \ + +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_add_wrap, v_uint8x16, __lsx_vadd_b) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_add_wrap, v_int8x16, __lsx_vadd_b) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_add_wrap, v_uint16x8, __lsx_vadd_h) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_add_wrap, v_int16x8, __lsx_vadd_h) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_sub_wrap, v_uint8x16, __lsx_vsub_b) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_sub_wrap, v_int8x16, __lsx_vsub_b) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_sub_wrap, v_uint16x8, __lsx_vsub_h) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_sub_wrap, v_int16x8, __lsx_vsub_h) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_mul_wrap, v_uint16x8, __lsx_vmul_h) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_mul_wrap, v_int16x8, __lsx_vmul_h) + +inline v_uint8x16 v_mul_wrap(const v_uint8x16& a, const v_uint8x16& b) +{ + __m128i a0 = a.val, b0 = b.val; + __m128i p0 = __lsx_vmulwev_h_bu(a0, b0); + __m128i p1 = __lsx_vmulwod_h_bu(a0, b0); + return v_uint8x16(__lsx_vpackev_b(p1, p0)); +} + +inline v_int8x16 v_mul_wrap(const v_int8x16& a, const v_int8x16& b) +{ + return v_reinterpret_as_s8(v_mul_wrap(v_reinterpret_as_u8(a), v_reinterpret_as_u8(b))); +} + +// Multiply and expand +inline void v_mul_expand(const v_uint8x16& a, const v_uint8x16& b, + v_uint16x8& c, v_uint16x8& d) +{ + __m128i a0 = a.val, b0 = b.val; + __m128i p0 = __lsx_vmulwev_h_bu(a0, b0); + __m128i p1 = __lsx_vmulwod_h_bu(a0, b0); + c.val = __lsx_vilvl_h(p1, p0); + d.val = __lsx_vilvh_h(p1, p0); +} +inline void v_mul_expand(const v_int8x16& a, const v_int8x16& b, + v_int16x8& c, v_int16x8& d) +{ + __m128i a0 = a.val, b0 = b.val; + __m128i p0 = __lsx_vmulwev_h_b(a0, b0); + __m128i p1 = __lsx_vmulwod_h_b(a0, b0); + c.val = __lsx_vilvl_h(p1, p0); + d.val = __lsx_vilvh_h(p1, p0); +} +inline void v_mul_expand(const v_int16x8& a, const v_int16x8& b, + v_int32x4& c, v_int32x4& d) +{ + __m128i a0 = a.val, b0 = b.val; + __m128i p0 = __lsx_vmulwev_w_h(a0, b0); + __m128i p1 = __lsx_vmulwod_w_h(a0, b0); + c.val = __lsx_vilvl_w(p1, p0); + d.val = __lsx_vilvh_w(p1, p0); +} +inline void v_mul_expand(const v_uint16x8& a, const v_uint16x8& b, + v_uint32x4& c, v_uint32x4& d) +{ + __m128i a0 = a.val, b0 = b.val; + __m128i p0 = __lsx_vmulwev_w_hu(a0, b0); + __m128i p1 = __lsx_vmulwod_w_hu(a0, b0); + c.val = __lsx_vilvl_w(p1, p0); + d.val = __lsx_vilvh_w(p1, p0); +} +inline void v_mul_expand(const v_uint32x4& a, const v_uint32x4& b, + v_uint64x2& c, v_uint64x2& d) +{ + __m128i a0 = a.val, b0 = b.val; + __m128i p0 = __lsx_vmulwev_d_wu(a0, b0); + __m128i p1 = __lsx_vmulwod_d_wu(a0, b0); + c.val = __lsx_vilvl_d(p1, p0); + d.val = __lsx_vilvh_d(p1, p0); +} +inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) +{ return v_int16x8(__lsx_vmuh_h(a.val, b.val)); } +inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) +{ return v_uint16x8(__lsx_vmuh_hu(a.val, b.val)); } + +/** Bitwise shifts **/ +#define OPENCV_HAL_IMPL_LSX_SHIFT_OP(_Tpuvec, _Tpsvec, suffix, srai) \ + inline _Tpuvec operator << (const _Tpuvec& a, int imm) \ + { return _Tpuvec(__lsx_vsll_##suffix(a.val, __lsx_vreplgr2vr_##suffix(imm))); } \ + inline _Tpsvec operator << (const _Tpsvec& a, int imm) \ + { return _Tpsvec(__lsx_vsll_##suffix(a.val, __lsx_vreplgr2vr_##suffix(imm))); } \ + inline _Tpuvec operator >> (const _Tpuvec& a, int imm) \ + { return _Tpuvec(__lsx_vsrl_##suffix(a.val, __lsx_vreplgr2vr_##suffix(imm))); } \ + inline _Tpsvec operator >> (const _Tpsvec& a, int imm) \ + { return _Tpsvec(srai(a.val, __lsx_vreplgr2vr_##suffix(imm))); } \ + template \ + inline _Tpuvec v_shl(const _Tpuvec& a) \ + { return _Tpuvec(__lsx_vslli_##suffix(a.val, imm)); } \ + template \ + inline _Tpsvec v_shl(const _Tpsvec& a) \ + { return _Tpsvec(__lsx_vslli_##suffix(a.val, imm)); } \ + template \ + inline _Tpuvec v_shr(const _Tpuvec& a) \ + { return _Tpuvec(__lsx_vsrli_##suffix(a.val, imm)); } \ + template \ + inline _Tpsvec v_shr(const _Tpsvec& a) \ + { return _Tpsvec(__lsx_vsrai_##suffix(a.val, imm)); } \ + +OPENCV_HAL_IMPL_LSX_SHIFT_OP(v_uint16x8, v_int16x8, h, __lsx_vsra_h) +OPENCV_HAL_IMPL_LSX_SHIFT_OP(v_uint32x4, v_int32x4, w, __lsx_vsra_w) +OPENCV_HAL_IMPL_LSX_SHIFT_OP(v_uint64x2, v_int64x2, d, __lsx_vsra_d) + +/** Bitwise logic **/ +#define OPENCV_HAL_IMPL_LSX_LOGIC_OP(_Tpvec, suffix) \ + OPENCV_HAL_IMPL_LSX_BIN_OP(&, _Tpvec, __lsx_vand_##suffix) \ + OPENCV_HAL_IMPL_LSX_BIN_OP(|, _Tpvec, __lsx_vor_##suffix) \ + OPENCV_HAL_IMPL_LSX_BIN_OP(^, _Tpvec, __lsx_vxor_##suffix) \ + inline _Tpvec operator ~(const _Tpvec& a) \ + { return _Tpvec(__lsx_vnori_b(a.val, 0)); } \ + +OPENCV_HAL_IMPL_LSX_LOGIC_OP(v_uint8x16, v) +OPENCV_HAL_IMPL_LSX_LOGIC_OP(v_int8x16, v) +OPENCV_HAL_IMPL_LSX_LOGIC_OP(v_uint16x8, v) +OPENCV_HAL_IMPL_LSX_LOGIC_OP(v_int16x8, v) +OPENCV_HAL_IMPL_LSX_LOGIC_OP(v_uint32x4, v) +OPENCV_HAL_IMPL_LSX_LOGIC_OP(v_int32x4, v) +OPENCV_HAL_IMPL_LSX_LOGIC_OP(v_uint64x2, v) +OPENCV_HAL_IMPL_LSX_LOGIC_OP(v_int64x2, v) + +#define OPENCV_HAL_IMPL_LSX_FLOAT_BIN_OP(bin_op, _Tpvec, intrin, cast) \ + inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ + { return _Tpvec(intrin((__m128i)(a.val), (__m128i)(b.val))); } \ + inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ + { __m128i c = intrin((__m128i)(a.val), (__m128i)b.val); \ + a.val = cast(c); \ + return a;} + +#define OPENCV_HAL_IMPL_LSX_FLOAT_LOGIC_OP(_Tpvec, cast) \ + OPENCV_HAL_IMPL_LSX_FLOAT_BIN_OP(&, _Tpvec, __lsx_vand_v, cast) \ + OPENCV_HAL_IMPL_LSX_FLOAT_BIN_OP(|, _Tpvec, __lsx_vor_v, cast) \ + OPENCV_HAL_IMPL_LSX_FLOAT_BIN_OP(^, _Tpvec, __lsx_vxor_v, cast) \ + inline _Tpvec operator ~ (const _Tpvec& a) \ + { return _Tpvec(__lsx_vnori_b((__m128i)(a.val), 0)); } \ + +OPENCV_HAL_IMPL_LSX_FLOAT_LOGIC_OP(v_float32x4, _lsx_128_castsi128_ps) +OPENCV_HAL_IMPL_LSX_FLOAT_LOGIC_OP(v_float64x2, _lsx_128_castsi128_pd) + +/** Select **/ +#define OPENCV_HAL_IMPL_LSX_SELECT(_Tpvec) \ + inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ + { return _Tpvec(__lsx_vbitsel_v(b.val, a.val, mask.val)); } \ + +OPENCV_HAL_IMPL_LSX_SELECT(v_uint8x16) +OPENCV_HAL_IMPL_LSX_SELECT(v_int8x16) +OPENCV_HAL_IMPL_LSX_SELECT(v_uint16x8) +OPENCV_HAL_IMPL_LSX_SELECT(v_int16x8) +OPENCV_HAL_IMPL_LSX_SELECT(v_uint32x4) +OPENCV_HAL_IMPL_LSX_SELECT(v_int32x4) + +inline v_float32x4 v_select(const v_float32x4 &mask, const v_float32x4 &a, const v_float32x4 &b) +{ return v_float32x4(__lsx_vbitsel_v((__m128i)b.val, (__m128i)a.val, (__m128i)mask.val)); } +inline v_float64x2 v_select(const v_float64x2 &mask, const v_float64x2 &a, const v_float64x2 &b) +{ return v_float64x2(__lsx_vbitsel_v((__m128i)b.val, (__m128i)a.val, (__m128i)mask.val)); } + +/** Comparison **/ +#define OPENCV_HAL_IMPL_LSX_CMP_OP_OV(_Tpvec) \ + inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \ + { return ~( a == b ); } \ + inline _Tpvec operator < (const _Tpvec& a, const _Tpvec& b) \ + { return b > a ; } \ + inline _Tpvec operator >= (const _Tpvec& a, const _Tpvec& b) \ + { return ~(a < b); } \ + inline _Tpvec operator <= (const _Tpvec& a, const _Tpvec& b) \ + { return b >= a; } \ + +#define OPENCV_HAL_IMPL_LSX_CMP_OP_INT(_Tpuvec, _Tpsvec, suffix, usuffix) \ + inline _Tpuvec operator == (const _Tpuvec& a, const _Tpuvec& b) \ + { return _Tpuvec(__lsx_vseq_##suffix(a.val, b.val)); } \ + inline _Tpuvec operator > (const _Tpuvec& a, const _Tpuvec& b) \ + { return _Tpuvec(__lsx_vslt_##usuffix(b.val, a.val)); } \ + inline _Tpsvec operator == (const _Tpsvec& a, const _Tpsvec& b) \ + { return _Tpsvec(__lsx_vseq_##suffix(a.val, b.val)); } \ + inline _Tpsvec operator > (const _Tpsvec& a, const _Tpsvec& b) \ + { return _Tpsvec(__lsx_vslt_##suffix(b.val, a.val)); } \ + OPENCV_HAL_IMPL_LSX_CMP_OP_OV(_Tpuvec) \ + OPENCV_HAL_IMPL_LSX_CMP_OP_OV(_Tpsvec) + +OPENCV_HAL_IMPL_LSX_CMP_OP_INT(v_uint8x16, v_int8x16, b, bu) +OPENCV_HAL_IMPL_LSX_CMP_OP_INT(v_uint16x8, v_int16x8, h, hu) +OPENCV_HAL_IMPL_LSX_CMP_OP_INT(v_uint32x4, v_int32x4, w, wu) + +#define OPENCV_HAL_IMPL_LSX_CMP_OP_64BIT(_Tpvec, suffix) \ + inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \ + { return _Tpvec(__lsx_vseq_##suffix(a.val, b.val)); } \ + inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \ + { return ~(a == b); } + +OPENCV_HAL_IMPL_LSX_CMP_OP_64BIT(v_uint64x2, d) +OPENCV_HAL_IMPL_LSX_CMP_OP_64BIT(v_int64x2, d) + +#define OPENCV_HAL_IMPL_LSX_CMP_FLT(bin_op, suffix, _Tpvec, ssuffix) \ + inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ + { return _Tpvec(__lsx_##suffix##_##ssuffix(a.val, b.val)); } \ + +#define OPENCV_HAL_IMPL_LSX_CMP_OP_FLT(_Tpvec, ssuffix) \ + OPENCV_HAL_IMPL_LSX_CMP_FLT(==, vfcmp_ceq, _Tpvec, ssuffix) \ + OPENCV_HAL_IMPL_LSX_CMP_FLT(!=, vfcmp_cne, _Tpvec, ssuffix) \ + OPENCV_HAL_IMPL_LSX_CMP_FLT(<, vfcmp_clt, _Tpvec, ssuffix) \ + OPENCV_HAL_IMPL_LSX_CMP_FLT(<=, vfcmp_cle, _Tpvec, ssuffix) \ + +OPENCV_HAL_IMPL_LSX_CMP_OP_FLT(v_float32x4, s) +OPENCV_HAL_IMPL_LSX_CMP_OP_FLT(v_float64x2, d) + +inline v_float32x4 operator > (const v_float32x4 &a, const v_float32x4 &b) +{ return v_float32x4(__lsx_vfcmp_clt_s(b.val, a.val)); } + +inline v_float32x4 operator >= (const v_float32x4 &a, const v_float32x4 &b) +{ return v_float32x4(__lsx_vfcmp_cle_s(b.val, a.val)); } + +inline v_float64x2 operator > (const v_float64x2 &a, const v_float64x2 &b) +{ return v_float64x2(__lsx_vfcmp_clt_d(b.val, a.val)); } + +inline v_float64x2 operator >= (const v_float64x2 &a, const v_float64x2 &b) +{ return v_float64x2(__lsx_vfcmp_cle_d(b.val, a.val)); } + +inline v_float32x4 v_not_nan(const v_float32x4& a) +{ return v_float32x4(__lsx_vfcmp_cor_s(a.val, a.val)); } + +inline v_float64x2 v_not_nan(const v_float64x2& a) +{ return v_float64x2(__lsx_vfcmp_cor_d(a.val, a.val)); } + +/** min/max **/ +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_min, v_uint8x16, __lsx_vmin_bu) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_max, v_uint8x16, __lsx_vmax_bu) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_min, v_int8x16, __lsx_vmin_b) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_max, v_int8x16, __lsx_vmax_b) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_min, v_uint16x8, __lsx_vmin_hu) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_max, v_uint16x8, __lsx_vmax_hu) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_min, v_int16x8, __lsx_vmin_h) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_max, v_int16x8, __lsx_vmax_h) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_min, v_uint32x4, __lsx_vmin_wu) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_max, v_uint32x4, __lsx_vmax_wu) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_min, v_int32x4, __lsx_vmin_w) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_max, v_int32x4, __lsx_vmax_w) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_min, v_float32x4, __lsx_vfmin_s) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_max, v_float32x4, __lsx_vfmax_s) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_min, v_float64x2, __lsx_vfmin_d) +OPENCV_HAL_IMPL_LSX_BIN_FUNC(v_max, v_float64x2, __lsx_vfmax_d) + +template 16)), + bool is_first = (imm == 0), + bool is_half = (imm == 8), + bool is_second = (imm == 16), + bool is_other = (((imm > 0) && (imm < 8)) || ((imm > 8) && (imm < 16)))> +class v_lsx_palignr_u8_class; + +template +class v_lsx_palignr_u8_class; + +template +class v_lsx_palignr_u8_class +{ +public: + inline __m128i operator()(const __m128i& a, const __m128i& b) const + { + return a; + } +}; + +template +class v_lsx_palignr_u8_class +{ +public: + inline __m128i operator()(const __m128i& a, const __m128i& b) const + { + return __lsx_vshuf4i_d(a, b, 0x9); + } +}; + +template +class v_lsx_palignr_u8_class +{ +public: + inline __m128i operator()(const __m128i& a, const __m128i& b) const + { + return b; + } +}; + +template +class v_lsx_palignr_u8_class +{ +public: + inline __m128i operator()(const __m128i& a, const __m128i& b) const + { + enum { imm2 = (sizeof(__m128i) - imm) }; + return __lsx_vor_v(__lsx_vbsrl_v(a, imm), __lsx_vbsll_v(b, imm2)); + } +}; + +template +inline __m128i v_lsx_palignr_u8(const __m128i& a, const __m128i& b) +{ + CV_StaticAssert((imm >= 0) && (imm <= 16), "Invalid imm for v_lsx_palignr_u8"); + return v_lsx_palignr_u8_class()(a, b); +} +/** Rotate **/ +#define OPENCV_HAL_IMPL_LSX_ROTATE_CAST(_Tpvec, cast) \ + template \ + inline _Tpvec v_rotate_right(const _Tpvec &a) \ + { \ + enum { imm2 = (imm * sizeof(typename _Tpvec::lane_type))}; \ + __m128i ret = __lsx_vbsrl_v((__m128i)a.val, imm2); \ + return _Tpvec(cast(ret)); \ + } \ + template \ + inline _Tpvec v_rotate_left(const _Tpvec &a) \ + { \ + enum { imm2 = (imm * sizeof(typename _Tpvec::lane_type))}; \ + __m128i ret = __lsx_vbsll_v((__m128i)a.val, imm2); \ + return _Tpvec(cast(ret)); \ + } \ + template \ + inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ + { \ + enum { imm2 = (imm * sizeof(typename _Tpvec::lane_type))}; \ + return _Tpvec(cast(v_lsx_palignr_u8((__m128i)a.val, (__m128i)b.val))); \ + } \ + template \ + inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ + { \ + enum { imm2 = ((_Tpvec::nlanes - imm) * sizeof(typename _Tpvec::lane_type))}; \ + return _Tpvec(cast(v_lsx_palignr_u8((__m128i)b.val, (__m128i)a.val))); \ + } + +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_uint8x16, OPENCV_HAL_NOP) \ +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_int8x16, OPENCV_HAL_NOP) \ +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_uint16x8, OPENCV_HAL_NOP) \ +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_int16x8, OPENCV_HAL_NOP) \ +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_uint32x4, OPENCV_HAL_NOP) \ +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_int32x4, OPENCV_HAL_NOP) \ +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_uint64x2, OPENCV_HAL_NOP) \ +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_int64x2, OPENCV_HAL_NOP) \ + +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_float32x4, _lsx_128_castsi128_ps) +OPENCV_HAL_IMPL_LSX_ROTATE_CAST(v_float64x2, _lsx_128_castsi128_pd) + +/** Rverse **/ +inline v_uint8x16 v_reverse(const v_uint8x16 &a) +{ + __m128i vec = __lsx_vshuf4i_b(a.val, 0x1B); + return v_uint8x16(__lsx_vshuf4i_w(vec, 0x1B)); +} + +inline v_int8x16 v_reverse(const v_int8x16 &a) +{ return v_reinterpret_as_s8(v_reverse(v_reinterpret_as_u8(a))); } + +inline v_uint16x8 v_reverse(const v_uint16x8 &a) +{ + __m128i vec = __lsx_vshuf4i_h(a.val, 0x1B); + return v_uint16x8(__lsx_vshuf4i_w(vec, 0x4E)); +} + +inline v_int16x8 v_reverse(const v_int16x8 &a) +{ return v_reinterpret_as_s16(v_reverse(v_reinterpret_as_u16(a))); } + +inline v_uint32x4 v_reverse(const v_uint32x4 &a) +{ return v_uint32x4(__lsx_vshuf4i_w(a.val, 0x1B)); } + +inline v_int32x4 v_reverse(const v_int32x4 &a) +{ return v_int32x4(__lsx_vshuf4i_w(a.val, 0x1B)); } + +inline v_uint64x2 v_reverse(const v_uint64x2 &a) +{ return v_uint64x2(__lsx_vshuf4i_w(a.val, 0x4E)); } + +inline v_int64x2 v_reverse(const v_int64x2 &a) +{ return v_int64x2(__lsx_vshuf4i_w(a.val, 0x4E)); } + +inline v_float32x4 v_reverse(const v_float32x4 &a) +{ return v_reinterpret_as_f32(v_reverse(v_reinterpret_as_u32(a))); } + +inline v_float64x2 v_reverse(const v_float64x2 &a) +{ return v_reinterpret_as_f64(v_reverse(v_reinterpret_as_u64(a))); } + +////////////// Reduce and mask //////////// + +/** Reduce **/ +// this function is return a[0]+a[1]+...+a[31] +inline unsigned v_reduce_sum(const v_uint8x16& a) +{ + __m128i t1 = __lsx_vhaddw_hu_bu(a.val, a.val); + __m128i t2 = __lsx_vhaddw_wu_hu(t1, t1); + __m128i t3 = __lsx_vhaddw_du_wu(t2, t2); + __m128i t4 = __lsx_vhaddw_qu_du(t3, t3); + return (unsigned)__lsx_vpickve2gr_w(t4, 0); +} + +inline int v_reduce_sum(const v_int8x16 &a) +{ + __m128i t1 = __lsx_vhaddw_h_b(a.val, a.val); + __m128i t2 = __lsx_vhaddw_w_h(t1, t1); + __m128i t3 = __lsx_vhaddw_d_w(t2, t2); + __m128i t4 = __lsx_vhaddw_q_d(t3, t3); + return (int)__lsx_vpickve2gr_w(t4, 0); +} + +#define OPENCV_HAL_IMPL_LSX_REDUCE_16(_Tpvec, sctype, func, intrin) \ + inline sctype v_reduce_##func(const _Tpvec& a) \ + { \ + __m128i val = intrin(a.val, __lsx_vbsrl_v(a.val, 8)); \ + val = intrin(val, __lsx_vbsrl_v(val, 4)); \ + val = intrin(val, __lsx_vbsrl_v(val, 2)); \ + val = intrin(val, __lsx_vbsrl_v(val, 1)); \ + return (sctype)__lsx_vpickve2gr_b(val, 0); \ + } + +OPENCV_HAL_IMPL_LSX_REDUCE_16(v_uint8x16, uchar, min, __lsx_vmin_bu) +OPENCV_HAL_IMPL_LSX_REDUCE_16(v_uint8x16, uchar, max, __lsx_vmax_bu) +OPENCV_HAL_IMPL_LSX_REDUCE_16(v_int8x16, schar, min, __lsx_vmin_b) +OPENCV_HAL_IMPL_LSX_REDUCE_16(v_int8x16, schar, max, __lsx_vmax_b) + +#define OPENCV_HAL_IMPL_LSX_REDUCE_8(_Tpvec, sctype, func, intrin) \ + inline sctype v_reduce_##func(const _Tpvec &a) \ + { \ + __m128i val = intrin(a.val, __lsx_vbsrl_v(a.val, 8)); \ + val = intrin(val, __lsx_vbsrl_v(val, 4)); \ + val = intrin(val, __lsx_vbsrl_v(val, 2)); \ + return (sctype)__lsx_vpickve2gr_h(val, 0); \ + } + +OPENCV_HAL_IMPL_LSX_REDUCE_8(v_uint16x8, ushort, min, __lsx_vmin_hu) +OPENCV_HAL_IMPL_LSX_REDUCE_8(v_uint16x8, ushort, max, __lsx_vmax_hu) +OPENCV_HAL_IMPL_LSX_REDUCE_8(v_int16x8, short, min, __lsx_vmin_h) +OPENCV_HAL_IMPL_LSX_REDUCE_8(v_int16x8, short, max, __lsx_vmax_h) + +#define OPENCV_HAL_IMPL_LSX_REDUCE_4(_Tpvec, sctype, func, intrin) \ + inline sctype v_reduce_##func(const _Tpvec &a) \ + { \ + __m128i val = intrin(a.val, __lsx_vbsrl_v(a.val, 8)); \ + val = intrin(val, __lsx_vbsrl_v(val, 4)); \ + return (sctype)__lsx_vpickve2gr_w(val, 0); \ + } + +OPENCV_HAL_IMPL_LSX_REDUCE_4(v_uint32x4, unsigned, min, __lsx_vmin_wu) +OPENCV_HAL_IMPL_LSX_REDUCE_4(v_uint32x4, unsigned, max, __lsx_vmax_wu) +OPENCV_HAL_IMPL_LSX_REDUCE_4(v_int32x4, int, min, __lsx_vmin_w) +OPENCV_HAL_IMPL_LSX_REDUCE_4(v_int32x4, int, max, __lsx_vmax_w) + +#define OPENCV_HAL_IMPL_LSX_REDUCE_FLT(func, intrin) \ + inline float v_reduce_##func(const v_float32x4 &a) \ + { \ + __m128 val = a.val; \ + val = intrin(val, (__m128)__lsx_vbsrl_v((__m128i)val, 8)); \ + val = intrin(val, (__m128)__lsx_vbsrl_v((__m128i)val, 4)); \ + float *fval = (float*)&val; \ + return fval[0]; \ + } + +OPENCV_HAL_IMPL_LSX_REDUCE_FLT(min, __lsx_vfmin_s) +OPENCV_HAL_IMPL_LSX_REDUCE_FLT(max, __lsx_vfmax_s) + +inline int v_reduce_sum(const v_int32x4 &a) +{ + __m128i t1 = __lsx_vhaddw_d_w(a.val, a.val); + __m128i t2 = __lsx_vhaddw_q_d(t1, t1); + return (int)__lsx_vpickve2gr_w(t2, 0); +} + +inline unsigned v_reduce_sum(const v_uint32x4 &a) +{ + __m128i t1 = __lsx_vhaddw_du_wu(a.val, a.val); + __m128i t2 = __lsx_vhaddw_qu_du(t1, t1); + return (int)__lsx_vpickve2gr_w(t2, 0); +} + +inline int v_reduce_sum(const v_int16x8 &a) +{ + __m128i t1 = __lsx_vhaddw_w_h(a.val, a.val); + __m128i t2 = __lsx_vhaddw_d_w(t1, t1); + __m128i t3 = __lsx_vhaddw_q_d(t2, t2); + return (int)__lsx_vpickve2gr_w(t3, 0); +} + +inline unsigned v_reduce_sum(const v_uint16x8 &a) +{ + __m128i t1 = __lsx_vhaddw_wu_hu(a.val, a.val); + __m128i t2 = __lsx_vhaddw_du_wu(t1, t1); + __m128i t3 = __lsx_vhaddw_qu_du(t2, t2); + return (int)__lsx_vpickve2gr_w(t3, 0); +} + +inline float v_reduce_sum(const v_float32x4 &a) +{ + __m128i val = (__m128i)a.val; + val = __lsx_vbsrl_v(val, 8); + __m128 result = __lsx_vfadd_s(a.val, (__m128)val); + float *pa = (float*)&result; + return (float)(pa[0] + pa[1]); +} + +inline uint64 v_reduce_sum(const v_uint64x2 &a) +{ + __m128i t0 = __lsx_vhaddw_qu_du(a.val, a.val); + return (uint64)__lsx_vpickve2gr_du(t0, 0); +} + +inline int64 v_reduce_sum(const v_int64x2 &a) +{ + __m128i t0 = __lsx_vhaddw_q_d(a.val, a.val); + return (int64)__lsx_vpickve2gr_d(t0, 0); +} + +inline double v_reduce_sum(const v_float64x2 &a) +{ + double *pa = (double*)&a; + return pa[0] + pa[1]; +} + +inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, + const v_float32x4& c, const v_float32x4& d) +{ + __m128i a0 = (__m128i)a.val; + __m128i b0 = (__m128i)b.val; + __m128i c0 = (__m128i)c.val; + __m128i d0 = (__m128i)d.val; + __m128i ac_l = __lsx_vilvl_w(c0, a0); + __m128i ac_h = __lsx_vilvh_w(c0, a0); + __m128i bd_l = __lsx_vilvl_w(d0, b0); + __m128i bd_h = __lsx_vilvh_w(d0, b0); + __m128 ac = __lsx_vfadd_s((__m128)ac_l, (__m128)ac_h); + __m128 bd = __lsx_vfadd_s((__m128)bd_l, (__m128)bd_h); + return v_float32x4(__lsx_vfadd_s((__m128)__lsx_vilvl_w((__m128i)bd, (__m128i)ac), + (__m128)__lsx_vilvh_w((__m128i)bd, (__m128i)ac))); +} + +inline unsigned v_reduce_sad(const v_int8x16& a, const v_int8x16& b) +{ + __m128i t0 = __lsx_vabsd_b(a.val, b.val); + __m128i t1 = __lsx_vhaddw_hu_bu(t0, t0); + __m128i t2 = __lsx_vhaddw_wu_hu(t1, t1); + __m128i t3 = __lsx_vhaddw_du_wu(t2, t2); + __m128i t4 = __lsx_vhaddw_qu_du(t3, t3); + return (unsigned)__lsx_vpickve2gr_w(t4, 0); +} + +inline unsigned v_reduce_sad(const v_uint8x16& a, const v_uint8x16& b) +{ + __m128i t0 = __lsx_vabsd_bu(a.val, b.val); + __m128i t1 = __lsx_vhaddw_hu_bu(t0, t0); + __m128i t2 = __lsx_vhaddw_wu_hu(t1, t1); + __m128i t3 = __lsx_vhaddw_du_wu(t2, t2); + __m128i t4 = __lsx_vhaddw_qu_du(t3, t3); + return (unsigned)__lsx_vpickve2gr_w(t4, 0); +} + +inline unsigned v_reduce_sad(const v_uint16x8& a, const v_uint16x8& b) +{ + __m128i t0 = __lsx_vabsd_hu(a.val, b.val); + __m128i t1 = __lsx_vhaddw_wu_hu(t0, t0); + __m128i t2 = __lsx_vhaddw_du_wu(t1, t1); + __m128i t3 = __lsx_vhaddw_qu_du(t2, t2); + return (unsigned)__lsx_vpickve2gr_w(t3, 0); +} + +inline unsigned v_reduce_sad(const v_int16x8& a, const v_int16x8& b) +{ + __m128i t0 = __lsx_vabsd_h(a.val, b.val); + __m128i t1 = __lsx_vhaddw_wu_hu(t0, t0); + __m128i t2 = __lsx_vhaddw_du_wu(t1, t1); + __m128i t3 = __lsx_vhaddw_qu_du(t2, t2); + return (unsigned)__lsx_vpickve2gr_w(t3, 0); +} + +inline unsigned v_reduce_sad(const v_uint32x4& a, const v_uint32x4& b) +{ + __m128i t0 = __lsx_vabsd_wu(a.val, b.val); + __m128i t1 = __lsx_vhaddw_du_wu(t0, t0); + __m128i t2 = __lsx_vhaddw_qu_du(t1, t1); + return (unsigned)__lsx_vpickve2gr_w(t2, 0); +} + +inline unsigned v_reduce_sad(const v_int32x4& a, const v_int32x4& b) +{ + __m128i t0 = __lsx_vabsd_w(a.val, b.val); + __m128i t1 = __lsx_vhaddw_du_wu(t0, t0); + __m128i t2 = __lsx_vhaddw_qu_du(t1, t1); + return (unsigned)__lsx_vpickve2gr_w(t2, 0); +} + +inline float v_reduce_sad(const v_float32x4& a, const v_float32x4& b) +{ + v_float32x4 a_b = a - b; + return v_reduce_sum(v_float32x4((__m128i)a_b.val & __lsx_vreplgr2vr_w(0x7fffffff))); +} + +/** Popcount **/ +#define OPENCV_HAL_IMPL_LSX_POPCOUNT(_Tpvec, _Tp, suffix) \ +inline _Tpvec v_popcount(const _Tp& a) \ +{ return _Tpvec(__lsx_vpcnt_##suffix(a.val)); } + +OPENCV_HAL_IMPL_LSX_POPCOUNT(v_uint8x16, v_uint8x16, b); +OPENCV_HAL_IMPL_LSX_POPCOUNT(v_uint8x16, v_int8x16, b); +OPENCV_HAL_IMPL_LSX_POPCOUNT(v_uint16x8, v_uint16x8, h); +OPENCV_HAL_IMPL_LSX_POPCOUNT(v_uint16x8, v_int16x8, h); +OPENCV_HAL_IMPL_LSX_POPCOUNT(v_uint32x4, v_uint32x4, w); +OPENCV_HAL_IMPL_LSX_POPCOUNT(v_uint32x4, v_int32x4, w); +OPENCV_HAL_IMPL_LSX_POPCOUNT(v_uint64x2, v_uint64x2, d); +OPENCV_HAL_IMPL_LSX_POPCOUNT(v_uint64x2, v_int64x2, d); + +/** Mask **/ +#define OPENCV_HAL_IMPL_REINTERPRET_INT(ft, tt) \ +inline tt reinterpret_int(ft x) { union {ft l; tt i;} v; v.l = x; return v.i; } +OPENCV_HAL_IMPL_REINTERPRET_INT(uchar, schar) +OPENCV_HAL_IMPL_REINTERPRET_INT(schar, schar) +OPENCV_HAL_IMPL_REINTERPRET_INT(ushort, short) +OPENCV_HAL_IMPL_REINTERPRET_INT(short, short) +OPENCV_HAL_IMPL_REINTERPRET_INT(unsigned, int) +OPENCV_HAL_IMPL_REINTERPRET_INT(int, int) +OPENCV_HAL_IMPL_REINTERPRET_INT(float, int) +OPENCV_HAL_IMPL_REINTERPRET_INT(uint64, int64) +OPENCV_HAL_IMPL_REINTERPRET_INT(int64, int64) +OPENCV_HAL_IMPL_REINTERPRET_INT(double, int64) + +inline int v_signmask(const v_int8x16& a) +{ + __m128i result = __lsx_vmskltz_b(a.val); + return __lsx_vpickve2gr_w(result, 0); +} +inline int v_signmask(const v_uint8x16& a) +{ return v_signmask(v_reinterpret_as_s8(a)) ;} + +inline int v_signmask(const v_int16x8 &a) +{ + __m128i result = __lsx_vmskltz_h(a.val); + return __lsx_vpickve2gr_w(result, 0); +} +inline int v_signmask(const v_uint16x8 &a) +{ return v_signmask(v_reinterpret_as_s16(a)); } + +inline int v_signmask(const v_uint32x4& a) +{ + __m128i result = __lsx_vmskltz_w(a.val); + return __lsx_vpickve2gr_w(result, 0); +} +inline int v_signmask(const v_int32x4& a) +{ return v_signmask(v_reinterpret_as_u32(a)); } + +inline int v_signmask(const v_uint64x2& a) +{ + __m128i result = __lsx_vmskltz_d(a.val); + return __lsx_vpickve2gr_w(result, 0); +} +inline int v_signmask(const v_int64x2& a) +{ return v_signmask(v_reinterpret_as_u64(a)); } + +inline int v_signmask(const v_float32x4& a) +{ return v_signmask(*(v_int32x4*)(&a)); } + +inline int v_signmask(const v_float64x2& a) +{ return v_signmask(*(v_int64x2*)(&a)); } + +inline int v_scan_forward(const v_int8x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))); } +inline int v_scan_forward(const v_uint8x16& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))); } +inline int v_scan_forward(const v_int16x8& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 2; } +inline int v_scan_forward(const v_uint16x8& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 2; } +inline int v_scan_forward(const v_int32x4& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 4; } +inline int v_scan_forward(const v_uint32x4& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 4; } +inline int v_scan_forward(const v_float32x4& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 4; } +inline int v_scan_forward(const v_int64x2& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 8; } +inline int v_scan_forward(const v_uint64x2& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 8; } +inline int v_scan_forward(const v_float64x2& a) { return trailingZeros32(v_signmask(v_reinterpret_as_s8(a))) / 8; } + +/** Checks **/ +#define OPENCV_HAL_IMPL_LSX_CHECK(_Tpvec, allmask) \ + inline bool v_check_all(const _Tpvec& a) { return v_signmask(a) == allmask; } \ + inline bool v_check_any(const _Tpvec& a) { return v_signmask(a) != 0; } +OPENCV_HAL_IMPL_LSX_CHECK(v_uint8x16, 65535) +OPENCV_HAL_IMPL_LSX_CHECK(v_int8x16, 65535) +OPENCV_HAL_IMPL_LSX_CHECK(v_uint16x8, 255); +OPENCV_HAL_IMPL_LSX_CHECK(v_int16x8, 255); +OPENCV_HAL_IMPL_LSX_CHECK(v_uint32x4, 15) +OPENCV_HAL_IMPL_LSX_CHECK(v_int32x4, 15) +OPENCV_HAL_IMPL_LSX_CHECK(v_uint64x2, 3) +OPENCV_HAL_IMPL_LSX_CHECK(v_int64x2, 3) +OPENCV_HAL_IMPL_LSX_CHECK(v_float32x4, 15) +OPENCV_HAL_IMPL_LSX_CHECK(v_float64x2, 3) + +///////////// Other math ///////////// + +/** Some frequent operations **/ +#define OPENCV_HAL_IMPL_LSX_MULADD(_Tpvec, suffix) \ + inline _Tpvec v_fma(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { return _Tpvec(__lsx_vfmadd_##suffix(a.val, b.val, c.val)); } \ + inline _Tpvec v_muladd(const _Tpvec& a, const _Tpvec &b, const _Tpvec& c) \ + { return _Tpvec(__lsx_vfmadd_##suffix(a.val, b.val, c.val)); } \ + inline _Tpvec v_sqrt(const _Tpvec& x) \ + { return _Tpvec(__lsx_vfsqrt_##suffix(x.val)); } \ + inline _Tpvec v_sqr_magnitude(const _Tpvec& a, const _Tpvec& b) \ + { return v_fma(a, a, b * b); } \ + inline _Tpvec v_magnitude(const _Tpvec& a, const _Tpvec& b) \ + { return v_sqrt(v_fma(a, a, b * b)); } + +OPENCV_HAL_IMPL_LSX_MULADD(v_float32x4, s) +OPENCV_HAL_IMPL_LSX_MULADD(v_float64x2, d) + +inline v_int32x4 v_fma(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) +{ return v_int32x4(__lsx_vmadd_w(c.val, a.val, b.val)); } + +inline v_int32x4 v_muladd(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) +{ return v_fma(a, b, c); } + +inline v_float32x4 v_invsqrt(const v_float32x4& x) +{ + return v_float32x4(__lsx_vfrsqrt_s(x.val)); +} + +inline v_float64x2 v_invsqrt(const v_float64x2& x) +{ + return v_float64x2(__lsx_vfrsqrt_d(x.val)); +} + +/** Absolute values **/ +#define OPENCV_HAL_IMPL_LSX_ABS(_Tpvec, suffix) \ + inline v_u##_Tpvec v_abs(const v_##_Tpvec& x) \ + { return v_u##_Tpvec(__lsx_vabsd_##suffix(x.val, __lsx_vldi(0))); } + +OPENCV_HAL_IMPL_LSX_ABS(int8x16, b) +OPENCV_HAL_IMPL_LSX_ABS(int16x8, h) +OPENCV_HAL_IMPL_LSX_ABS(int32x4, w) + +inline v_float32x4 v_abs(const v_float32x4& x) +{ return v_float32x4(*((__m128i*)&x) & __lsx_vreplgr2vr_w(0x7fffffff)); } +inline v_float64x2 v_abs(const v_float64x2& x) +{ return v_float64x2(*((__m128i*)&x) & __lsx_vreplgr2vr_d(0x7fffffffffffffff)); } + +/** Absolute difference **/ + +inline v_uint8x16 v_absdiff(const v_uint8x16& a, const v_uint8x16& b) +{ return (v_uint8x16)__lsx_vabsd_bu(a.val, b.val); } +inline v_uint16x8 v_absdiff(const v_uint16x8& a, const v_uint16x8& b) +{ return (v_uint16x8)__lsx_vabsd_hu(a.val, b.val); } +inline v_uint32x4 v_absdiff(const v_uint32x4& a, const v_uint32x4& b) +{ return (v_uint32x4)__lsx_vabsd_wu(a.val, b.val); } + +inline v_uint8x16 v_absdiff(const v_int8x16& a, const v_int8x16& b) +{ return (v_uint8x16)__lsx_vabsd_b(a.val, b.val); } +inline v_uint16x8 v_absdiff(const v_int16x8& a, const v_int16x8& b) +{ return (v_uint16x8)__lsx_vabsd_h(a.val, b.val); } +inline v_uint32x4 v_absdiff(const v_int32x4& a, const v_int32x4& b) +{ return (v_uint32x4)__lsx_vabsd_w(a.val, b.val); } + +inline v_float32x4 v_absdiff(const v_float32x4& a, const v_float32x4& b) +{ return v_abs(a - b); } + +inline v_float64x2 v_absdiff(const v_float64x2& a, const v_float64x2& b) +{ return v_abs(a - b); } + +/** Saturating absolute difference **/ +inline v_int8x16 v_absdiffs(const v_int8x16& a, const v_int8x16& b) +{ + v_int8x16 d = a - b; + v_int8x16 m = a < b; + return (d ^ m) - m; +} +inline v_int16x8 v_absdiffs(const v_int16x8& a, const v_int16x8& b) +{ return v_max(a, b) - v_min(a, b); } + +///////// Conversions ///////// + +/** Rounding **/ +inline v_int32x4 v_round(const v_float32x4& a) +{ return v_int32x4(__lsx_vftint_w_s(a.val)); } + +inline v_int32x4 v_round(const v_float64x2& a) +{ return v_int32x4(__lsx_vftint_w_d(a.val, a.val)); } + +inline v_int32x4 v_round(const v_float64x2& a, const v_float64x2& b) +{ return v_int32x4(__lsx_vftint_w_d(b.val, a.val)); } + +inline v_int32x4 v_trunc(const v_float32x4& a) +{ return v_int32x4(__lsx_vftintrz_w_s(a.val)); } + +inline v_int32x4 v_trunc(const v_float64x2& a) +{ return v_int32x4(__lsx_vftintrz_w_d(a.val, a.val)); } + +inline v_int32x4 v_floor(const v_float32x4& a) +{ return v_int32x4(__lsx_vftintrz_w_s(__m128(__lsx_vfrintrm_s(a.val)))); } + +inline v_int32x4 v_floor(const v_float64x2& a) +{ return v_trunc(v_float64x2(__lsx_vfrintrm_d(a.val))); } + +inline v_int32x4 v_ceil(const v_float32x4& a) +{ return v_int32x4(__lsx_vftintrz_w_s(__m128(__lsx_vfrintrp_s(a.val)))); } + +inline v_int32x4 v_ceil(const v_float64x2& a) +{ return v_trunc(v_float64x2(__lsx_vfrintrp_d(a.val))); } + +/** To float **/ +inline v_float32x4 v_cvt_f32(const v_int32x4& a) +{ return v_float32x4(__lsx_vffint_s_w(a.val)); } + +inline v_float32x4 v_cvt_f32(const v_float64x2& a) +{ return v_float32x4(__lsx_vfcvt_s_d(a.val, a.val)); } + +inline v_float32x4 v_cvt_f32(const v_float64x2& a, const v_float64x2& b) +{ return v_float32x4(__lsx_vfcvt_s_d(b.val, a.val)); } + +inline v_float64x2 v_cvt_f64(const v_int32x4& a) +{ return v_float64x2(__lsx_vffintl_d_w(a.val)); } + +inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) +{ return v_float64x2(__lsx_vffinth_d_w(a.val)); } + +inline v_float64x2 v_cvt_f64(const v_float32x4& a) +{ return v_float64x2(__lsx_vfcvtl_d_s(a.val)); } + +inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) +{ return v_float64x2(__lsx_vfcvth_d_s(a.val)); } + +inline v_float64x2 v_cvt_f64(const v_int64x2& v) +{ return v_float64x2(__lsx_vffint_d_l(v.val)); } + + +//////////////// Lookup table access //////////////// +inline v_int8x16 v_lut(const schar* tab, const int* idx) +{ + return v_int8x16(_v128_setr_b(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]], + tab[idx[4]], tab[idx[5]], tab[idx[6]], tab[idx[7]], tab[idx[8]], + tab[idx[9]], tab[idx[10]], tab[idx[11]], tab[idx[12]], tab[idx[13]], + tab[idx[14]], tab[idx[15]])); +} + +inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) +{ + return v_int8x16(_v128_setr_h(*(const short*)(tab + idx[0]), *(const short*)(tab + idx[1]), + *(const short*)(tab + idx[2]), *(const short*)(tab + idx[3]), *(const short*)(tab + idx[4]), + *(const short*)(tab + idx[5]), *(const short*)(tab + idx[6]), *(const short*)(tab + idx[7]))); +} + +inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) +{ + return v_int8x16(_v128_setr_w(*(const int*)(tab + idx[0]), *(const int*)(tab + idx[1]), + *(const int*)(tab + idx[2]), *(const int*)(tab + idx[3]))); +} + +inline v_uint8x16 v_lut(const uchar* tab, const int* idx) +{ return v_reinterpret_as_u8(v_lut((const schar*)tab, idx)); } +inline v_uint8x16 v_lut_pairs(const uchar* tab, const int* idx) +{ return v_reinterpret_as_u8(v_lut_pairs((const schar*)tab, idx)); } +inline v_uint8x16 v_lut_quads(const uchar* tab, const int* idx) +{ return v_reinterpret_as_u8(v_lut_quads((const schar*)tab, idx)); } + +inline v_int16x8 v_lut(const short* tab, const int* idx) +{ + return v_int16x8(_v128_setr_h(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]], + tab[idx[4]], tab[idx[5]], tab[idx[6]], tab[idx[7]])); +} +inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) +{ + return v_int16x8(_v128_setr_w(*(const int*)(tab + idx[0]), *(const int*)(tab + idx[1]), + *(const int*)(tab + idx[2]), *(const int*)(tab + idx[3]))); +} +inline v_int16x8 v_lut_quads(const short* tab, const int* idx) +{ + return v_int16x8(_v128_setr_d(*(const int64_t*)(tab + idx[0]), *(const int64_t*)(tab + idx[1]))); +} + +inline v_uint16x8 v_lut(const ushort* tab, const int* idx) +{ return v_reinterpret_as_u16(v_lut((const short *)tab, idx)); } +inline v_uint16x8 v_lut_pairs(const ushort* tab, const int* idx) +{ return v_reinterpret_as_u16(v_lut_pairs((const short *)tab, idx)); } +inline v_uint16x8 v_lut_quads(const ushort* tab, const int* idx) +{ return v_reinterpret_as_u16(v_lut_quads((const short *)tab, idx)); } + +inline v_int32x4 v_lut(const int* tab, const int* idx) +{ + return v_int32x4(_v128_setr_w(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]])); +} +inline v_int32x4 v_lut_pairs(const int *tab, const int* idx) +{ + return v_int32x4(_v128_setr_d(*(const int64_t*)(tab + idx[0]), *(const int64_t*)(tab + idx[1]))); +} +inline v_int32x4 v_lut_quads(const int* tab, const int* idx) +{ + return v_int32x4(__lsx_vld(tab + idx[0], 0)); +} + +inline v_uint32x4 v_lut(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut((const int *)tab, idx)); } +inline v_uint32x4 v_lut_pairs(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut_pairs((const int *)tab, idx)); } +inline v_uint32x4 v_lut_quads(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut_quads((const int *)tab, idx)); } + +inline v_int64x2 v_lut(const int64_t* tab, const int *idx) +{ + return v_int64x2(_v128_setr_d(tab[idx[0]], tab[idx[1]])); +} +inline v_int64x2 v_lut_pairs(const int64_t* tab, const int* idx) +{ + return v_int64x2(__lsx_vld(tab + idx[0], 0)); +} + +inline v_uint64x2 v_lut(const uint64_t* tab, const int* idx) { return v_reinterpret_as_u64(v_lut((const int64_t *)tab, idx)); } +inline v_uint64x2 v_lut_pairs(const uint64_t* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_pairs((const int64_t *)tab, idx)); } + +inline v_float32x4 v_lut(const float* tab, const int* idx) +{ + return v_float32x4(_v128_setr_ps(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]])); +} +inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) +{ + return v_float32x4((__m128)_v128_setr_pd(*(const double*)(tab + idx[0]), *(const double*)(tab + idx[1]))); +} +inline v_float32x4 v_lut_quads(const float* tab, const int* idx) +{ + return v_float32x4((__m128)__lsx_vld(tab + idx[0], 0)); +} + +inline v_float64x2 v_lut(const double* tab, const int* idx) +{ + return v_float64x2(_v128_setr_pd(tab[idx[0]], tab[idx[1]])); +} +inline v_float64x2 v_lut_pairs(const double* tab, const int* idx) +{ + return v_float64x2((__m128d)__lsx_vld(tab + idx[0], 0)); +} + +inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) +{ + int *idx = (int*)&idxvec.val; + return v_lut(tab, idx); +} + +inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) +{ + return v_reinterpret_as_u32(v_lut((const int *)tab, idxvec)); +} + +inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) +{ + const int *idx = (const int*)&idxvec.val; + return v_lut(tab, idx); +} + +inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) +{ + const int *idx = (const int*)&idxvec.val; + return v_lut(tab, idx); +} + +inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_float32x4& x, v_float32x4& y) +{ + const int *idx = (const int*)&idxvec.val; + __m128i xy0 = __lsx_vld(tab + idx[0], 0); + __m128i xy1 = __lsx_vld(tab + idx[1], 0); + __m128i xy2 = __lsx_vld(tab + idx[2], 0); + __m128i xy3 = __lsx_vld(tab + idx[3], 0); + __m128i xy01 = __lsx_vilvl_d(xy1, xy0); + __m128i xy23 = __lsx_vilvl_d(xy3, xy2); + __m128i xxyy02 = __lsx_vilvl_w(xy23, xy01); + __m128i xxyy13 = __lsx_vilvh_w(xy23, xy01); + x = v_float32x4((__m128)__lsx_vilvl_w(xxyy13, xxyy02)); + y = v_float32x4((__m128)__lsx_vilvh_w(xxyy13, xxyy02)); +} + +inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_float64x2& x, v_float64x2& y) +{ + const int* idx = (const int*)&idxvec.val; + __m128i xy0 = __lsx_vld(tab + idx[0], 0); + __m128i xy1 = __lsx_vld(tab + idx[1], 0); + x = v_float64x2((__m128d)__lsx_vilvl_d(xy1, xy0)); + y = v_float64x2((__m128d)__lsx_vilvh_d(xy1, xy0)); +} + +inline v_int8x16 v_interleave_pairs(const v_int8x16& vec) +{ + return v_int8x16(__lsx_vshuf_b(vec.val, vec.val, + _v128_setr_d(0x0705060403010200, 0x0f0d0e0c0b090a08))); +} +inline v_uint8x16 v_interleave_pairs(const v_uint8x16& vec) +{ return v_reinterpret_as_u8(v_interleave_pairs(v_reinterpret_as_s8(vec))); } +inline v_int8x16 v_interleave_quads(const v_int8x16& vec) +{ + return v_int8x16(__lsx_vshuf_b(vec.val, vec.val, + _v128_setr_d(0x0703060205010400, 0x0f0b0e0a0d090c08))); +} +inline v_uint8x16 v_interleave_quads(const v_uint8x16& vec) +{ return v_reinterpret_as_u8(v_interleave_quads(v_reinterpret_as_s8(vec))); } + +inline v_int16x8 v_interleave_pairs(const v_int16x8& vec) +{ + return v_int16x8(__lsx_vshuf_b(vec.val, vec.val, + _v128_setr_d(0x0706030205040100, 0x0f0e0b0a0d0c0908))); +} +inline v_uint16x8 v_interleave_pairs(const v_uint16x8& vec) +{ return v_reinterpret_as_u16(v_interleave_pairs(v_reinterpret_as_s16(vec))); } +inline v_int16x8 v_interleave_quads(const v_int16x8& vec) +{ + return v_int16x8(__lsx_vshuf_b(vec.val, vec.val, + _v128_setr_d(0x0b0a030209080100, 0x0f0e07060d0c0504))); +} +inline v_uint16x8 v_interleave_quads(const v_uint16x8& vec) +{ return v_reinterpret_as_u16(v_interleave_quads(v_reinterpret_as_s16(vec))); } + +inline v_int32x4 v_interleave_pairs(const v_int32x4& vec) +{ + return v_int32x4(__lsx_vshuf4i_w(vec.val, 0xd8)); +} +inline v_uint32x4 v_interleave_pairs(const v_uint32x4& vec) +{ return v_reinterpret_as_u32(v_interleave_pairs(v_reinterpret_as_s32(vec))); } + +inline v_float32x4 v_interleave_pairs(const v_float32x4& vec) +{ return v_reinterpret_as_f32(v_interleave_pairs(v_reinterpret_as_s32(vec))); } + +inline v_int8x16 v_pack_triplets(const v_int8x16& vec) +{ + __m128i zero = __lsx_vldi(0); + return v_int8x16(__lsx_vshuf_b(zero, vec.val, + _v128_set_d(0x1211100f0e0d0c0a, 0x0908060504020100))); +} +inline v_uint8x16 v_pack_triplets(const v_uint8x16& vec) +{ return v_reinterpret_as_u8(v_pack_triplets(v_reinterpret_as_s8(vec))); } + +inline v_int16x8 v_pack_triplets(const v_int16x8& vec) +{ + __m128i zero = __lsx_vldi(0); + return v_int16x8(__lsx_vshuf_b(zero, vec.val, + _v128_set_d(0x11100f0e0d0c0b0a, 0x0908050403020100))); +} +inline v_uint16x8 v_pack_triplets(const v_uint16x8& vec) +{ return v_reinterpret_as_u16(v_pack_triplets(v_reinterpret_as_s16(vec))); } + +inline v_int32x4 v_pack_triplets(const v_int32x4& vec) { return vec; } +inline v_uint32x4 v_pack_triplets(const v_uint32x4& vec) { return vec; } +inline v_float32x4 v_pack_triplets(const v_float32x4& vec) { return vec; } + +//////////// Matrix operations ///////// + +/////////// Dot Product ///////// + +// 16 >> 32 +inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) +{ + __m128i x = a.val, y = b.val; + return v_int32x4(__lsx_vmaddwod_w_h(__lsx_vmulwev_w_h(x, y), x, y)); +} +inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) +{ + __m128i x = a.val, y = b.val, z = c.val; + __m128i t = __lsx_vmaddwev_w_h(z, x, y); + return v_int32x4(__lsx_vmaddwod_w_h(t, x, y)); +} + +// 32 >> 64 +inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) +{ + __m128i x = a.val, y = b.val; + return v_int64x2(__lsx_vmaddwod_d_w(__lsx_vmulwev_d_w(x, y), x, y)); +} +inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) +{ + __m128i x = a.val, y = b.val, z = c.val; + __m128i t = __lsx_vmaddwev_d_w(z, x, y); + return v_int64x2(__lsx_vmaddwod_d_w(t, x, y)); +} + +// 8 >> 32 +inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) +{ + __m128i x = a.val, y = b.val; + __m128i even = __lsx_vmulwev_h_bu(x, y); + __m128i odd = __lsx_vmulwod_h_bu(x, y); + __m128i prod0 = __lsx_vhaddw_wu_hu(even, even); + __m128i prod1 = __lsx_vhaddw_wu_hu(odd, odd); + return v_uint32x4(__lsx_vadd_w(prod0, prod1)); +} + +inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) +{ return v_dotprod_expand(a, b) + c ;} + +inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) +{ + __m128i x = a.val, y = b.val; + __m128i even = __lsx_vmulwev_h_b(x, y); + __m128i odd = __lsx_vmulwod_h_b(x, y); + __m128i prod0 = __lsx_vhaddw_w_h(even, even); + __m128i prod1 = __lsx_vhaddw_w_h(odd, odd); + return v_int32x4(__lsx_vadd_w(prod0, prod1)); +} +inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) +{ return v_dotprod_expand(a, b) + c; } + +// 16 >> 64 +inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) +{ + __m128i x = a.val, y = b.val; + __m128i even = __lsx_vmulwev_w_hu(x, y); + __m128i odd = __lsx_vmulwod_w_hu(x, y); + __m128i prod0 = __lsx_vhaddw_du_wu(even, even); + __m128i prod1 = __lsx_vhaddw_du_wu(odd, odd); + return v_uint64x2(__lsx_vadd_d(prod0, prod1)); +} +inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) +{ return v_dotprod_expand(a, b) + c; } + +inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) +{ + __m128i x = a.val, y = b.val; + __m128i even = __lsx_vmulwev_w_h(x, y); + __m128i odd = __lsx_vmulwod_w_h(x, y); + __m128i prod0 = __lsx_vhaddw_d_w(even, even); + __m128i prod1 = __lsx_vhaddw_d_w(odd, odd); + return v_int64x2(__lsx_vadd_d(prod0, prod1)); +} +inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) +{ return v_dotprod_expand(a, b) + c; } + +//32 >> 64f +inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b) +{ return v_cvt_f64(v_dotprod(a, b)); } +inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b, const v_float64x2& c) +{ return v_dotprod_expand(a, b) + c; } + + +///////// Fast Dot Product ////// + +// 16 >> 32 +inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) +{ return v_dotprod(a, b); } +inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) +{ return v_dotprod(a, b, c); } + +// 32 >> 64 +inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) +{ return v_dotprod(a, b); } +inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) +{ return v_dotprod(a, b, c); } + +// 8 >> 32 +inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b) +{ return v_dotprod_expand(a, b); } +inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) +{ return v_dotprod_expand(a, b, c); } + +inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) +{ return v_dotprod_expand(a, b); } +inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) +{ return v_dotprod_expand(a, b, c); } + +// 16 >> 64 +inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b) +{ + __m128i x = a.val, y = b.val; + __m128i even = __lsx_vmulwev_w_hu(x, y); + __m128i odd = __lsx_vmulwod_w_hu(x, y); + __m128i prod0 = __lsx_vhaddw_du_wu(even, even); + __m128i prod1 = __lsx_vhaddw_du_wu(odd, odd); + return v_uint64x2(__lsx_vilvl_d(__lsx_vhaddw_qu_du(prod0, prod0), __lsx_vhaddw_qu_du(prod1, prod1))); +} +inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) +{ return v_dotprod_expand_fast(a, b) + c; } + +inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) +{ + __m128i x = a.val, y = b.val; + __m128i prod = __lsx_vmaddwod_w_h(__lsx_vmulwev_w_h(x, y), x, y); + __m128i sign = __lsx_vsrai_w(prod, 31); + __m128i lo = __lsx_vilvl_w(sign, prod); + __m128i hi = __lsx_vilvh_w(sign, prod); + return v_int64x2(__lsx_vadd_d(lo, hi)); +} +inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) +{ return v_dotprod_expand_fast(a, b) + c; } + +// 32 >> 64f +inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b) +{ return v_dotprod_expand(a, b); } +inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b, const v_float64x2& c) +{ return v_dotprod_expand(a, b, c); } + +inline v_float32x4 v_matmul(const v_float32x4& v, const v_float32x4& m0, + const v_float32x4& m1, const v_float32x4& m2, const v_float32x4& m3) +{ + __m128i x = (__m128i)v.val; + __m128 v0 = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(x, 0x0), m0.val); + __m128 v1 = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(x, 0x55), m1.val); + __m128 v2 = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(x, 0xAA), m2.val); + __m128 v3 = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(x, 0xFF), m3.val); + + return v_float32x4(__lsx_vfadd_s(__lsx_vfadd_s(v0, v1), __lsx_vfadd_s(v2, v3))); +} + +inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, + const v_float32x4& m1, const v_float32x4& m2, const v_float32x4& a) +{ + __m128i x = (__m128i)v.val; + __m128 v0 = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(x, 0x0), m0.val); + __m128 v1 = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(x, 0x55), m1.val); + __m128 v2 = __lsx_vfmadd_s((__m128)__lsx_vshuf4i_w(x, 0xAA), m2.val, a.val); + + return v_float32x4(__lsx_vfadd_s(__lsx_vfadd_s(v0, v1), v2)); +} + +#define OPENCV_HAL_IMPL_LSX_TRANSPOSE4X4(_Tpvec, cast_from, cast_to) \ + inline void v_transpose4x4(const _Tpvec& a0, const _Tpvec& a1, \ + const _Tpvec& a2, const _Tpvec& a3, \ + _Tpvec& b0, _Tpvec& b1, _Tpvec& b2, _Tpvec& b3) \ + { \ + __m128i t0 = cast_from(__lsx_vilvl_w(a1.val, a0.val)); \ + __m128i t1 = cast_from(__lsx_vilvl_w(a3.val, a2.val)); \ + __m128i t2 = cast_from(__lsx_vilvh_w(a1.val, a0.val)); \ + __m128i t3 = cast_from(__lsx_vilvh_w(a3.val, a2.val)); \ + b0.val = cast_to(__lsx_vilvl_d(t1, t0)); \ + b1.val = cast_to(__lsx_vilvh_d(t1, t0)); \ + b2.val = cast_to(__lsx_vilvl_d(t3, t2)); \ + b3.val = cast_to(__lsx_vilvh_d(t3, t2)); \ + } + +OPENCV_HAL_IMPL_LSX_TRANSPOSE4X4(v_uint32x4, OPENCV_HAL_NOP, OPENCV_HAL_NOP) +OPENCV_HAL_IMPL_LSX_TRANSPOSE4X4(v_int32x4, OPENCV_HAL_NOP, OPENCV_HAL_NOP) + +inline void v_transpose4x4(const v_float32x4& a0, const v_float32x4& a1, + const v_float32x4& a2, const v_float32x4& a3, + v_float32x4& b0, v_float32x4& b1, v_float32x4& b2, v_float32x4& b3) +{ + __m128i vec0 = (__m128i)a0.val, vec1 = (__m128i)a1.val; + __m128i vec2 = (__m128i)a2.val, vec3 = (__m128i)a3.val; + __m128i t0 = __lsx_vilvl_w(vec1, vec0); + __m128i t1 = __lsx_vilvl_w(vec3, vec2); + __m128i t2 = __lsx_vilvh_w(vec1, vec0); + __m128i t3 = __lsx_vilvh_w(vec3, vec2); + b0.val = __m128(__lsx_vilvl_d(t1, t0)); + b1.val = __m128(__lsx_vilvh_d(t1, t0)); + b2.val = __m128(__lsx_vilvl_d(t3, t2)); + b3.val = __m128(__lsx_vilvh_d(t3, t2)); +} + +////////////////// Value reordering //////////////// + +/* Expand */ +#define OPENCV_HAL_IMPL_LSX_EXPAND(_Tpvec, _Tpwvec, _Tp, intrin_lo, intrin_hi) \ + inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ + { \ + b0.val = intrin_lo(a.val, 0); \ + b1.val = intrin_hi(a.val); \ + } \ + inline _Tpwvec v_expand_low(const _Tpvec& a) \ + { return _Tpwvec(intrin_lo(a.val, 0)); } \ + inline _Tpwvec v_expand_high(const _Tpvec& a) \ + { return _Tpwvec(intrin_hi(a.val)); } \ + inline _Tpwvec v_load_expand(const _Tp* ptr) \ + { \ + __m128i a = __lsx_vld(ptr, 0); \ + return _Tpwvec(intrin_lo(a, 0)); \ + } + +OPENCV_HAL_IMPL_LSX_EXPAND(v_uint8x16, v_uint16x8, uchar, __lsx_vsllwil_hu_bu, __lsx_vexth_hu_bu) +OPENCV_HAL_IMPL_LSX_EXPAND(v_int8x16, v_int16x8, schar, __lsx_vsllwil_h_b, __lsx_vexth_h_b) +OPENCV_HAL_IMPL_LSX_EXPAND(v_uint16x8, v_uint32x4, ushort, __lsx_vsllwil_wu_hu, __lsx_vexth_wu_hu) +OPENCV_HAL_IMPL_LSX_EXPAND(v_int16x8, v_int32x4, short, __lsx_vsllwil_w_h, __lsx_vexth_w_h) +OPENCV_HAL_IMPL_LSX_EXPAND(v_uint32x4, v_uint64x2, unsigned, __lsx_vsllwil_du_wu, __lsx_vexth_du_wu) +OPENCV_HAL_IMPL_LSX_EXPAND(v_int32x4, v_int64x2, int, __lsx_vsllwil_d_w, __lsx_vexth_d_w) + +#define OPENCV_HAL_IMPL_LSX_EXPAND_Q(_Tpvec, _Tp, intrin_lo, intrin_hi) \ + inline _Tpvec v_load_expand_q(const _Tp* ptr) \ + { \ + __m128i a = __lsx_vld(ptr, 0); \ + __m128i b = intrin_lo(a, 0); \ + return _Tpvec(intrin_hi(b, 0)); \ + } + +OPENCV_HAL_IMPL_LSX_EXPAND_Q(v_uint32x4, uchar, __lsx_vsllwil_hu_bu, __lsx_vsllwil_wu_hu) +OPENCV_HAL_IMPL_LSX_EXPAND_Q(v_int32x4, schar, __lsx_vsllwil_h_b, __lsx_vsllwil_w_h) + +/* pack */ +// 16 +inline v_int8x16 v_pack(const v_int16x8& a, const v_int16x8& b) +{ return v_int8x16(_lsx_packs_h(a.val, b.val)); } + +inline v_uint8x16 v_pack(const v_uint16x8& a, const v_uint16x8& b) +{ return v_uint8x16(__lsx_vssrlrni_bu_h(b.val, a.val, 0)); } + +inline v_uint8x16 v_pack_u(const v_int16x8& a, const v_int16x8& b) +{ return v_uint8x16(_lsx_packus_h(a.val, b.val)); } + +inline void v_pack_store(schar* ptr, const v_int16x8& a) +{ v_store_low(ptr, v_pack(a, a)); } + +inline void v_pack_store(uchar* ptr, const v_uint16x8& a) +{ v_store_low(ptr, v_pack(a, a)); } + +inline void v_pack_u_store(uchar* ptr, const v_int16x8& a) +{ v_store_low(ptr, v_pack_u(a, a)); } + +template inline +v_uint8x16 v_rshr_pack(const v_uint16x8& a, const v_uint16x8& b) +{ return v_uint8x16(__lsx_vssrlrni_bu_h(b.val, a.val, n)); } + +template inline +void v_rshr_pack_store(uchar* ptr, const v_uint16x8& a) +{ __lsx_vstelm_d(__lsx_vssrlrni_bu_h(a.val, a.val, n), ptr, 0, 0); } + +template inline +v_uint8x16 v_rshr_pack_u(const v_int16x8& a, const v_int16x8& b) +{ return v_uint8x16(__lsx_vssrarni_bu_h(b.val, a.val, n)); } + +template inline +void v_rshr_pack_u_store(uchar* ptr, const v_int16x8& a) +{ __lsx_vstelm_d(__lsx_vssrarni_bu_h(a.val, a.val, n), ptr, 0, 0); } + +template inline +v_int8x16 v_rshr_pack(const v_int16x8& a, const v_int16x8& b) +{ return v_int8x16(__lsx_vssrarni_b_h(b.val, a.val, n)); } + +template inline +void v_rshr_pack_store(schar* ptr, const v_int16x8& a) +{ __lsx_vstelm_d(__lsx_vssrarni_b_h(a.val, a.val, n), ptr, 0, 0); } + +//32 +inline v_int16x8 v_pack(const v_int32x4& a, const v_int32x4& b) +{ return v_int16x8(__lsx_vssrarni_h_w(b.val, a.val, 0)); } + +inline v_uint16x8 v_pack(const v_uint32x4& a, const v_uint32x4& b) +{ return v_uint16x8(__lsx_vssrlrni_hu_w(b.val, a.val, 0)); } + +inline v_uint16x8 v_pack_u(const v_int32x4& a, const v_int32x4& b) +{ return v_uint16x8(__lsx_vssrarni_hu_w(b.val, a.val, 0)); } + +inline void v_pack_store(short* ptr, const v_int32x4& a) +{ v_store_low(ptr, v_pack(a, a)); } + +inline void v_pack_store(ushort *ptr, const v_uint32x4& a) +{ __lsx_vstelm_d(__lsx_vssrlrni_hu_w(a.val, a.val, 0), ptr, 0, 0); } + +inline void v_pack_u_store(ushort* ptr, const v_int32x4& a) +{ __lsx_vstelm_d(__lsx_vssrarni_hu_w(a.val, a.val, 0), ptr, 0, 0); } + +template inline +v_uint16x8 v_rshr_pack(const v_uint32x4& a, const v_uint32x4& b) +{ return v_uint16x8(__lsx_vssrlrni_hu_w(b.val, a.val, n)); } + +template inline +void v_rshr_pack_store(ushort* ptr, const v_uint32x4& a) +{ __lsx_vstelm_d(__lsx_vssrlrni_hu_w(a.val, a.val, n), ptr, 0, 0); } + +template inline +v_uint16x8 v_rshr_pack_u(const v_int32x4& a, const v_int32x4& b) +{ return v_uint16x8(__lsx_vssrarni_hu_w(b.val, a.val, n)); } + +template inline +void v_rshr_pack_u_store(ushort* ptr, const v_int32x4& a) +{ __lsx_vstelm_d(__lsx_vssrarni_hu_w(a.val, a.val, n), ptr, 0, 0); } + +template inline +v_int16x8 v_rshr_pack(const v_int32x4& a, const v_int32x4& b) +{ return v_int16x8(__lsx_vssrarni_h_w(b.val, a.val, n)); } + +template inline +void v_rshr_pack_store(short* ptr, const v_int32x4& a) +{ __lsx_vstelm_d(__lsx_vssrarni_h_w(a.val, a.val, n), ptr, 0, 0); } + +// 64 +// Non-saturaing pack +inline v_uint32x4 v_pack(const v_uint64x2& a, const v_uint64x2& b) +{ return v_uint32x4(__lsx_vpickev_w(b.val, a.val)); } + +inline v_int32x4 v_pack(const v_int64x2& a, const v_int64x2& b) +{ return v_reinterpret_as_s32(v_pack(v_reinterpret_as_u64(a), v_reinterpret_as_u64(b))); } + +inline void v_pack_store(unsigned* ptr, const v_uint64x2& a) +{ __lsx_vstelm_d(__lsx_vshuf4i_w(a.val, 0x08), ptr, 0, 0); } + +inline void v_pack_store(int *ptr, const v_int64x2& a) +{ v_pack_store((unsigned*)ptr, v_reinterpret_as_u64(a)); } + +template inline +v_uint32x4 v_rshr_pack(const v_uint64x2& a, const v_uint64x2& b) +{ return v_uint32x4(__lsx_vsrlrni_w_d(b.val, a.val, n)); } + +template inline +void v_rshr_pack_store(unsigned* ptr, const v_uint64x2& a) +{ __lsx_vstelm_d(__lsx_vsrlrni_w_d(a.val, a.val, n), ptr, 0, 0); } + +template inline +v_int32x4 v_rshr_pack(const v_int64x2& a, const v_int64x2& b) +{ return v_int32x4(__lsx_vsrarni_w_d(b.val, a.val, n)); } + +template inline +void v_rshr_pack_store(int* ptr, const v_int64x2& a) +{ __lsx_vstelm_d(__lsx_vsrarni_w_d(a.val, a.val, n), ptr, 0, 0); } + +// pack boolean +inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) +{ return v_uint8x16(__lsx_vssrarni_b_h(b.val, a.val, 0)); } + +inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, + const v_uint32x4& c, const v_uint32x4& d) +{ + __m128i ab = __lsx_vssrarni_h_w(b.val, a.val, 0); + __m128i cd = __lsx_vssrarni_h_w(d.val, c.val, 0); + return v_uint8x16(__lsx_vssrarni_b_h(cd, ab, 0)); +} + +inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uint64x2& c, + const v_uint64x2& d, const v_uint64x2& e, const v_uint64x2& f, + const v_uint64x2& g, const v_uint64x2& h) +{ + __m128i ab = __lsx_vssrarni_w_d(b.val, a.val, 0); + __m128i cd = __lsx_vssrarni_w_d(d.val, c.val, 0); + __m128i ef = __lsx_vssrarni_w_d(f.val, e.val, 0); + __m128i gh = __lsx_vssrarni_w_d(h.val, g.val, 0); + + __m128i abcd = __lsx_vssrarni_h_w(cd, ab, 0); + __m128i efgh = __lsx_vssrarni_h_w(gh, ef, 0); + return v_uint8x16(__lsx_vssrarni_b_h(efgh, abcd, 0)); +} + +/* Recombine */ +// its up there with load and store operations + +/* Extract */ +#define OPENCV_HAL_IMPL_LSX_EXTRACT(_Tpvec) \ + template \ + inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ + { return v_rotate_right(a, b); } + +OPENCV_HAL_IMPL_LSX_EXTRACT(v_uint8x16) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_int8x16) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_uint16x8) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_int16x8) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_uint32x4) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_int32x4) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_uint64x2) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_int64x2) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_float32x4) +OPENCV_HAL_IMPL_LSX_EXTRACT(v_float64x2) + +#define OPENCV_HAL_IMPL_LSX_EXTRACT_N(_Tpvec, _Twvec, intrin) \ +template \ +inline _Twvec v_extract_n(const _Tpvec& a) \ +{ return (_Twvec)intrin(a.val, i); } + +OPENCV_HAL_IMPL_LSX_EXTRACT_N(v_uint8x16, uchar, __lsx_vpickve2gr_b) +OPENCV_HAL_IMPL_LSX_EXTRACT_N(v_int8x16, schar, __lsx_vpickve2gr_b) +OPENCV_HAL_IMPL_LSX_EXTRACT_N(v_uint16x8, ushort, __lsx_vpickve2gr_h) +OPENCV_HAL_IMPL_LSX_EXTRACT_N(v_int16x8, short, __lsx_vpickve2gr_h) +OPENCV_HAL_IMPL_LSX_EXTRACT_N(v_uint32x4, uint, __lsx_vpickve2gr_w) +OPENCV_HAL_IMPL_LSX_EXTRACT_N(v_int32x4, int, __lsx_vpickve2gr_w) +OPENCV_HAL_IMPL_LSX_EXTRACT_N(v_uint64x2, uint64, __lsx_vpickve2gr_d) +OPENCV_HAL_IMPL_LSX_EXTRACT_N(v_int64x2, int64, __lsx_vpickve2gr_d) + +template +inline float v_extract_n(const v_float32x4& v) +{ + union { uint iv; float fv; } d; + d.iv = __lsx_vpickve2gr_w(v.val, i); + return d.fv; +} + +template +inline double v_extract_n(const v_float64x2& v) +{ + union { uint64 iv; double dv; } d; + d.iv = __lsx_vpickve2gr_d(v.val, i); + return d.dv; +} + +template +inline v_uint32x4 v_broadcast_element(const v_uint32x4& a) +{ return v_uint32x4(__lsx_vreplvei_w(a.val, i)); } + +template +inline v_int32x4 v_broadcast_element(const v_int32x4& a) +{ return v_int32x4(__lsx_vreplvei_w(a.val, i)); } + +template +inline v_float32x4 v_broadcast_element(const v_float32x4& a) +{ return v_float32x4((__m128)__lsx_vreplvei_w((__m128i)a.val, i)); } + +/////////////////// load deinterleave ////////////////////////////// + +inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + + a.val = __lsx_vpickev_b(t1, t0); + b.val = __lsx_vpickod_b(t1, t0); +} + +inline void v_load_deinterleave(const ushort* ptr, v_uint16x8& a, v_uint16x8& b) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + a.val = __lsx_vpickev_h(t1, t0); + b.val = __lsx_vpickod_h(t1, t0); +} + +inline void v_load_deinterleave(const unsigned* ptr, v_uint32x4& a, v_uint32x4& b) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + a.val = __lsx_vpickev_w(t1, t0); + b.val = __lsx_vpickod_w(t1, t0); +} + +inline void v_load_deinterleave(const uint64* ptr, v_uint64x2& a, v_uint64x2& b) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + a.val = __lsx_vilvl_d(t1, t0); + b.val = __lsx_vilvh_d(t1, t0); +} + +inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b, v_uint8x16& c) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + __m128i t2 = __lsx_vld(ptr, 32); + const __m128i shuff0 = _v128_setr_b(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0); + const __m128i shuff1 = _v128_setr_b(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); + __m128i a0 = __lsx_vbitsel_v(t0, t1, shuff0); + __m128i b0 = __lsx_vbitsel_v(t1, t0, shuff1); + __m128i c0 = __lsx_vbitsel_v(t1, t0, shuff0); + const __m128i shuff_a = _v128_setr_b(0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 17, 20, 23, 26, 29); + const __m128i shuff_b = _v128_setr_b(1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30); + const __m128i shuff_c = _v128_setr_b(2, 5, 8, 11, 14, 1, 4, 7, 10, 13, 16, 19, 22, 25, 28, 31); + + a.val = __lsx_vshuf_b(t2, a0, shuff_a); + b.val = __lsx_vshuf_b(t2, b0, shuff_b); + c.val = __lsx_vshuf_b(t2, c0, shuff_c); +} + +inline void v_load_deinterleave(const ushort* ptr, v_uint16x8& a, v_uint16x8& b, v_uint16x8& c) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + __m128i t2 = __lsx_vld(ptr, 32); + const __m128i shuff0 = _v128_setr_h(0, 0, -1, 0, 0, -1, 0, 0); + const __m128i shuff1 = _v128_setr_h(0, -1, 0, 0, -1, 0, 0, -1); + + __m128i a0 = __lsx_vbitsel_v(t0, t1, shuff1); + __m128i b0 = __lsx_vbitsel_v(t0, t1, shuff0); + __m128i c0 = __lsx_vbitsel_v(t1, t0, shuff0); + + const __m128i shuff_a = _v128_setr_b(0, 1, 6, 7, 12, 13, 2, 3, 8, 9, 14, 15, 20, 21, 26, 27); + const __m128i shuff_b = _v128_setr_b(2, 3, 8, 9, 14, 15, 4, 5, 10, 11, 16, 17, 22, 23, 28, 29); + const __m128i shuff_c = _v128_setr_b(4, 5, 10, 11, 0, 1, 6, 7, 12, 13, 18, 19, 24, 25, 30, 31); + + a.val = __lsx_vshuf_b(t2, a0, shuff_a); + b.val = __lsx_vshuf_b(t2, b0, shuff_b); + c.val = __lsx_vshuf_b(t2, c0, shuff_c); +} + +inline void v_load_deinterleave(const unsigned* ptr, v_uint32x4& a, v_uint32x4& b, v_uint32x4& c) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + __m128i t2 = __lsx_vld(ptr, 32); + + __m128i a0 = __lsx_vpermi_w(t1, t0, 0xAC); + __m128i b0 = __lsx_vpermi_w(t1, t0, 0xC5); + __m128i c0 = __lsx_vpermi_w(t1, t0, 0x5A); + + a.val = __lsx_vextrins_w(a0, t2, 0x31); + b0 = __lsx_vshuf4i_w(b0, 0x38); + c0 = __lsx_vshuf4i_w(c0, 0x8); + b.val = __lsx_vextrins_w(b0, t2, 0x32); + c.val = __lsx_vpermi_w(t2, c0, 0xC4); +} + +inline void v_load_deinterleave(const uint64* ptr, v_uint64x2& a, v_uint64x2& b, v_uint64x2& c) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + __m128i t2 = __lsx_vld(ptr, 32); + + a.val = __lsx_vshuf4i_d(t0, t1, 0xC); + b.val = __lsx_vshuf4i_d(t0, t2, 0x9); + c.val = __lsx_vshuf4i_d(t1, t2, 0xC); +} + +inline void v_load_deinterleave(const uchar* ptr, v_uint8x16& a, v_uint8x16& b, v_uint8x16& c, v_uint8x16& d) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + __m128i t2 = __lsx_vld(ptr, 32); + __m128i t3 = __lsx_vld(ptr, 48); + + __m128i ac_lo = __lsx_vpickev_b(t1, t0); + __m128i bd_lo = __lsx_vpickod_b(t1, t0); + __m128i ac_hi = __lsx_vpickev_b(t3, t2); + __m128i bd_hi = __lsx_vpickod_b(t3, t2); + + a.val = __lsx_vpickev_b(ac_hi, ac_lo); + c.val = __lsx_vpickod_b(ac_hi, ac_lo); + b.val = __lsx_vpickev_b(bd_hi, bd_lo); + d.val = __lsx_vpickod_b(bd_hi, bd_lo); +} + +inline void v_load_deinterleave(const ushort* ptr, v_uint16x8& a, v_uint16x8& b, v_uint16x8& c, v_uint16x8& d) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + __m128i t2 = __lsx_vld(ptr, 32); + __m128i t3 = __lsx_vld(ptr, 48); + + __m128i ac_lo = __lsx_vpickev_h(t1, t0); + __m128i bd_lo = __lsx_vpickod_h(t1, t0); + __m128i ac_hi = __lsx_vpickev_h(t3, t2); + __m128i bd_hi = __lsx_vpickod_h(t3, t2); + + a.val = __lsx_vpickev_h(ac_hi, ac_lo); + c.val = __lsx_vpickod_h(ac_hi, ac_lo); + b.val = __lsx_vpickev_h(bd_hi, bd_lo); + d.val = __lsx_vpickod_h(bd_hi, bd_lo); +} + +inline void v_load_deinterleave(const unsigned* ptr, v_uint32x4& a, v_uint32x4& b, v_uint32x4& c, v_uint32x4& d) +{ + __m128i p0 = __lsx_vld(ptr, 0); + __m128i p1 = __lsx_vld(ptr, 16); + __m128i p2 = __lsx_vld(ptr, 32); + __m128i p3 = __lsx_vld(ptr, 48); + + __m128i t0 = __lsx_vilvl_w(p1, p0); + __m128i t1 = __lsx_vilvl_w(p3, p2); + __m128i t2 = __lsx_vilvh_w(p1, p0); + __m128i t3 = __lsx_vilvh_w(p3, p2); + a.val = __lsx_vilvl_d(t1, t0); + b.val = __lsx_vilvh_d(t1, t0); + c.val = __lsx_vilvl_d(t3, t2); + d.val = __lsx_vilvh_d(t3, t2); +} + +inline void v_load_deinterleave(const uint64* ptr, v_uint64x2& a, v_uint64x2& b, v_uint64x2& c, v_uint64x2& d) +{ + __m128i t0 = __lsx_vld(ptr, 0); + __m128i t1 = __lsx_vld(ptr, 16); + __m128i t2 = __lsx_vld(ptr, 32); + __m128i t3 = __lsx_vld(ptr, 48); + + a.val = __lsx_vilvl_d(t2, t0); + b.val = __lsx_vilvh_d(t2, t0); + c.val = __lsx_vilvl_d(t3, t1); + d.val = __lsx_vilvh_d(t3, t1); +} + +////////////////////////// store interleave //////////////////////////////// + +inline void v_store_interleave(uchar* ptr, const v_uint8x16& a, const v_uint8x16& b, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i v0 = __lsx_vilvl_b(b.val, a.val); + __m128i v1 = __lsx_vilvh_b(b.val, a.val); + + __lsx_vst(v0, ptr, 0); + __lsx_vst(v1, ptr, 16); +} + +inline void v_store_interleave(ushort* ptr, const v_uint16x8& a, const v_uint16x8& b, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i v0 = __lsx_vilvl_h(b.val, a.val); + __m128i v1 = __lsx_vilvh_h(b.val, a.val); + + __lsx_vst(v0, ptr, 0); + __lsx_vst(v1, ptr, 16); +} + +inline void v_store_interleave(unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i v0 = __lsx_vilvl_w(b.val, a.val); + __m128i v1 = __lsx_vilvh_w(b.val, a.val); + + __lsx_vst(v0, ptr, 0); + __lsx_vst(v1, ptr, 16); +} + +inline void v_store_interleave(uint64* ptr, const v_uint64x2& a, const v_uint64x2& b, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i v0 = __lsx_vilvl_d(b.val, a.val); + __m128i v1 = __lsx_vilvh_d(b.val, a.val); + + __lsx_vst(v0, ptr, 0); + __lsx_vst(v1, ptr, 16); +} + +inline void v_store_interleave(uchar* ptr, const v_uint8x16& a, const v_uint8x16& b, const v_uint8x16& c, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i ab_lo = __lsx_vilvl_b(b.val, a.val); + __m128i ab_hi = __lsx_vilvh_b(b.val, a.val); + __m128i v_c = c.val; + const __m128i shuff0 = _v128_setr_b(0, 1, 16, 2, 3, 17, 4, 5, 18, 6, 7, 19, 8, 9, 20, 10); + const __m128i shuff1 = _v128_setr_b(11, 21, 12, 13, 22, 14, 15, 23, 0, 0, 0, 0, 0, 0, 0, 0); + const __m128i shuff2 = _v128_setr_b(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 24, 18, 19, 25, 20, 21); + const __m128i shuff3 = _v128_setr_b(26, 6, 7, 27, 8, 9, 28, 10, 11, 29, 12, 13, 30, 14, 15, 31); + __m128i abc = __lsx_vpermi_w(v_c, ab_hi, 0xE4); + + __m128i dst0 = __lsx_vshuf_b(v_c, ab_lo, shuff0); + __m128i dst1 = __lsx_vshuf_b(v_c, ab_lo, shuff1); + __m128i dst2 = __lsx_vshuf_b(v_c, ab_hi, shuff3); + dst1 = __lsx_vshuf_b(abc, dst1, shuff2); + + __lsx_vst(dst0, ptr, 0); + __lsx_vst(dst1, ptr, 16); + __lsx_vst(dst2, ptr, 32); +} + +inline void v_store_interleave(ushort* ptr, const v_uint16x8& a, const v_uint16x8& b, const v_uint16x8& c, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i ab_lo = __lsx_vilvl_h(b.val, a.val); + __m128i ab_hi = __lsx_vilvh_h(b.val, a.val); + __m128i v_c = c.val; + const __m128i shuff0 = _v128_setr_b(0, 1, 2, 3, 16, 17, 4, 5, 6, 7, 18, 19, 8, 9, 10, 11); + const __m128i shuff1 = _v128_setr_b(20, 21, 12, 13, 14, 15, 22, 23, 0, 0, 0, 0, 0, 0, 0, 0); + const __m128i shuff2 = _v128_setr_b(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 24, 25, 20, 21); + const __m128i shuff3 = _v128_setr_b(6, 7, 26, 27, 8, 9, 10, 11, 28, 29, 12, 13, 14, 15, 30, 31); + __m128i abc = __lsx_vpermi_w(v_c, ab_hi, 0xE4); + + __m128i dst0 = __lsx_vshuf_b(v_c, ab_lo, shuff0); + __m128i dst1 = __lsx_vshuf_b(v_c, ab_lo, shuff1); + __m128i dst2 = __lsx_vshuf_b(v_c, ab_hi, shuff3); + dst1 = __lsx_vshuf_b(abc, dst1, shuff2); + + __lsx_vst(dst0, ptr, 0); + __lsx_vst(dst1, ptr, 16); + __lsx_vst(dst2, ptr, 32); +} + +inline void v_store_interleave(unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b, const v_uint32x4& c, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i v_c = c.val; + __m128i ab_lo = __lsx_vilvl_w(b.val, a.val); //a0 b0 a1 b1 + __m128i ab_hi = __lsx_vilvh_w(b.val, a.val); //a2 b2 a3 b3 + __m128i bc_od = __lsx_vpackod_w(v_c, b.val); // b1 c1 b3 c3 + + __m128i dst0 = __lsx_vshuf4i_w(ab_lo, 0xB4); //a0 b0 b1 a1 + __m128i dst1 = __lsx_vilvl_d(ab_hi, bc_od); //b1 c1 a2 b2 + __m128i dst2 = __lsx_vpermi_w(bc_od, ab_hi, 0xE8); //a2, a3, b3, c3 + + dst0 = __lsx_vextrins_w(dst0, v_c, 0x20); + dst2 = __lsx_vextrins_w(dst2, v_c, 0x2); + __lsx_vst(dst0, ptr, 0); //a0 b0 c0 a1 + __lsx_vst(dst1, ptr, 16); //b1 c1 a2 b2 + __lsx_vst(dst2, ptr, 32); //c2 a3 b3 c3 +} + +inline void v_store_interleave(uint64* ptr, const v_uint64x2& a, const v_uint64x2& b, const v_uint64x2& c, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i dst0 = __lsx_vilvl_d(b.val, a.val); + __m128i dst1 = __lsx_vpermi_w(a.val, c.val, 0xE4); + __m128i dst2 = __lsx_vilvh_d(c.val, b.val); + + __lsx_vst(dst0, ptr, 0); + __lsx_vst(dst1, ptr, 16); + __lsx_vst(dst2, ptr, 32); +} + +inline void v_store_interleave(uchar* ptr, const v_uint8x16& a, const v_uint8x16& b, + const v_uint8x16& c, const v_uint8x16& d, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i ab_lo = __lsx_vilvl_b(b.val, a.val); + __m128i ab_hi = __lsx_vilvh_b(b.val, a.val); + __m128i cd_lo = __lsx_vilvl_b(d.val, c.val); + __m128i cd_hi = __lsx_vilvh_b(d.val, c.val); + + __m128i dst0 = __lsx_vilvl_h(cd_lo, ab_lo); + __m128i dst1 = __lsx_vilvh_h(cd_lo, ab_lo); + __m128i dst2 = __lsx_vilvl_h(cd_hi, ab_hi); + __m128i dst3 = __lsx_vilvh_h(cd_hi, ab_hi); + + __lsx_vst(dst0, ptr, 0); + __lsx_vst(dst1, ptr, 16); + __lsx_vst(dst2, ptr, 32); + __lsx_vst(dst3, ptr, 48); +} + +inline void v_store_interleave(ushort* ptr, const v_uint16x8& a, const v_uint16x8& b, + const v_uint16x8& c, const v_uint16x8& d, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i ab_lo = __lsx_vilvl_h(b.val, a.val); + __m128i ab_hi = __lsx_vilvh_h(b.val, a.val); + __m128i cd_lo = __lsx_vilvl_h(d.val, c.val); + __m128i cd_hi = __lsx_vilvh_h(d.val, c.val); + + __m128i dst0 = __lsx_vilvl_w(cd_lo, ab_lo); + __m128i dst1 = __lsx_vilvh_w(cd_lo, ab_lo); + __m128i dst2 = __lsx_vilvl_w(cd_hi, ab_hi); + __m128i dst3 = __lsx_vilvh_w(cd_hi, ab_hi); + + __lsx_vst(dst0, ptr, 0); + __lsx_vst(dst1, ptr, 16); + __lsx_vst(dst2, ptr, 32); + __lsx_vst(dst3, ptr, 48); +} + +inline void v_store_interleave(unsigned* ptr, const v_uint32x4& a, const v_uint32x4& b, + const v_uint32x4& c, const v_uint32x4& d, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i ab_lo = __lsx_vilvl_w(b.val, a.val); + __m128i ab_hi = __lsx_vilvh_w(b.val, a.val); + __m128i cd_lo = __lsx_vilvl_w(d.val, c.val); + __m128i cd_hi = __lsx_vilvh_w(d.val, c.val); + + __m128i dst0 = __lsx_vilvl_d(cd_lo, ab_lo); + __m128i dst1 = __lsx_vilvh_d(cd_lo, ab_lo); + __m128i dst2 = __lsx_vilvl_d(cd_hi, ab_hi); + __m128i dst3 = __lsx_vilvh_d(cd_hi, ab_hi); + + __lsx_vst(dst0, ptr, 0); + __lsx_vst(dst1, ptr, 16); + __lsx_vst(dst2, ptr, 32); + __lsx_vst(dst3, ptr, 48); +} + +inline void v_store_interleave(uint64* ptr, const v_uint64x2& a, const v_uint64x2& b, + const v_uint64x2& c, const v_uint64x2& d, + hal::StoreMode /*mode*/ = hal::STORE_UNALIGNED) +{ + __m128i dst0 = __lsx_vilvl_d(b.val, a.val); + __m128i dst2 = __lsx_vilvh_d(b.val, a.val); + __m128i dst1 = __lsx_vilvl_d(d.val, c.val); + __m128i dst3 = __lsx_vilvh_d(d.val, c.val); + + __lsx_vst(dst0, ptr, 0); + __lsx_vst(dst1, ptr, 16); + __lsx_vst(dst2, ptr, 32); + __lsx_vst(dst3, ptr, 48); +} + +#define OPENCV_HAL_IMPL_LSX_LOADSTORE_INTERLEAVE(_Tpvec0, _Tp0, suffix0, _Tpvec1, _Tp1, suffix1) \ +inline void v_load_deinterleave(const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0) \ +{ \ + _Tpvec1 a1, b1; \ + v_load_deinterleave((const _Tp1*)ptr, a1, b1); \ + a0 = v_reinterpret_as_##suffix0(a1); \ + b0 = v_reinterpret_as_##suffix0(b1); \ +} \ +inline void v_load_deinterleave(const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, _Tpvec0& c0) \ +{ \ + _Tpvec1 a1, b1, c1; \ + v_load_deinterleave((const _Tp1*)ptr, a1, b1, c1); \ + a0 = v_reinterpret_as_##suffix0(a1); \ + b0 = v_reinterpret_as_##suffix0(b1); \ + c0 = v_reinterpret_as_##suffix0(c1); \ +} \ +inline void v_load_deinterleave(const _Tp0* ptr, _Tpvec0& a0, _Tpvec0& b0, \ + _Tpvec0& c0, _Tpvec0& d0) \ +{ \ + _Tpvec1 a1, b1, c1, d1; \ + v_load_deinterleave((const _Tp1*)ptr, a1, b1, c1, d1); \ + a0 = v_reinterpret_as_##suffix0(a1); \ + b0 = v_reinterpret_as_##suffix0(b1); \ + c0 = v_reinterpret_as_##suffix0(c1); \ + d0 = v_reinterpret_as_##suffix0(d1); \ +} \ +inline void v_store_interleave(_Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ + hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ +{ \ + _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ + _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ + v_store_interleave((_Tp1*)ptr, a1, b1); \ +} \ +inline void v_store_interleave(_Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, const _Tpvec0& c0,\ + hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ +{ \ + _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ + _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ + _Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \ + v_store_interleave((_Tp1*)ptr, a1, b1, c1); \ +} \ +inline void v_store_interleave(_Tp0* ptr, const _Tpvec0& a0, const _Tpvec0& b0, \ + const _Tpvec0& c0, const _Tpvec0& d0, \ + hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ +{ \ + _Tpvec1 a1 = v_reinterpret_as_##suffix1(a0); \ + _Tpvec1 b1 = v_reinterpret_as_##suffix1(b0); \ + _Tpvec1 c1 = v_reinterpret_as_##suffix1(c0); \ + _Tpvec1 d1 = v_reinterpret_as_##suffix1(d0); \ + v_store_interleave((_Tp1*)ptr, a1, b1, c1, d1); \ +} + +OPENCV_HAL_IMPL_LSX_LOADSTORE_INTERLEAVE(v_int8x16, schar, s8, v_uint8x16, uchar, u8) +OPENCV_HAL_IMPL_LSX_LOADSTORE_INTERLEAVE(v_int16x8, short, s16, v_uint16x8, ushort, u16) +OPENCV_HAL_IMPL_LSX_LOADSTORE_INTERLEAVE(v_int32x4, int, s32, v_uint32x4, unsigned, u32) +OPENCV_HAL_IMPL_LSX_LOADSTORE_INTERLEAVE(v_float32x4, float, f32, v_uint32x4, unsigned, u32) +OPENCV_HAL_IMPL_LSX_LOADSTORE_INTERLEAVE(v_int64x2, int64, s64, v_uint64x2, uint64, u64) +OPENCV_HAL_IMPL_LSX_LOADSTORE_INTERLEAVE(v_float64x2, double, f64, v_uint64x2, uint64, u64) + +// +// FP16 +// + +inline v_float32x4 v_load_expand(const float16_t* ptr) +{ +#if CV_FP16 + return v_float32x4(__lsx_vfcvtl_s_h((__m128)__lsx_vld(ptr, 0))); +#else + float CV_DECL_ALIGNED(32) buf[4]; + for (int i = 0; i < 4; i++) + buf[i] = (float)ptr[i]; + return v_float32x4((__m128)__lsx_vld(buf, 0)); +#endif +} + +inline void v_pack_store(float16_t* ptr, const v_float32x4& a) +{ +#if CV_FP16 + __m128i res = (__m218i)__lsx_vfcvt_h_s(a.val, a.val); + __lsx_vstelm_d(res, ptr, 0, 0); +#else + float CV_DECL_ALIGNED(32) buf[4]; + v_store_aligned(buf, a); + for (int i = 0; i < 4; i++) + ptr[i] = float16_t(buf[i]); +#endif +} + +// +// end of FP16 +// + +inline void v_cleanup() {} + +CV_CPU_OPTIMIZATION_HAL_NAMESPACE_END + +//! @endcond + +} // cv:: + +#endif // OPENCV_HAL_INTRIN_LSX_HPP diff --git a/modules/core/include/opencv2/core/hal/intrin_neon.hpp b/modules/core/include/opencv2/core/hal/intrin_neon.hpp index 2ae64ca8e4..b4ed6dec0f 100644 --- a/modules/core/include/opencv2/core/hal/intrin_neon.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_neon.hpp @@ -131,13 +131,22 @@ OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_I64(int64x2, int64x1, s64) OPENCV_HAL_IMPL_NEON_UTILS_SUFFIX_F64(float64x2, float64x1,f64) #endif +//////////// Compatibility layer //////////// +template struct VTraits { + static inline int vlanes() { return T::nlanes; } + enum { max_nlanes = T::nlanes, nlanes = T::nlanes }; + using lane_type = typename T::lane_type; +}; + +template +inline typename VTraits::lane_type v_get0(const T& v) \ +{ \ + return v.get0(); \ +} //////////// Types //////////// struct v_uint8x16 { - typedef uchar lane_type; - enum { nlanes = 16 }; - v_uint8x16() {} explicit v_uint8x16(uint8x16_t v) : val(v) {} v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7, @@ -146,19 +155,22 @@ struct v_uint8x16 uchar v[] = {v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}; val = vld1q_u8(v); } + uint8x16_t val; + +private: + friend struct VTraits; + enum { nlanes = 16 }; + typedef uchar lane_type; + + friend typename VTraits::lane_type v_get0(const v_uint8x16& v); uchar get0() const { return vgetq_lane_u8(val, 0); } - - uint8x16_t val; }; struct v_int8x16 { - typedef schar lane_type; - enum { nlanes = 16 }; - v_int8x16() {} explicit v_int8x16(int8x16_t v) : val(v) {} v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7, @@ -167,19 +179,22 @@ struct v_int8x16 schar v[] = {v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}; val = vld1q_s8(v); } + int8x16_t val; + +private: + friend struct VTraits; + enum { nlanes = 16 }; + typedef schar lane_type; + + friend typename VTraits::lane_type v_get0(const v_int8x16& v); schar get0() const { return vgetq_lane_s8(val, 0); } - - int8x16_t val; }; struct v_uint16x8 { - typedef ushort lane_type; - enum { nlanes = 8 }; - v_uint16x8() {} explicit v_uint16x8(uint16x8_t v) : val(v) {} v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7) @@ -187,19 +202,22 @@ struct v_uint16x8 ushort v[] = {v0, v1, v2, v3, v4, v5, v6, v7}; val = vld1q_u16(v); } + uint16x8_t val; + +private: + friend struct VTraits; + enum { nlanes = 8 }; + typedef ushort lane_type; + + friend typename VTraits::lane_type v_get0(const v_uint16x8& v); ushort get0() const { return vgetq_lane_u16(val, 0); } - - uint16x8_t val; }; struct v_int16x8 { - typedef short lane_type; - enum { nlanes = 8 }; - v_int16x8() {} explicit v_int16x8(int16x8_t v) : val(v) {} v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7) @@ -207,19 +225,22 @@ struct v_int16x8 short v[] = {v0, v1, v2, v3, v4, v5, v6, v7}; val = vld1q_s16(v); } + int16x8_t val; + +private: + friend struct VTraits; + enum { nlanes = 8 }; + typedef short lane_type; + + friend typename VTraits::lane_type v_get0(const v_int16x8& v); short get0() const { return vgetq_lane_s16(val, 0); } - - int16x8_t val; }; struct v_uint32x4 { - typedef unsigned lane_type; - enum { nlanes = 4 }; - v_uint32x4() {} explicit v_uint32x4(uint32x4_t v) : val(v) {} v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3) @@ -227,19 +248,22 @@ struct v_uint32x4 unsigned v[] = {v0, v1, v2, v3}; val = vld1q_u32(v); } + uint32x4_t val; + +private: + friend struct VTraits; + enum { nlanes = 4 }; + typedef unsigned lane_type; + + friend typename VTraits::lane_type v_get0(const v_uint32x4& v); unsigned get0() const { return vgetq_lane_u32(val, 0); } - - uint32x4_t val; }; struct v_int32x4 { - typedef int lane_type; - enum { nlanes = 4 }; - v_int32x4() {} explicit v_int32x4(int32x4_t v) : val(v) {} v_int32x4(int v0, int v1, int v2, int v3) @@ -247,18 +271,22 @@ struct v_int32x4 int v[] = {v0, v1, v2, v3}; val = vld1q_s32(v); } + int32x4_t val; + +private: + friend struct VTraits; + enum { nlanes = 4 }; + typedef int lane_type; + + friend typename VTraits::lane_type v_get0(const v_int32x4& v); int get0() const { return vgetq_lane_s32(val, 0); } - int32x4_t val; }; struct v_float32x4 { - typedef float lane_type; - enum { nlanes = 4 }; - v_float32x4() {} explicit v_float32x4(float32x4_t v) : val(v) {} v_float32x4(float v0, float v1, float v2, float v3) @@ -266,18 +294,22 @@ struct v_float32x4 float v[] = {v0, v1, v2, v3}; val = vld1q_f32(v); } + float32x4_t val; + +private: + friend struct VTraits; + enum { nlanes = 4 }; + typedef float lane_type; + + friend typename VTraits::lane_type v_get0(const v_float32x4& v); float get0() const { return vgetq_lane_f32(val, 0); } - float32x4_t val; }; struct v_uint64x2 { - typedef uint64 lane_type; - enum { nlanes = 2 }; - v_uint64x2() {} explicit v_uint64x2(uint64x2_t v) : val(v) {} v_uint64x2(uint64 v0, uint64 v1) @@ -285,18 +317,21 @@ struct v_uint64x2 uint64 v[] = {v0, v1}; val = vld1q_u64(v); } + uint64x2_t val; +private: + friend struct VTraits; + enum { nlanes = 2 }; + typedef uint64 lane_type; + + friend typename VTraits::lane_type v_get0(const v_uint64x2& v); uint64 get0() const { return vgetq_lane_u64(val, 0); } - uint64x2_t val; }; struct v_int64x2 { - typedef int64 lane_type; - enum { nlanes = 2 }; - v_int64x2() {} explicit v_int64x2(int64x2_t v) : val(v) {} v_int64x2(int64 v0, int64 v1) @@ -304,19 +339,23 @@ struct v_int64x2 int64 v[] = {v0, v1}; val = vld1q_s64(v); } + int64x2_t val; + +private: + friend struct VTraits; + enum { nlanes = 2 }; + typedef int64 lane_type; + + friend typename VTraits::lane_type v_get0(const v_int64x2& v); int64 get0() const { return vgetq_lane_s64(val, 0); } - int64x2_t val; }; #if CV_SIMD128_64F struct v_float64x2 { - typedef double lane_type; - enum { nlanes = 2 }; - v_float64x2() {} explicit v_float64x2(float64x2_t v) : val(v) {} v_float64x2(double v0, double v1) @@ -324,11 +363,18 @@ struct v_float64x2 double v[] = {v0, v1}; val = vld1q_f64(v); } + + float64x2_t val; +private: + friend struct VTraits; + enum { nlanes = 2 }; + typedef double lane_type; + + friend typename VTraits::lane_type v_get0(const v_float64x2& v); double get0() const { return vgetq_lane_f64(val, 0); } - float64x2_t val; }; #endif @@ -460,71 +506,56 @@ inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, } #define OPENCV_HAL_IMPL_NEON_BIN_OP(bin_op, _Tpvec, intrin) \ -inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ +inline _Tpvec bin_op (const _Tpvec& a, const _Tpvec& b) \ { \ return _Tpvec(intrin(a.val, b.val)); \ -} \ -inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ -{ \ - a.val = intrin(a.val, b.val); \ - return a; \ } -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint8x16, vqaddq_u8) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint8x16, vqsubq_u8) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int8x16, vqaddq_s8) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int8x16, vqsubq_s8) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint16x8, vqaddq_u16) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint16x8, vqsubq_u16) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int16x8, vqaddq_s16) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int16x8, vqsubq_s16) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int32x4, vaddq_s32) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int32x4, vsubq_s32) -OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_int32x4, vmulq_s32) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint32x4, vaddq_u32) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint32x4, vsubq_u32) -OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_uint32x4, vmulq_u32) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float32x4, vaddq_f32) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float32x4, vsubq_f32) -OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float32x4, vmulq_f32) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int64x2, vaddq_s64) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int64x2, vsubq_s64) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint64x2, vaddq_u64) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint64x2, vsubq_u64) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint8x16, vqaddq_u8) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint8x16, vqsubq_u8) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int8x16, vqaddq_s8) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int8x16, vqsubq_s8) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint16x8, vqaddq_u16) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint16x8, vqsubq_u16) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int16x8, vqaddq_s16) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int16x8, vqsubq_s16) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int32x4, vaddq_s32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int32x4, vsubq_s32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_int32x4, vmulq_s32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint32x4, vaddq_u32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint32x4, vsubq_u32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_uint32x4, vmulq_u32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_float32x4, vaddq_f32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_float32x4, vsubq_f32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_float32x4, vmulq_f32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_int64x2, vaddq_s64) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_int64x2, vsubq_s64) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_uint64x2, vaddq_u64) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_uint64x2, vsubq_u64) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float32x4, vdivq_f32) -OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float64x2, vaddq_f64) -OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float64x2, vsubq_f64) -OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float64x2, vmulq_f64) -OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float64x2, vdivq_f64) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_div, v_float32x4, vdivq_f32) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_add, v_float64x2, vaddq_f64) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_sub, v_float64x2, vsubq_f64) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_mul, v_float64x2, vmulq_f64) +OPENCV_HAL_IMPL_NEON_BIN_OP(v_div, v_float64x2, vdivq_f64) #else -inline v_float32x4 operator / (const v_float32x4& a, const v_float32x4& b) +inline v_float32x4 v_div (const v_float32x4& a, const v_float32x4& b) { float32x4_t reciprocal = vrecpeq_f32(b.val); reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal); reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal); return v_float32x4(vmulq_f32(a.val, reciprocal)); } -inline v_float32x4& operator /= (v_float32x4& a, const v_float32x4& b) -{ - float32x4_t reciprocal = vrecpeq_f32(b.val); - reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal); - reciprocal = vmulq_f32(vrecpsq_f32(b.val, reciprocal), reciprocal); - a.val = vmulq_f32(a.val, reciprocal); - return a; -} #endif // saturating multiply 8-bit, 16-bit #define OPENCV_HAL_IMPL_NEON_MUL_SAT(_Tpvec, _Tpwvec) \ - inline _Tpvec operator * (const _Tpvec& a, const _Tpvec& b) \ + inline _Tpvec v_mul (const _Tpvec& a, const _Tpvec& b) \ { \ _Tpwvec c, d; \ v_mul_expand(a, b, c, d); \ return v_pack(c, d); \ - } \ - inline _Tpvec& operator *= (_Tpvec& a, const _Tpvec& b) \ - { a = a * b; return a; } + } OPENCV_HAL_IMPL_NEON_MUL_SAT(v_int8x16, v_int16x8) OPENCV_HAL_IMPL_NEON_MUL_SAT(v_uint8x16, v_uint16x8) @@ -698,7 +729,7 @@ inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { - return v_dotprod_expand(a, b) + c; + return v_add(v_dotprod_expand(a, b), c); } inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) @@ -715,7 +746,7 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { - return v_dotprod_expand(a, b) + c; + return v_add(v_dotprod_expand(a, b), c); } #endif // 16 >> 64 @@ -735,7 +766,7 @@ inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) return v_uint64x2(vaddq_u64(s0, s1)); } inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) -{ return v_dotprod_expand(a, b) + c; } +{ return v_add(v_dotprod_expand(a, b), c); } inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) { @@ -752,7 +783,7 @@ inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) } inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) -{ return v_dotprod_expand(a, b) + c; } +{ return v_add(v_dotprod_expand(a, b), c); } // 32 >> 64f #if CV_SIMD128_64F @@ -760,7 +791,7 @@ inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b) { return v_cvt_f64(v_dotprod(a, b)); } inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b, const v_float64x2& c) -{ return v_dotprod_expand(a, b) + c; } +{ return v_add(v_dotprod_expand(a, b), c); } #endif //////// Fast Dot Product //////// @@ -850,7 +881,7 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b } inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { - return v_dotprod_expand_fast(a, b) + c; + return v_add(v_dotprod_expand_fast(a, b), c); } inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) @@ -861,7 +892,7 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) } inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { - return v_dotprod_expand_fast(a, b) + c; + return v_add(v_dotprod_expand_fast(a, b), c); } #endif @@ -875,7 +906,7 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b return v_uint64x2(vaddq_u64(s0, s1)); } inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) -{ return v_dotprod_expand_fast(a, b) + c; } +{ return v_add(v_dotprod_expand_fast(a, b), c); } inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) { @@ -884,22 +915,22 @@ inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) return v_int64x2(vaddl_s32(vget_low_s32(prod), vget_high_s32(prod))); } inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) -{ return v_dotprod_expand_fast(a, b) + c; } +{ return v_add(v_dotprod_expand_fast(a, b), c); } // 32 >> 64f #if CV_SIMD128_64F inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b) { return v_cvt_f64(v_dotprod_fast(a, b)); } inline v_float64x2 v_dotprod_expand_fast(const v_int32x4& a, const v_int32x4& b, const v_float64x2& c) -{ return v_dotprod_expand_fast(a, b) + c; } +{ return v_add(v_dotprod_expand_fast(a, b), c); } #endif #define OPENCV_HAL_IMPL_NEON_LOGIC_OP(_Tpvec, suffix) \ - OPENCV_HAL_IMPL_NEON_BIN_OP(&, _Tpvec, vandq_##suffix) \ - OPENCV_HAL_IMPL_NEON_BIN_OP(|, _Tpvec, vorrq_##suffix) \ - OPENCV_HAL_IMPL_NEON_BIN_OP(^, _Tpvec, veorq_##suffix) \ - inline _Tpvec operator ~ (const _Tpvec& a) \ + OPENCV_HAL_IMPL_NEON_BIN_OP(v_and, _Tpvec, vandq_##suffix) \ + OPENCV_HAL_IMPL_NEON_BIN_OP(v_or, _Tpvec, vorrq_##suffix) \ + OPENCV_HAL_IMPL_NEON_BIN_OP(v_xor, _Tpvec, veorq_##suffix) \ + inline _Tpvec v_not (const _Tpvec& a) \ { \ return _Tpvec(vreinterpretq_##suffix##_u8(vmvnq_u8(vreinterpretq_u8_##suffix(a.val)))); \ } @@ -914,21 +945,16 @@ OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_uint64x2, u64) OPENCV_HAL_IMPL_NEON_LOGIC_OP(v_int64x2, s64) #define OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(bin_op, intrin) \ -inline v_float32x4 operator bin_op (const v_float32x4& a, const v_float32x4& b) \ +inline v_float32x4 bin_op (const v_float32x4& a, const v_float32x4& b) \ { \ return v_float32x4(vreinterpretq_f32_s32(intrin(vreinterpretq_s32_f32(a.val), vreinterpretq_s32_f32(b.val)))); \ -} \ -inline v_float32x4& operator bin_op##= (v_float32x4& a, const v_float32x4& b) \ -{ \ - a.val = vreinterpretq_f32_s32(intrin(vreinterpretq_s32_f32(a.val), vreinterpretq_s32_f32(b.val))); \ - return a; \ } -OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(&, vandq_s32) -OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(|, vorrq_s32) -OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(^, veorq_s32) +OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_and, vandq_s32) +OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_or, vorrq_s32) +OPENCV_HAL_IMPL_NEON_FLT_BIT_OP(v_xor, veorq_s32) -inline v_float32x4 operator ~ (const v_float32x4& a) +inline v_float32x4 v_not (const v_float32x4& a) { return v_float32x4(vreinterpretq_f32_s32(vmvnq_s32(vreinterpretq_s32_f32(a.val)))); } @@ -942,7 +968,7 @@ inline v_float32x4 v_sqrt(const v_float32x4& x) inline v_float32x4 v_invsqrt(const v_float32x4& x) { v_float32x4 one = v_setall_f32(1.0f); - return one / v_sqrt(x); + return v_div(one, v_sqrt(x)); } #else inline v_float32x4 v_sqrt(const v_float32x4& x) @@ -975,21 +1001,16 @@ inline v_float32x4 v_abs(v_float32x4 x) #if CV_SIMD128_64F #define OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(bin_op, intrin) \ -inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \ +inline v_float64x2 bin_op (const v_float64x2& a, const v_float64x2& b) \ { \ return v_float64x2(vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val)))); \ -} \ -inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \ -{ \ - a.val = vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val))); \ - return a; \ } -OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(&, vandq_s64) -OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(|, vorrq_s64) -OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(^, veorq_s64) +OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_and, vandq_s64) +OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_or, vorrq_s64) +OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(v_xor, veorq_s64) -inline v_float64x2 operator ~ (const v_float64x2& a) +inline v_float64x2 v_not (const v_float64x2& a) { return v_float64x2(vreinterpretq_f64_s32(vmvnq_s32(vreinterpretq_s32_f64(a.val)))); } @@ -1002,7 +1023,7 @@ inline v_float64x2 v_sqrt(const v_float64x2& x) inline v_float64x2 v_invsqrt(const v_float64x2& x) { v_float64x2 one = v_setall_f64(1.0f); - return one / v_sqrt(x); + return v_div(one, v_sqrt(x)); } inline v_float64x2 v_abs(v_float64x2 x) @@ -1037,17 +1058,17 @@ OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_max, vmaxq_f64) #endif #define OPENCV_HAL_IMPL_NEON_INT_CMP_OP(_Tpvec, cast, suffix, not_suffix) \ -inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \ +inline _Tpvec v_eq (const _Tpvec& a, const _Tpvec& b) \ { return _Tpvec(cast(vceqq_##suffix(a.val, b.val))); } \ -inline _Tpvec operator != (const _Tpvec& a, const _Tpvec& b) \ +inline _Tpvec v_ne (const _Tpvec& a, const _Tpvec& b) \ { return _Tpvec(cast(vmvnq_##not_suffix(vceqq_##suffix(a.val, b.val)))); } \ -inline _Tpvec operator < (const _Tpvec& a, const _Tpvec& b) \ +inline _Tpvec v_lt (const _Tpvec& a, const _Tpvec& b) \ { return _Tpvec(cast(vcltq_##suffix(a.val, b.val))); } \ -inline _Tpvec operator > (const _Tpvec& a, const _Tpvec& b) \ +inline _Tpvec v_gt (const _Tpvec& a, const _Tpvec& b) \ { return _Tpvec(cast(vcgtq_##suffix(a.val, b.val))); } \ -inline _Tpvec operator <= (const _Tpvec& a, const _Tpvec& b) \ +inline _Tpvec v_le (const _Tpvec& a, const _Tpvec& b) \ { return _Tpvec(cast(vcleq_##suffix(a.val, b.val))); } \ -inline _Tpvec operator >= (const _Tpvec& a, const _Tpvec& b) \ +inline _Tpvec v_ge (const _Tpvec& a, const _Tpvec& b) \ { return _Tpvec(cast(vcgeq_##suffix(a.val, b.val))); } OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint8x16, OPENCV_HAL_NOP, u8, u8) @@ -1067,48 +1088,48 @@ static inline uint64x2_t vmvnq_u64(uint64x2_t a) OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint64x2, OPENCV_HAL_NOP, u64, u64) OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int64x2, vreinterpretq_s64_u64, s64, u64) #else -static inline v_uint64x2 operator == (const v_uint64x2& a, const v_uint64x2& b) +static inline v_uint64x2 v_eq(const v_uint64x2& a, const v_uint64x2& b) { uint32x4_t cmp = vceqq_u32(vreinterpretq_u32_u64(a.val), vreinterpretq_u32_u64(b.val)); uint32x4_t v_eq = vandq_u32(cmp, vrev64q_u32(cmp)); return v_uint64x2(vreinterpretq_u64_u32(v_eq)); } -static inline v_uint64x2 operator != (const v_uint64x2& a, const v_uint64x2& b) +static inline v_uint64x2 v_ne(const v_uint64x2& a, const v_uint64x2& b) { uint64x2_t v_mask = vorrq_u64(vsubq_u64(a.val, b.val), vsubq_u64(b.val, a.val)); int64x2_t v_smask = vshrq_n_s64(vreinterpretq_s64_u64(v_mask), 63); return v_uint64x2(vreinterpretq_u64_s64(v_smask)); } -static inline v_int64x2 operator == (const v_int64x2& a, const v_int64x2& b) +static inline v_int64x2 v_eq(const v_int64x2& a, const v_int64x2& b) { uint32x4_t cmp = vceqq_u32(vreinterpretq_u32_s64(a.val), vreinterpretq_u32_s64(b.val)); uint32x4_t v_eq = vandq_u32(cmp, vrev64q_u32(cmp)); return v_int64x2(vreinterpretq_s64_u32(v_eq)); } -static inline v_int64x2 operator != (const v_int64x2& a, const v_int64x2& b) +static inline v_int64x2 v_ne(const v_int64x2& a, const v_int64x2& b) { int64x2_t v_mask = vorrq_s64(vsubq_s64(a.val, b.val), vsubq_s64(b.val, a.val)); int64x2_t v_smask = vshrq_n_s64(v_mask, 63); return v_int64x2(v_smask); } -static inline v_uint64x2 operator > (const v_uint64x2& a, const v_uint64x2& b) +static inline v_uint64x2 v_gt(const v_uint64x2& a, const v_uint64x2& b) { int64x2_t v_mask = vreinterpretq_s64_u64(vsubq_u64(b.val, a.val)); return v_uint64x2(vreinterpretq_u64_s64(vshrq_n_s64(v_mask, 63))); } -static inline v_uint64x2 operator < (const v_uint64x2& a, const v_uint64x2& b) +static inline v_uint64x2 v_lt(const v_uint64x2& a, const v_uint64x2& b) { int64x2_t v_mask = vreinterpretq_s64_u64(vsubq_u64(a.val, b.val)); return v_uint64x2(vreinterpretq_u64_s64(vshrq_n_s64(v_mask, 63))); } -static inline v_int64x2 operator > (const v_int64x2& a, const v_int64x2& b) +static inline v_int64x2 v_gt(const v_int64x2& a, const v_int64x2& b) { int64x2_t v_mask = vsubq_s64(b.val, a.val); return v_int64x2(vshrq_n_s64(v_mask, 63)); } -static inline v_int64x2 operator < (const v_int64x2& a, const v_int64x2& b) +static inline v_int64x2 v_lt (const v_int64x2& a, const v_int64x2& b) { int64x2_t v_mask = vsubq_s64(a.val, b.val); return v_int64x2(vshrq_n_s64(v_mask, 63)); @@ -1224,9 +1245,9 @@ inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_ // trade efficiency for convenience #define OPENCV_HAL_IMPL_NEON_SHIFT_OP(_Tpvec, suffix, _Tps, ssuffix) \ -inline _Tpvec operator << (const _Tpvec& a, int n) \ +inline _Tpvec v_shl (const _Tpvec& a, int n) \ { return _Tpvec(vshlq_##suffix(a.val, vdupq_n_##ssuffix((_Tps)n))); } \ -inline _Tpvec operator >> (const _Tpvec& a, int n) \ +inline _Tpvec v_shr (const _Tpvec& a, int n) \ { return _Tpvec(vshlq_##suffix(a.val, vdupq_n_##ssuffix((_Tps)-n))); } \ template inline _Tpvec v_shl(const _Tpvec& a) \ { return _Tpvec(vshlq_n_##suffix(a.val, n)); } \ @@ -1248,13 +1269,13 @@ OPENCV_HAL_IMPL_NEON_SHIFT_OP(v_int64x2, s64, int64, s64) template inline _Tpvec v_rotate_right(const _Tpvec& a) \ { return _Tpvec(vextq_##suffix(a.val, vdupq_n_##suffix(0), n)); } \ template inline _Tpvec v_rotate_left(const _Tpvec& a) \ -{ return _Tpvec(vextq_##suffix(vdupq_n_##suffix(0), a.val, _Tpvec::nlanes - n)); } \ +{ return _Tpvec(vextq_##suffix(vdupq_n_##suffix(0), a.val, VTraits<_Tpvec>::nlanes - n)); } \ template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \ { return a; } \ template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ { return _Tpvec(vextq_##suffix(a.val, b.val, n)); } \ template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ -{ return _Tpvec(vextq_##suffix(b.val, a.val, _Tpvec::nlanes - n)); } \ +{ return _Tpvec(vextq_##suffix(b.val, a.val, VTraits<_Tpvec>::nlanes - n)); } \ template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \ { CV_UNUSED(b); return a; } diff --git a/modules/core/misc/objc/common/Range.h b/modules/core/misc/objc/common/Range.h index fae138c6f9..12bd48f547 100644 --- a/modules/core/misc/objc/common/Range.h +++ b/modules/core/misc/objc/common/Range.h @@ -21,14 +21,13 @@ NS_ASSUME_NONNULL_BEGIN */ CV_EXPORTS @interface Range : NSObject -#ifdef __cplusplus -@property(readonly) cv::Range& nativeRef; -#endif - #pragma mark - Properties @property int start; @property int end; +#ifdef __cplusplus +@property(readonly) cv::Range& nativeRef; +#endif #pragma mark - Constructors @@ -36,6 +35,10 @@ CV_EXPORTS @interface Range : NSObject - (instancetype)initWithStart:(int)start end:(int)end; - (instancetype)initWithVals:(NSArray*)vals; +#ifdef __cplusplus ++ (instancetype)fromNative:(cv::Range&)range; +#endif + #pragma mark - Methods /** @@ -94,10 +97,6 @@ CV_EXPORTS @interface Range : NSObject */ - (NSString*)description; -#ifdef __cplusplus -+ (instancetype)fromNative:(cv::Range&)range; -#endif - @end NS_ASSUME_NONNULL_END diff --git a/modules/core/misc/objc/common/Range.m b/modules/core/misc/objc/common/Range.m deleted file mode 100644 index a4e155214e..0000000000 --- a/modules/core/misc/objc/common/Range.m +++ /dev/null @@ -1,86 +0,0 @@ -// -// Range.m -// -// Created by Giles Payne on 2019/10/08. -// - -#import "Range.h" - -@implementation Range - -- (instancetype)init { - return [self initWithStart:0 end: 0]; -} - -- (instancetype)initWithStart:(int)start end:(int)end { - self = [super init]; - if (self != nil) { - self.start = start; - self.end = end; - } - return self; -} - -- (instancetype)initWithVals:(NSArray*)vals { - self = [self init]; - if (self != nil) { - [self set:vals]; - } - return self; -} - -- (void)set:(NSArray*)vals { - self.start = (vals != nil && vals.count > 0) ? vals[0].intValue : 0; - self.end = (vals != nil && vals.count > 1 ) ? vals[1].intValue : 0; -} - -- (int)size { - return [self empty] ? 0 : self.end - self.start; -} - -- (BOOL)empty { - return self.end <= self.start; -} - -+ (Range*)all { - return [[Range alloc] initWithStart:INT_MIN end:INT_MAX]; -} - -- (Range*)intersection:(Range*)r1 { - Range* out = [[Range alloc] initWithStart:MAX(r1.start, self.start) end:MIN(r1.end, self.end)]; - out.end = MAX(out.end, out.start); - return out; -} - -- (Range*)shift:(int)delta { - return [[Range alloc] initWithStart:self.start + delta end:self.end + delta]; -} - -- (Range*)clone { - return [[Range alloc] initWithStart:self.start end:self.end]; -} - -- (BOOL)isEqual:(id)other { - if (other == self) { - return YES; - } else if (![other isKindOfClass:[Range class]]) { - return NO; - } else { - Range* it = (Range*)other; - return self.start == it.start && self.end == it.end; - } -} - -- (NSUInteger)hash { - int prime = 31; - uint32_t result = 1; - result = prime * result + self.start; - result = prime * result + self.end; - return result; -} - -- (NSString *)description { - return [NSString stringWithFormat:@"Range {%d, %d}", self.start, self.end]; -} - -@end diff --git a/modules/core/misc/objc/common/Range.mm b/modules/core/misc/objc/common/Range.mm index e12a8186ca..55ba49e3ad 100644 --- a/modules/core/misc/objc/common/Range.mm +++ b/modules/core/misc/objc/common/Range.mm @@ -1,5 +1,5 @@ // -// Range.m +// Range.mm // // Created by Giles Payne on 2019/10/08. // @@ -10,6 +10,22 @@ cv::Range native; } +- (int)start { + return native.start; +} + +- (void)setStart:(int)val { + native.start = val; +} + +- (int)end { + return native.end; +} + +- (void)setEnd:(int)val { + native.end = val; +} + - (cv::Range&)nativeRef { return native; } @@ -35,6 +51,10 @@ return self; } ++ (instancetype)fromNative:(cv::Range&)range { + return [[Range alloc] initWithStart:range.start end:range.end]; +} + - (void)set:(NSArray*)vals { self.start = (vals != nil && vals.count > 0) ? vals[0].intValue : 0; self.end = (vals != nil && vals.count > 1 ) ? vals[1].intValue : 0; @@ -85,10 +105,6 @@ return result; } -+ (instancetype)fromNative:(cv::Range&)range { - return [[Range alloc] initWithStart:range.start end:range.end]; -} - - (NSString *)description { return [NSString stringWithFormat:@"Range {%d, %d}", self.start, self.end]; } diff --git a/modules/core/misc/objc/gen_dict.json b/modules/core/misc/objc/gen_dict.json index e3f28bf529..e01b32d6dc 100644 --- a/modules/core/misc/objc/gen_dict.json +++ b/modules/core/misc/objc/gen_dict.json @@ -169,7 +169,7 @@ "Range": { "objc_type": "Range*", "to_cpp": "%(n)s.nativeRef", - "from_cpp": "[Point3i fromNative:%(n)s]" + "from_cpp": "[Range fromNative:%(n)s]" }, "Rect": { "objc_type": "Rect2i*", diff --git a/modules/core/src/convert.hpp b/modules/core/src/convert.hpp index 7cc6ad5f0c..ca14cd86c0 100644 --- a/modules/core/src/convert.hpp +++ b/modules/core/src/convert.hpp @@ -684,8 +684,8 @@ static inline void v_store_pair_as(unsigned* ptr, const v_float64& a, const v_fl static inline void vx_load_as(const double* ptr, v_float32& a) { - const int VECSZ = v_float32::nlanes; - float buf[VECSZ*2]; + const int VECSZ = VTraits::vlanes(); + float buf[VTraits::max_nlanes*2]; for( int i = 0; i < VECSZ; i++ ) buf[i] = saturate_cast(ptr[i]); @@ -694,8 +694,8 @@ static inline void vx_load_as(const double* ptr, v_float32& a) static inline void vx_load_as(const uint64_t* ptr, v_float32& a) { - const int VECSZ = v_float32::nlanes; - float buf[VECSZ*2]; + const int VECSZ = VTraits::vlanes(); + float buf[VTraits::max_nlanes*2]; for( int i = 0; i < VECSZ; i++ ) buf[i] = saturate_cast(ptr[i]); @@ -704,8 +704,8 @@ static inline void vx_load_as(const uint64_t* ptr, v_float32& a) static inline void vx_load_as(const int64_t* ptr, v_float32& a) { - const int VECSZ = v_float32::nlanes; - float buf[VECSZ*2]; + const int VECSZ = VTraits::vlanes(); + float buf[VTraits::max_nlanes*2]; for( int i = 0; i < VECSZ; i++ ) buf[i] = saturate_cast(ptr[i]); @@ -715,19 +715,19 @@ static inline void vx_load_as(const int64_t* ptr, v_float32& a) template static inline void vx_load_pair_as(const double* ptr, _Tdvec& a, _Tdvec& b) { - const int VECSZ = _Tdvec::nlanes; - typename _Tdvec::lane_type buf[VECSZ*2]; + const int VECSZ = VTraits<_Tdvec>::vlanes(); + typename VTraits<_Tdvec>::lane_type buf[VTraits<_Tdvec>::max_nlanes*2]; for( int i = 0; i < VECSZ*2; i++ ) - buf[i] = saturate_cast(ptr[i]); + buf[i] = saturate_cast::lane_type>(ptr[i]); a = vx_load(buf); b = vx_load(buf + VECSZ); } static inline void v_store_as(double* ptr, const v_float32& a) { - const int VECSZ = v_float32::nlanes; - float buf[VECSZ]; + const int VECSZ = VTraits::vlanes(); + float buf[VTraits::max_nlanes]; v_store(buf, a); for( int i = 0; i < VECSZ; i++ ) @@ -737,8 +737,8 @@ static inline void v_store_as(double* ptr, const v_float32& a) template static inline void v_store_pair_as(double* ptr, const _Tsvec& a, const _Tsvec& b) { - const int VECSZ = _Tsvec::nlanes; - typename _Tsvec::lane_type buf[VECSZ*2]; + const int VECSZ = VTraits<_Tsvec>::vlanes(); + typename VTraits<_Tsvec>::lane_type buf[VTraits<_Tsvec>::max_nlanes*2]; v_store(buf, a); v_store(buf + VECSZ, b); for( int i = 0; i < VECSZ*2; i++ ) diff --git a/modules/core/src/mathfuncs_core.simd.hpp b/modules/core/src/mathfuncs_core.simd.hpp index 1bf36bb174..2aa107b9be 100644 --- a/modules/core/src/mathfuncs_core.simd.hpp +++ b/modules/core/src/mathfuncs_core.simd.hpp @@ -93,13 +93,13 @@ struct v_atan_f32 { v_float32 ax = v_abs(x); v_float32 ay = v_abs(y); - v_float32 c = v_min(ax, ay) / (v_max(ax, ay) + eps); - v_float32 cc = c * c; - v_float32 a = v_fma(v_fma(v_fma(cc, p7, p5), cc, p3), cc, p1)*c; - a = v_select(ax >= ay, a, val90 - a); - a = v_select(x < z, val180 - a, a); - a = v_select(y < z, val360 - a, a); - return a * s; + v_float32 c = v_div(v_min(ax, ay), v_add(v_max(ax, ay), this->eps)); + v_float32 cc = v_mul(c, c); + v_float32 a = v_mul(v_fma(v_fma(v_fma(cc, this->p7, this->p5), cc, this->p3), cc, this->p1), c); + a = v_select(v_ge(ax, ay), a, v_sub(this->val90, a)); + a = v_select(v_lt(x, this->z), v_sub(this->val180, a), a); + a = v_select(v_lt(y, this->z), v_sub(this->val360, a), a); + return v_mul(a, this->s); } v_float32 eps; @@ -125,7 +125,7 @@ static void fastAtan32f_(const float *Y, const float *X, float *angle, int len, float scale = angleInDegrees ? 1.f : (float)(CV_PI/180); int i = 0; #if CV_SIMD - const int VECSZ = v_float32::nlanes; + const int VECSZ = VTraits::vlanes(); v_atan_f32 v(scale); for( ; i < len; i += VECSZ*2 ) @@ -198,7 +198,7 @@ void magnitude32f(const float* x, const float* y, float* mag, int len) int i = 0; #if CV_SIMD - const int VECSZ = v_float32::nlanes; + const int VECSZ = VTraits::vlanes(); for( ; i < len; i += VECSZ*2 ) { if( i + VECSZ*2 > len ) @@ -209,8 +209,8 @@ void magnitude32f(const float* x, const float* y, float* mag, int len) } v_float32 x0 = vx_load(x + i), x1 = vx_load(x + i + VECSZ); v_float32 y0 = vx_load(y + i), y1 = vx_load(y + i + VECSZ); - x0 = v_sqrt(v_muladd(x0, x0, y0*y0)); - x1 = v_sqrt(v_muladd(x1, x1, y1*y1)); + x0 = v_sqrt(v_muladd(x0, x0, v_mul(y0, y0))); + x1 = v_sqrt(v_muladd(x1, x1, v_mul(y1, y1))); v_store(mag + i, x0); v_store(mag + i + VECSZ, x1); } @@ -231,7 +231,7 @@ void magnitude64f(const double* x, const double* y, double* mag, int len) int i = 0; #if CV_SIMD_64F - const int VECSZ = v_float64::nlanes; + const int VECSZ = VTraits::vlanes(); for( ; i < len; i += VECSZ*2 ) { if( i + VECSZ*2 > len ) @@ -242,8 +242,8 @@ void magnitude64f(const double* x, const double* y, double* mag, int len) } v_float64 x0 = vx_load(x + i), x1 = vx_load(x + i + VECSZ); v_float64 y0 = vx_load(y + i), y1 = vx_load(y + i + VECSZ); - x0 = v_sqrt(v_muladd(x0, x0, y0*y0)); - x1 = v_sqrt(v_muladd(x1, x1, y1*y1)); + x0 = v_sqrt(v_muladd(x0, x0, v_mul(y0, y0))); + x1 = v_sqrt(v_muladd(x1, x1, v_mul(y1, y1))); v_store(mag + i, x0); v_store(mag + i + VECSZ, x1); } @@ -265,7 +265,7 @@ void invSqrt32f(const float* src, float* dst, int len) int i = 0; #if CV_SIMD - const int VECSZ = v_float32::nlanes; + const int VECSZ = VTraits::vlanes(); for( ; i < len; i += VECSZ*2 ) { if( i + VECSZ*2 > len ) @@ -293,7 +293,7 @@ void invSqrt64f(const double* src, double* dst, int len) int i = 0; #if CV_SIMD_64F - const int VECSZ = v_float64::nlanes; + const int VECSZ = VTraits::vlanes(); for ( ; i < len; i += VECSZ*2) { if( i + VECSZ*2 > len ) @@ -321,7 +321,7 @@ void sqrt32f(const float* src, float* dst, int len) int i = 0; #if CV_SIMD - const int VECSZ = v_float32::nlanes; + const int VECSZ = VTraits::vlanes(); for( ; i < len; i += VECSZ*2 ) { if( i + VECSZ*2 > len ) @@ -350,7 +350,7 @@ void sqrt64f(const double* src, double* dst, int len) int i = 0; #if CV_SIMD_64F - const int VECSZ = v_float64::nlanes; + const int VECSZ = VTraits::vlanes(); for( ; i < len; i += VECSZ*2 ) { if( i + VECSZ*2 > len ) @@ -452,7 +452,7 @@ void exp32f( const float *_x, float *y, int n ) float postscale = (float)exp_postscale; #if CV_SIMD - const int VECSZ = v_float32::nlanes; + const int VECSZ = VTraits::vlanes(); const v_float32 vprescale = vx_setall_f32((float)exp_prescale); const v_float32 vpostscale = vx_setall_f32((float)exp_postscale); const v_float32 vminval = vx_setall_f32(minval); @@ -481,26 +481,26 @@ void exp32f( const float *_x, float *y, int n ) xf0 = v_min(v_max(xf0, vminval), vmaxval); xf1 = v_min(v_max(xf1, vminval), vmaxval); - xf0 *= vprescale; - xf1 *= vprescale; + xf0 = v_mul(xf0, vprescale); + xf1 = v_mul(xf1, vprescale); v_int32 xi0 = v_round(xf0); v_int32 xi1 = v_round(xf1); - xf0 = (xf0 - v_cvt_f32(xi0))*vpostscale; - xf1 = (xf1 - v_cvt_f32(xi1))*vpostscale; + xf0 = v_mul(v_sub(xf0, v_cvt_f32(xi0)), vpostscale); + xf1 = v_mul(v_sub(xf1, v_cvt_f32(xi1)), vpostscale); - v_float32 yf0 = v_lut(expTab_f, xi0 & vidxmask); - v_float32 yf1 = v_lut(expTab_f, xi1 & vidxmask); + v_float32 yf0 = v_lut(expTab_f, v_and(xi0, vidxmask)); + v_float32 yf1 = v_lut(expTab_f, v_and(xi1, vidxmask)); v_int32 v0 = vx_setzero_s32(), v127 = vx_setall_s32(127), v255 = vx_setall_s32(255); - xi0 = v_min(v_max(v_shr(xi0) + v127, v0), v255); - xi1 = v_min(v_max(v_shr(xi1) + v127, v0), v255); + xi0 = v_min(v_max(v_add(v_shr<6>(xi0), v127), v0), v255); + xi1 = v_min(v_max(v_add(v_shr<6>(xi1), v127), v0), v255); - yf0 *= v_reinterpret_as_f32(v_shl<23>(xi0)); - yf1 *= v_reinterpret_as_f32(v_shl<23>(xi1)); + yf0 = v_mul(yf0, v_reinterpret_as_f32(v_shl<23>(xi0))); + yf1 = v_mul(yf1, v_reinterpret_as_f32(v_shl<23>(xi1))); - v_float32 zf0 = xf0 + vA1; - v_float32 zf1 = xf1 + vA1; + v_float32 zf0 = v_add(xf0, vA1); + v_float32 zf1 = v_add(xf1, vA1); zf0 = v_fma(zf0, xf0, vA2); zf1 = v_fma(zf1, xf1, vA2); @@ -511,8 +511,8 @@ void exp32f( const float *_x, float *y, int n ) zf0 = v_fma(zf0, xf0, vA4); zf1 = v_fma(zf1, xf1, vA4); - zf0 *= yf0; - zf1 *= yf1; + zf0 = v_mul(zf0, yf0); + zf1 = v_mul(zf1, yf1); if( y_aligned ) { @@ -566,7 +566,7 @@ void exp64f( const double *_x, double *y, int n ) double maxval = (exp_max_val/exp_prescale); #if CV_SIMD_64F - const int VECSZ = v_float64::nlanes; + const int VECSZ = VTraits::vlanes(); const v_float64 vprescale = vx_setall_f64(exp_prescale); const v_float64 vpostscale = vx_setall_f64(exp_postscale); const v_float64 vminval = vx_setall_f64(minval); @@ -596,30 +596,30 @@ void exp64f( const double *_x, double *y, int n ) xf0 = v_min(v_max(xf0, vminval), vmaxval); xf1 = v_min(v_max(xf1, vminval), vmaxval); - xf0 *= vprescale; - xf1 *= vprescale; + xf0 = v_mul(xf0, vprescale); + xf1 = v_mul(xf1, vprescale); v_int32 xi0 = v_round(xf0); v_int32 xi1 = v_round(xf1); - xf0 = (xf0 - v_cvt_f64(xi0))*vpostscale; - xf1 = (xf1 - v_cvt_f64(xi1))*vpostscale; + xf0 = v_mul(v_sub(xf0, v_cvt_f64(xi0)), vpostscale); + xf1 = v_mul(v_sub(xf1, v_cvt_f64(xi1)), vpostscale); - v_float64 yf0 = v_lut(expTab, xi0 & vidxmask); - v_float64 yf1 = v_lut(expTab, xi1 & vidxmask); + v_float64 yf0 = v_lut(expTab, v_and(xi0, vidxmask)); + v_float64 yf1 = v_lut(expTab, v_and(xi1, vidxmask)); v_int32 v0 = vx_setzero_s32(), v1023 = vx_setall_s32(1023), v2047 = vx_setall_s32(2047); - xi0 = v_min(v_max(v_shr(xi0) + v1023, v0), v2047); - xi1 = v_min(v_max(v_shr(xi1) + v1023, v0), v2047); + xi0 = v_min(v_max(v_add(v_shr<6>(xi0), v1023), v0), v2047); + xi1 = v_min(v_max(v_add(v_shr<6>(xi1), v1023), v0), v2047); v_int64 xq0, xq1, dummy; v_expand(xi0, xq0, dummy); v_expand(xi1, xq1, dummy); - yf0 *= v_reinterpret_as_f64(v_shl<52>(xq0)); - yf1 *= v_reinterpret_as_f64(v_shl<52>(xq1)); + yf0 = v_mul(yf0, v_reinterpret_as_f64(v_shl<52>(xq0))); + yf1 = v_mul(yf1, v_reinterpret_as_f64(v_shl<52>(xq1))); - v_float64 zf0 = xf0 + vA1; - v_float64 zf1 = xf1 + vA1; + v_float64 zf0 = v_add(xf0, vA1); + v_float64 zf1 = v_add(xf1, vA1); zf0 = v_fma(zf0, xf0, vA2); zf1 = v_fma(zf1, xf1, vA2); @@ -633,8 +633,8 @@ void exp64f( const double *_x, double *y, int n ) zf0 = v_fma(zf0, xf0, vA5); zf1 = v_fma(zf1, xf1, vA5); - zf0 *= yf0; - zf1 *= yf1; + zf0 = v_mul(zf0, yf0); + zf1 = v_mul(zf1, yf1); if( y_aligned ) { @@ -696,7 +696,7 @@ void log32f( const float *_x, float *y, int n ) const int* x = (const int*)_x; #if CV_SIMD - const int VECSZ = v_float32::nlanes; + const int VECSZ = VTraits::vlanes(); const v_float32 vln2 = vx_setall_f32((float)ln_2); const v_float32 v1 = vx_setall_f32(1.f); const v_float32 vshift = vx_setall_f32(-1.f/512); @@ -715,18 +715,18 @@ void log32f( const float *_x, float *y, int n ) } v_int32 h0 = vx_load(x + i); - v_int32 yi0 = (v_shr<23>(h0) & vx_setall_s32(255)) - vx_setall_s32(127); - v_int32 xi0 = (h0 & vx_setall_s32(LOGTAB_MASK2_32F)) | vx_setall_s32(127 << 23); + v_int32 yi0 = v_sub(v_and(v_shr<23>(h0), vx_setall_s32(255)), vx_setall_s32(127)); + v_int32 xi0 = v_or(v_and(h0, vx_setall_s32(LOGTAB_MASK2_32F)), vx_setall_s32(127 << 23)); - h0 = v_shr<23 - LOGTAB_SCALE - 1>(h0) & vx_setall_s32(LOGTAB_MASK*2); + h0 = v_and(v_shr<23 - 8 - 1>(h0), vx_setall_s32(((1 << 8) - 1) * 2)); v_float32 yf0, xf0; v_lut_deinterleave(logTab_f, h0, yf0, xf0); yf0 = v_fma(v_cvt_f32(yi0), vln2, yf0); - v_float32 delta = v_select(v_reinterpret_as_f32(h0 == vx_setall_s32(510)), vshift, vx_setall(0)); - xf0 = v_fma((v_reinterpret_as_f32(xi0) - v1), xf0, delta); + v_float32 delta = v_select(v_reinterpret_as_f32(v_eq(h0, vx_setall_s32(510))), vshift, vx_setall(0)); + xf0 = v_fma((v_sub(v_reinterpret_as_f32(xi0), v1)), xf0, delta); v_float32 zf0 = v_fma(xf0, vA0, vA1); zf0 = v_fma(zf0, xf0, vA2); @@ -771,7 +771,7 @@ void log64f( const double *x, double *y, int n ) int i = 0; #if CV_SIMD_64F - const int VECSZ = v_float64::nlanes; + const int VECSZ = VTraits::vlanes(); const v_float64 vln2 = vx_setall_f64(ln_2); const v_float64 @@ -791,20 +791,20 @@ void log64f( const double *x, double *y, int n ) v_int64 h0 = vx_load((const int64*)x + i); v_int32 yi0 = v_pack(v_shr<52>(h0), vx_setzero_s64()); - yi0 = (yi0 & vx_setall_s32(0x7ff)) - vx_setall_s32(1023); + yi0 = v_sub(v_and(yi0, vx_setall_s32(2047)), vx_setall_s32(1023)); - v_int64 xi0 = (h0 & vx_setall_s64(LOGTAB_MASK2_64F)) | vx_setall_s64((int64)1023 << 52); + v_int64 xi0 = v_or(v_and(h0, vx_setall_s64(LOGTAB_MASK2_64F)), vx_setall_s64((int64)1023 << 52)); h0 = v_shr<52 - LOGTAB_SCALE - 1>(h0); - v_int32 idx = v_pack(h0, h0) & vx_setall_s32(LOGTAB_MASK*2); + v_int32 idx = v_and(v_pack(h0, h0), vx_setall_s32(((1 << 8) - 1) * 2)); v_float64 xf0, yf0; v_lut_deinterleave(logTab, idx, yf0, xf0); yf0 = v_fma(v_cvt_f64(yi0), vln2, yf0); - v_float64 delta = v_cvt_f64(idx == vx_setall_s32(510))*vx_setall_f64(1./512); - xf0 = v_fma(v_reinterpret_as_f64(xi0) - vx_setall_f64(1.), xf0, delta); + v_float64 delta = v_mul(v_cvt_f64(v_eq(idx, vx_setall_s32(510))), vx_setall_f64(1. / 512)); + xf0 = v_fma(v_sub(v_reinterpret_as_f64(xi0), vx_setall_f64(1.)), xf0, delta); - v_float64 xq = xf0*xf0; + v_float64 xq = v_mul(xf0, xf0); v_float64 zf0 = v_fma(xq, vA0, vA2); v_float64 zf1 = v_fma(xq, vA1, vA3); zf0 = v_fma(zf0, xq, vA4); diff --git a/modules/core/src/matmul.simd.hpp b/modules/core/src/matmul.simd.hpp index 54a04c5f97..e3dbc552df 100644 --- a/modules/core/src/matmul.simd.hpp +++ b/modules/core/src/matmul.simd.hpp @@ -1584,7 +1584,7 @@ transform_16u( const ushort* src, ushort* dst, const float* m, int len, int scn, v_float32x4 _m2h = v_rotate_left<1>(_m2l); v_float32x4 _m3h = v_rotate_left<1>(_m3l); v_int16x8 _delta(0, -32768, -32768, -32768, -32768, -32768, -32768, 0); - for( ; x <= len*3 - v_uint16x8::nlanes; x += 3*v_uint16x8::nlanes/4 ) + for( ; x <= len*3 - VTraits::vlanes(); x += 3*VTraits::vlanes()/4 ) v_store(dst + x, v_rotate_right<1>(v_reinterpret_as_u16(v_add_wrap(v_pack( v_round(v_matmuladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand(src + x ))), _m0h, _m1h, _m2h, _m3h)), v_round(v_matmuladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand(src + x + 3))), _m0l, _m1l, _m2l, _m3l))), _delta)))); @@ -1664,10 +1664,10 @@ transform_32f( const float* src, float* dst, const float* m, int len, int scn, i v_float32x4 _m2 = v_load(m + 10); v_float32x4 _m3 = v_load(m + 15); v_float32x4 _m4(m[4], m[9], m[14], m[19]); - for( ; x < len*4; x += v_float32x4::nlanes ) + for( ; x < len*4; x += VTraits::vlanes() ) { v_float32x4 v_src = v_load(src + x); - v_store(dst + x, v_reduce_sum4(v_src * _m0, v_src * _m1, v_src * _m2, v_src * _m3) + _m4); + v_store(dst + x, v_add(v_reduce_sum4(v_mul(v_src, _m0), v_mul(v_src, _m1), v_mul(v_src, _m2), v_mul(v_src, _m3)), _m4)); } #else // CV_SIMD_WIDTH >= 16 && !CV_SIMD128 for( ; x < len*4; x += 4 ) @@ -2113,12 +2113,12 @@ MulTransposedR(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double for( k = 0; k < size.height; k++, tsrc += srcstep ) { v_float64x2 a = v_setall_f64((double)col_buf[k]); - s0 += a * v_load(tsrc+0); - s1 += a * v_load(tsrc+2); + s0 = v_add(s0, v_mul(a, v_load(tsrc + 0))); + s1 = v_add(s1, v_mul(a, v_load(tsrc + 2))); } - v_store((double*)(tdst+j), s0*v_scale); - v_store((double*)(tdst+j+2), s1*v_scale); + v_store((double*)(tdst+j), v_mul(s0, v_scale)); + v_store((double*)(tdst+j+2), v_mul(s1, v_scale)); } else #endif { @@ -2174,12 +2174,12 @@ MulTransposedR(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double for( k = 0; k < size.height; k++, tsrc+=srcstep, d+=deltastep ) { v_float64x2 a = v_setall_f64((double)col_buf[k]); - s0 += a * (v_load(tsrc+0) - v_load(d+0)); - s1 += a * (v_load(tsrc+2) - v_load(d+2)); + s0 = v_add(s0, v_mul(a, v_sub(v_load(tsrc + 0), v_load(d + 0)))); + s1 = v_add(s1, v_mul(a, v_sub(v_load(tsrc + 2), v_load(d + 2)))); } - v_store((double*)(tdst+j), s0*v_scale); - v_store((double*)(tdst+j+2), s1*v_scale); + v_store((double*)(tdst+j), v_mul(s0, v_scale)); + v_store((double*)(tdst+j+2), v_mul(s1, v_scale)); } else #endif @@ -2249,8 +2249,7 @@ MulTransposedL(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double v_float64x2 v_s = v_setzero_f64(); for( k = 0; k <= size.width - 4; k += 4 ) - v_s += (v_load(v_tsrc1+k) * v_load(v_tsrc2+k)) + - (v_load(v_tsrc1+k+2) * v_load(v_tsrc2+k+2)); + v_s = v_add(v_s, v_add(v_mul(v_load(v_tsrc1 + k), v_load(v_tsrc2 + k)), v_mul(v_load(v_tsrc1 + k + 2), v_load(v_tsrc2 + k + 2)))); s += v_reduce_sum(v_s); } else @@ -2303,8 +2302,7 @@ MulTransposedL(const Mat& srcmat, const Mat& dstmat, const Mat& deltamat, double v_float64x2 v_s = v_setzero_f64(); for( k = 0; k <= size.width - 4; k += 4, v_tdelta2 += delta_shift ) - v_s += ((v_load(v_tsrc2+k) - v_load(v_tdelta2)) * v_load(v_row_buf+k)) + - ((v_load(v_tsrc2+k+2) - v_load(v_tdelta2+2)) * v_load(v_row_buf+k+2)); + v_s = v_add(v_s, v_add(v_mul(v_sub(v_load(v_tsrc2 + k), v_load(v_tdelta2)), v_load(v_row_buf + k)), v_mul(v_sub(v_load(v_tsrc2 + k + 2), v_load(v_tdelta2 + 2)), v_load(v_row_buf + k + 2)))); s += v_reduce_sum(v_s); tdelta2 = (const dT *)(v_tdelta2); @@ -2566,7 +2564,7 @@ double dotProd_32s(const int* src1, const int* src2, int len) v_sum0 = v_dotprod_expand_fast(v_src10, v_src20, v_sum0); v_sum1 = v_dotprod_expand_fast(v_src11, v_src21, v_sum1); } - v_sum0 += v_sum1; + v_sum0 = v_add(v_sum0, v_sum1); #endif for (; i < len - step; i += step, src1 += step, src2 += step) { diff --git a/modules/core/src/matrix_transform.cpp b/modules/core/src/matrix_transform.cpp index c4c7a73b4c..5a80ac8ca7 100644 --- a/modules/core/src/matrix_transform.cpp +++ b/modules/core/src/matrix_transform.cpp @@ -356,10 +356,10 @@ void transposeND(InputArray src_, const std::vector& order, OutputArray dst #if CV_SIMD128 template CV_ALWAYS_INLINE void flipHoriz_single( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size, size_t esz ) { - typedef typename V::lane_type T; + typedef typename VTraits::lane_type T; int end = (int)(size.width*esz); int width = (end + 1)/2; - int width_1 = width & -v_uint8x16::nlanes; + int width_1 = width & -VTraits::vlanes(); int i, j; #if CV_STRONG_ALIGNMENT @@ -368,15 +368,15 @@ template CV_ALWAYS_INLINE void flipHoriz_single( const uchar* src, s for( ; size.height--; src += sstep, dst += dstep ) { - for( i = 0, j = end; i < width_1; i += v_uint8x16::nlanes, j -= v_uint8x16::nlanes ) + for( i = 0, j = end; i < width_1; i += VTraits::vlanes(), j -= VTraits::vlanes() ) { V t0, t1; t0 = v_load((T*)((uchar*)src + i)); - t1 = v_load((T*)((uchar*)src + j - v_uint8x16::nlanes)); + t1 = v_load((T*)((uchar*)src + j - VTraits::vlanes())); t0 = v_reverse(t0); t1 = v_reverse(t1); - v_store((T*)(dst + j - v_uint8x16::nlanes), t0); + v_store((T*)(dst + j - VTraits::vlanes()), t0); v_store((T*)(dst + i), t1); } if (isAligned(src, dst)) @@ -446,14 +446,14 @@ flipHoriz( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size, #if CV_STRONG_ALIGNMENT size_t alignmentMark = ((size_t)src)|((size_t)dst)|sstep|dstep; #endif - if (esz == 2 * v_uint8x16::nlanes) + if (esz == 2 * (size_t)VTraits::vlanes()) { int end = (int)(size.width*esz); int width = end/2; for( ; size.height--; src += sstep, dst += dstep ) { - for( int i = 0, j = end - 2 * v_uint8x16::nlanes; i < width; i += 2 * v_uint8x16::nlanes, j -= 2 * v_uint8x16::nlanes ) + for( int i = 0, j = end - 2 * VTraits::vlanes(); i < width; i += 2 * VTraits::vlanes(), j -= 2 * VTraits::vlanes() ) { #if CV_SIMD256 v_uint8x32 t0, t1; @@ -466,25 +466,25 @@ flipHoriz( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size, v_uint8x16 t0, t1, t2, t3; t0 = v_load((uchar*)src + i); - t1 = v_load((uchar*)src + i + v_uint8x16::nlanes); + t1 = v_load((uchar*)src + i + VTraits::vlanes()); t2 = v_load((uchar*)src + j); - t3 = v_load((uchar*)src + j + v_uint8x16::nlanes); + t3 = v_load((uchar*)src + j + VTraits::vlanes()); v_store(dst + j, t0); - v_store(dst + j + v_uint8x16::nlanes, t1); + v_store(dst + j + VTraits::vlanes(), t1); v_store(dst + i, t2); - v_store(dst + i + v_uint8x16::nlanes, t3); + v_store(dst + i + VTraits::vlanes(), t3); #endif } } } - else if (esz == v_uint8x16::nlanes) + else if (esz == (size_t)VTraits::vlanes()) { int end = (int)(size.width*esz); int width = end/2; for( ; size.height--; src += sstep, dst += dstep ) { - for( int i = 0, j = end - v_uint8x16::nlanes; i < width; i += v_uint8x16::nlanes, j -= v_uint8x16::nlanes ) + for( int i = 0, j = end - VTraits::vlanes(); i < width; i += VTraits::vlanes(), j -= VTraits::vlanes() ) { v_uint8x16 t0, t1; @@ -534,19 +534,19 @@ flipHoriz( const uchar* src, size_t sstep, uchar* dst, size_t dstep, Size size, for( ; size.height--; src += sstep, dst += dstep ) { - for ( int i = 0, j = end; i < width; i += v_uint8x16::nlanes + sizeof(uint64_t), j -= v_uint8x16::nlanes + sizeof(uint64_t) ) + for ( int i = 0, j = end; i < width; i += VTraits::vlanes() + sizeof(uint64_t), j -= VTraits::vlanes() + sizeof(uint64_t) ) { v_uint8x16 t0, t1; uint64_t t2, t3; t0 = v_load((uchar*)src + i); - t2 = *((uint64_t*)((uchar*)src + i + v_uint8x16::nlanes)); - t1 = v_load((uchar*)src + j - v_uint8x16::nlanes - sizeof(uint64_t)); + t2 = *((uint64_t*)((uchar*)src + i + VTraits::vlanes())); + t1 = v_load((uchar*)src + j - VTraits::vlanes() - sizeof(uint64_t)); t3 = *((uint64_t*)((uchar*)src + j - sizeof(uint64_t))); - v_store(dst + j - v_uint8x16::nlanes - sizeof(uint64_t), t0); + v_store(dst + j - VTraits::vlanes() - sizeof(uint64_t), t0); *((uint64_t*)(dst + j - sizeof(uint64_t))) = t2; v_store(dst + i, t1); - *((uint64_t*)(dst + i + v_uint8x16::nlanes)) = t3; + *((uint64_t*)(dst + i + VTraits::vlanes())) = t3; } } } diff --git a/modules/core/src/minmax.cpp b/modules/core/src/minmax.cpp index a31ccad83e..1bbcd08eca 100644 --- a/modules/core/src/minmax.cpp +++ b/modules/core/src/minmax.cpp @@ -141,7 +141,7 @@ CV_ALWAYS_INLINE uint64_t v_reduce_min(const v_uint64x2& a) CV_ALWAYS_INLINE v_uint64x2 v_select(const v_uint64x2& mask, const v_uint64x2& a, const v_uint64x2& b) { - return b ^ ((a ^ b) & mask); + return v_xor(b, v_and(v_xor(a, b), mask)); } #endif @@ -151,16 +151,16 @@ minMaxIdx_reduce_##suffix( VT &valMin, VT &valMax, IT &idxMin, IT &idxMax, IT &n T &minVal, T &maxVal, size_t &minIdx, size_t &maxIdx, \ size_t delta ) \ { \ - if ( v_check_any(idxMin != none) ) \ + if ( v_check_any(v_ne(idxMin, none)) ) \ { \ minVal = v_reduce_min(valMin); \ - minIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_setall_##suffix((IR)minVal) == valMin), \ + minIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_eq(v_setall_##suffix((IR)minVal), valMin)), \ idxMin, v_setall_##suffix2(maxLimit))) + delta; \ } \ - if ( v_check_any(idxMax != none) ) \ + if ( v_check_any(v_ne(idxMax, none)) ) \ { \ maxVal = v_reduce_max(valMax); \ - maxIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_setall_##suffix((IR)maxVal) == valMax), \ + maxIdx = (size_t)v_reduce_min(v_select(v_reinterpret_as_##suffix2(v_eq(v_setall_##suffix((IR)maxVal), valMax)), \ idxMax, v_setall_##suffix2(maxLimit))) + delta; \ } \ } @@ -210,18 +210,18 @@ static void minMaxIdx_8u(const uchar* src, const uchar* mask, int* minval, int* size_t* minidx, size_t* maxidx, int len, size_t startidx ) { #if CV_SIMD128 - if ( len >= v_uint8x16::nlanes ) + if ( len >= VTraits::vlanes() ) { int j, len0; int minVal, maxVal; size_t minIdx, maxIdx; minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - (int)0, (int)UCHAR_MAX, v_uint8x16::nlanes, len, startidx, j, len0 ); + (int)0, (int)UCHAR_MAX, VTraits::vlanes(), len, startidx, j, len0 ); - if ( j <= len0 - v_uint8x16::nlanes ) + if ( j <= len0 - VTraits::vlanes() ) { - v_uint8x16 inc = v_setall_u8(v_uint8x16::nlanes); + v_uint8x16 inc = v_setall_u8((uchar)VTraits::vlanes()); v_uint8x16 none = v_reinterpret_as_u8(v_setall_s8(-1)); v_uint8x16 idxStart(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); @@ -235,31 +235,31 @@ static void minMaxIdx_8u(const uchar* src, const uchar* mask, int* minval, int* if ( !mask ) { - for( ; k < std::min(len0, j + 15 * v_uint8x16::nlanes); k += v_uint8x16::nlanes ) + for( ; k < std::min(len0, j + 15 * VTraits::vlanes()); k += VTraits::vlanes() ) { v_uint8x16 data = v_load(src + k); - v_uint8x16 cmpMin = (data < valMin); - v_uint8x16 cmpMax = (data > valMax); + v_uint8x16 cmpMin = (v_lt(data, valMin)); + v_uint8x16 cmpMax = (v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; + idx = v_add(idx, inc); } } else { - for( ; k < std::min(len0, j + 15 * v_uint8x16::nlanes); k += v_uint8x16::nlanes ) + for( ; k < std::min(len0, j + 15 * VTraits::vlanes()); k += VTraits::vlanes() ) { v_uint8x16 data = v_load(src + k); - v_uint8x16 maskVal = v_load(mask + k) != v_setzero_u8(); - v_uint8x16 cmpMin = (data < valMin) & maskVal; - v_uint8x16 cmpMax = (data > valMax) & maskVal; + v_uint8x16 maskVal = v_ne(v_load(mask + k), v_setzero_u8()); + v_uint8x16 cmpMin = v_and(v_lt(data, valMin), maskVal); + v_uint8x16 cmpMax = v_and(v_gt(data, valMax), maskVal); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(cmpMin, data, valMin); valMax = v_select(cmpMax, data, valMax); - idx += inc; + idx = v_add(idx, inc); } } @@ -287,18 +287,18 @@ static void minMaxIdx_8s(const schar* src, const uchar* mask, int* minval, int* size_t* minidx, size_t* maxidx, int len, size_t startidx ) { #if CV_SIMD128 - if ( len >= v_int8x16::nlanes ) + if ( len >= VTraits::vlanes() ) { int j, len0; int minVal, maxVal; size_t minIdx, maxIdx; minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - (int)SCHAR_MIN, (int)SCHAR_MAX, v_int8x16::nlanes, len, startidx, j, len0 ); + (int)SCHAR_MIN, (int)SCHAR_MAX, VTraits::vlanes(), len, startidx, j, len0 ); - if ( j <= len0 - v_int8x16::nlanes ) + if ( j <= len0 - VTraits::vlanes() ) { - v_uint8x16 inc = v_setall_u8(v_int8x16::nlanes); + v_uint8x16 inc = v_setall_u8((uchar)VTraits::vlanes()); v_uint8x16 none = v_reinterpret_as_u8(v_setall_s8(-1)); v_uint8x16 idxStart(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); @@ -312,31 +312,31 @@ static void minMaxIdx_8s(const schar* src, const uchar* mask, int* minval, int* if ( !mask ) { - for( ; k < std::min(len0, j + 15 * v_int8x16::nlanes); k += v_int8x16::nlanes ) + for( ; k < std::min(len0, j + 15 * VTraits::vlanes()); k += VTraits::vlanes() ) { v_int8x16 data = v_load(src + k); - v_uint8x16 cmpMin = v_reinterpret_as_u8(data < valMin); - v_uint8x16 cmpMax = v_reinterpret_as_u8(data > valMax); + v_uint8x16 cmpMin = v_reinterpret_as_u8(v_lt(data, valMin)); + v_uint8x16 cmpMax = v_reinterpret_as_u8(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; + idx = v_add(idx, inc); } } else { - for( ; k < std::min(len0, j + 15 * v_int8x16::nlanes); k += v_int8x16::nlanes ) + for( ; k < std::min(len0, j + 15 * VTraits::vlanes()); k += VTraits::vlanes() ) { v_int8x16 data = v_load(src + k); - v_uint8x16 maskVal = v_load(mask + k) != v_setzero_u8(); - v_uint8x16 cmpMin = v_reinterpret_as_u8(data < valMin) & maskVal; - v_uint8x16 cmpMax = v_reinterpret_as_u8(data > valMax) & maskVal; + v_uint8x16 maskVal = v_ne(v_load(mask + k), v_setzero_u8()); + v_uint8x16 cmpMin = v_and(v_reinterpret_as_u8(v_lt(data, valMin)), maskVal); + v_uint8x16 cmpMax = v_and(v_reinterpret_as_u8(v_gt(data, valMax)), maskVal); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_s8(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_s8(cmpMax), data, valMax); - idx += inc; + idx = v_add(idx, inc); } } @@ -364,18 +364,18 @@ static void minMaxIdx_16u(const ushort* src, const uchar* mask, int* minval, int size_t* minidx, size_t* maxidx, int len, size_t startidx ) { #if CV_SIMD128 - if ( len >= v_uint16x8::nlanes ) + if ( len >= VTraits::vlanes() ) { int j, len0; int minVal, maxVal; size_t minIdx, maxIdx; minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - (int)0, (int)USHRT_MAX, v_uint16x8::nlanes, len, startidx, j, len0 ); + (int)0, (int)USHRT_MAX, VTraits::vlanes(), len, startidx, j, len0 ); - if ( j <= len0 - v_uint16x8::nlanes ) + if ( j <= len0 - VTraits::vlanes() ) { - v_uint16x8 inc = v_setall_u16(v_uint16x8::nlanes); + v_uint16x8 inc = v_setall_u16((uchar)VTraits::vlanes()); v_uint16x8 none = v_reinterpret_as_u16(v_setall_s16(-1)); v_uint16x8 idxStart(0, 1, 2, 3, 4, 5, 6, 7); @@ -389,31 +389,31 @@ static void minMaxIdx_16u(const ushort* src, const uchar* mask, int* minval, int if ( !mask ) { - for( ; k < std::min(len0, j + 8191 * v_uint16x8::nlanes); k += v_uint16x8::nlanes ) + for( ; k < std::min(len0, j + 8191 * VTraits::vlanes()); k += VTraits::vlanes() ) { v_uint16x8 data = v_load(src + k); - v_uint16x8 cmpMin = (data < valMin); - v_uint16x8 cmpMax = (data > valMax); + v_uint16x8 cmpMin = (v_lt(data, valMin)); + v_uint16x8 cmpMax = (v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; + idx = v_add(idx, inc); } } else { - for( ; k < std::min(len0, j + 8191 * v_uint16x8::nlanes); k += v_uint16x8::nlanes ) + for( ; k < std::min(len0, j + 8191 * VTraits::vlanes()); k += VTraits::vlanes() ) { v_uint16x8 data = v_load(src + k); - v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16(); - v_uint16x8 cmpMin = (data < valMin) & maskVal; - v_uint16x8 cmpMax = (data > valMax) & maskVal; + v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); + v_uint16x8 cmpMin = v_and(v_lt(data, valMin), maskVal); + v_uint16x8 cmpMax = v_and(v_gt(data, valMax), maskVal); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(cmpMin, data, valMin); valMax = v_select(cmpMax, data, valMax); - idx += inc; + idx = v_add(idx, inc); } } @@ -441,18 +441,18 @@ static void minMaxIdx_16s(const short* src, const uchar* mask, int* minval, int* size_t* minidx, size_t* maxidx, int len, size_t startidx ) { #if CV_SIMD128 - if ( len >= v_int16x8::nlanes ) + if ( len >= VTraits::vlanes() ) { int j, len0; int minVal, maxVal; size_t minIdx, maxIdx; minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - (int)SHRT_MIN, (int)SHRT_MAX, v_int16x8::nlanes, len, startidx, j, len0 ); + (int)SHRT_MIN, (int)SHRT_MAX, VTraits::vlanes(), len, startidx, j, len0 ); - if ( j <= len0 - v_int16x8::nlanes ) + if ( j <= len0 - VTraits::vlanes() ) { - v_uint16x8 inc = v_setall_u16(v_int16x8::nlanes); + v_uint16x8 inc = v_setall_u16((uchar)VTraits::vlanes()); v_uint16x8 none = v_reinterpret_as_u16(v_setall_s16(-1)); v_uint16x8 idxStart(0, 1, 2, 3, 4, 5, 6, 7); @@ -466,31 +466,31 @@ static void minMaxIdx_16s(const short* src, const uchar* mask, int* minval, int* if ( !mask ) { - for( ; k < std::min(len0, j + 8191 * v_int16x8::nlanes); k += v_int16x8::nlanes ) + for( ; k < std::min(len0, j + 8191 * VTraits::vlanes()); k += VTraits::vlanes() ) { v_int16x8 data = v_load(src + k); - v_uint16x8 cmpMin = v_reinterpret_as_u16(data < valMin); - v_uint16x8 cmpMax = v_reinterpret_as_u16(data > valMax); + v_uint16x8 cmpMin = v_reinterpret_as_u16(v_lt(data, valMin)); + v_uint16x8 cmpMax = v_reinterpret_as_u16(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; + idx = v_add(idx, inc); } } else { - for( ; k < std::min(len0, j + 8191 * v_int16x8::nlanes); k += v_int16x8::nlanes ) + for( ; k < std::min(len0, j + 8191 * VTraits::vlanes()); k += VTraits::vlanes() ) { v_int16x8 data = v_load(src + k); - v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16(); - v_uint16x8 cmpMin = v_reinterpret_as_u16(data < valMin) & maskVal; - v_uint16x8 cmpMax = v_reinterpret_as_u16(data > valMax) & maskVal; + v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); + v_uint16x8 cmpMin = v_and(v_reinterpret_as_u16(v_lt(data, valMin)), maskVal); + v_uint16x8 cmpMax = v_and(v_reinterpret_as_u16(v_gt(data, valMax)), maskVal); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_s16(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_s16(cmpMax), data, valMax); - idx += inc; + idx = v_add(idx, inc); } } @@ -518,14 +518,14 @@ static void minMaxIdx_32s(const int* src, const uchar* mask, int* minval, int* m size_t* minidx, size_t* maxidx, int len, size_t startidx ) { #if CV_SIMD128 - if ( len >= 2 * v_int32x4::nlanes ) + if ( len >= 2 * VTraits::vlanes() ) { - int j = 0, len0 = len & -(2 * v_int32x4::nlanes); + int j = 0, len0 = len & -(2 * VTraits::vlanes()); int minVal = *minval, maxVal = *maxval; size_t minIdx = *minidx, maxIdx = *maxidx; { - v_uint32x4 inc = v_setall_u32(v_int32x4::nlanes); + v_uint32x4 inc = v_setall_u32(VTraits::vlanes()); v_uint32x4 none = v_reinterpret_as_u32(v_setall_s32(-1)); v_uint32x4 idxStart(0, 1, 2, 3); @@ -539,49 +539,49 @@ static void minMaxIdx_32s(const int* src, const uchar* mask, int* minval, int* m if ( !mask ) { - for( ; k < std::min(len0, j + 32766 * 2 * v_int32x4::nlanes); k += 2 * v_int32x4::nlanes ) + for( ; k < std::min(len0, j + 32766 * 2 * VTraits::vlanes()); k += 2 * VTraits::vlanes() ) { v_int32x4 data = v_load(src + k); - v_uint32x4 cmpMin = v_reinterpret_as_u32(data < valMin); - v_uint32x4 cmpMax = v_reinterpret_as_u32(data > valMax); + v_uint32x4 cmpMin = v_reinterpret_as_u32(v_lt(data, valMin)); + v_uint32x4 cmpMax = v_reinterpret_as_u32(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; - data = v_load(src + k + v_int32x4::nlanes); - cmpMin = v_reinterpret_as_u32(data < valMin); - cmpMax = v_reinterpret_as_u32(data > valMax); + idx = v_add(idx, inc); + data = v_load(src + k + VTraits::vlanes()); + cmpMin = v_reinterpret_as_u32(v_lt(data, valMin)); + cmpMax = v_reinterpret_as_u32(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; + idx = v_add(idx, inc); } } else { - for( ; k < std::min(len0, j + 32766 * 2 * v_int32x4::nlanes); k += 2 * v_int32x4::nlanes ) + for( ; k < std::min(len0, j + 32766 * 2 * VTraits::vlanes()); k += 2 * VTraits::vlanes() ) { v_int32x4 data = v_load(src + k); - v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16(); + v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); v_int32x4 maskVal1, maskVal2; v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2); - v_uint32x4 cmpMin = v_reinterpret_as_u32((data < valMin) & maskVal1); - v_uint32x4 cmpMax = v_reinterpret_as_u32((data > valMax) & maskVal1); + v_uint32x4 cmpMin = v_reinterpret_as_u32(v_and(v_lt(data, valMin), maskVal1)); + v_uint32x4 cmpMax = v_reinterpret_as_u32(v_and(v_gt(data, valMax), maskVal1)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_s32(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_s32(cmpMax), data, valMax); - idx += inc; - data = v_load(src + k + v_int32x4::nlanes); - cmpMin = v_reinterpret_as_u32((data < valMin) & maskVal2); - cmpMax = v_reinterpret_as_u32((data > valMax) & maskVal2); + idx = v_add(idx, inc); + data = v_load(src + k + VTraits::vlanes()); + cmpMin = v_reinterpret_as_u32(v_and(v_lt(data, valMin), maskVal2)); + cmpMax = v_reinterpret_as_u32(v_and(v_gt(data, valMax), maskVal2)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_s32(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_s32(cmpMax), data, valMax); - idx += inc; + idx = v_add(idx, inc); } } @@ -609,18 +609,18 @@ static void minMaxIdx_32f(const float* src, const uchar* mask, float* minval, fl size_t* minidx, size_t* maxidx, int len, size_t startidx ) { #if CV_SIMD128 - if ( len >= 2 * v_float32x4::nlanes ) + if ( len >= 2 * VTraits::vlanes() ) { int j, len0; float minVal, maxVal; size_t minIdx, maxIdx; minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - FLT_MIN, FLT_MAX, 2 * v_float32x4::nlanes, len, startidx, j, len0 ); + FLT_MIN, FLT_MAX, 2 * VTraits::vlanes(), len, startidx, j, len0 ); - if ( j <= len0 - 2 * v_float32x4::nlanes ) + if ( j <= len0 - 2 * VTraits::vlanes() ) { - v_uint32x4 inc = v_setall_u32(v_float32x4::nlanes); + v_uint32x4 inc = v_setall_u32(VTraits::vlanes()); v_uint32x4 none = v_reinterpret_as_u32(v_setall_s32(-1)); v_uint32x4 idxStart(0, 1, 2, 3); @@ -634,49 +634,49 @@ static void minMaxIdx_32f(const float* src, const uchar* mask, float* minval, fl if ( !mask ) { - for( ; k < std::min(len0, j + 32766 * 2 * v_float32x4::nlanes); k += 2 * v_float32x4::nlanes ) + for( ; k < std::min(len0, j + 32766 * 2 * VTraits::vlanes()); k += 2 * VTraits::vlanes() ) { v_float32x4 data = v_load(src + k); - v_uint32x4 cmpMin = v_reinterpret_as_u32(data < valMin); - v_uint32x4 cmpMax = v_reinterpret_as_u32(data > valMax); + v_uint32x4 cmpMin = v_reinterpret_as_u32(v_lt(data, valMin)); + v_uint32x4 cmpMax = v_reinterpret_as_u32(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; - data = v_load(src + k + v_float32x4::nlanes); - cmpMin = v_reinterpret_as_u32(data < valMin); - cmpMax = v_reinterpret_as_u32(data > valMax); + idx = v_add(idx, inc); + data = v_load(src + k + VTraits::vlanes()); + cmpMin = v_reinterpret_as_u32(v_lt(data, valMin)); + cmpMax = v_reinterpret_as_u32(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; + idx = v_add(idx, inc); } } else { - for( ; k < std::min(len0, j + 32766 * 2 * v_float32x4::nlanes); k += 2 * v_float32x4::nlanes ) + for( ; k < std::min(len0, j + 32766 * 2 * VTraits::vlanes()); k += 2 * VTraits::vlanes() ) { v_float32x4 data = v_load(src + k); - v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16(); + v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); v_int32x4 maskVal1, maskVal2; v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2); - v_uint32x4 cmpMin = v_reinterpret_as_u32(v_reinterpret_as_s32(data < valMin) & maskVal1); - v_uint32x4 cmpMax = v_reinterpret_as_u32(v_reinterpret_as_s32(data > valMax) & maskVal1); + v_uint32x4 cmpMin = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_lt(data, valMin)), maskVal1)); + v_uint32x4 cmpMax = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_gt(data, valMax)), maskVal1)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_f32(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_f32(cmpMax), data, valMax); - idx += inc; - data = v_load(src + k + v_float32x4::nlanes); - cmpMin = v_reinterpret_as_u32(v_reinterpret_as_s32(data < valMin) & maskVal2); - cmpMax = v_reinterpret_as_u32(v_reinterpret_as_s32(data > valMax) & maskVal2); + idx = v_add(idx, inc); + data = v_load(src + k + VTraits::vlanes()); + cmpMin = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_lt(data, valMin)), maskVal2)); + cmpMax = v_reinterpret_as_u32(v_and(v_reinterpret_as_s32(v_gt(data, valMax)), maskVal2)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_f32(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_f32(cmpMax), data, valMax); - idx += inc; + idx = v_add(idx, inc); } } @@ -704,18 +704,18 @@ static void minMaxIdx_64f(const double* src, const uchar* mask, double* minval, size_t* minidx, size_t* maxidx, int len, size_t startidx ) { #if CV_SIMD128_64F - if ( len >= 4 * v_float64x2::nlanes ) + if ( len >= 4 * VTraits::vlanes() ) { int j, len0; double minVal, maxVal; size_t minIdx, maxIdx; minMaxIdx_init( src, mask, minval, maxval, minidx, maxidx, minVal, maxVal, minIdx, maxIdx, - DBL_MIN, DBL_MAX, 4 * v_float64x2::nlanes, len, startidx, j, len0 ); + DBL_MIN, DBL_MAX, 4 * VTraits::vlanes(), len, startidx, j, len0 ); - if ( j <= len0 - 4 * v_float64x2::nlanes ) + if ( j <= len0 - 4 * VTraits::vlanes() ) { - v_uint64x2 inc = v_setall_u64(v_float64x2::nlanes); + v_uint64x2 inc = v_setall_u64(VTraits::vlanes()); v_uint64x2 none = v_reinterpret_as_u64(v_setall_s64(-1)); v_uint64x2 idxStart(0, 1); @@ -729,84 +729,84 @@ static void minMaxIdx_64f(const double* src, const uchar* mask, double* minval, if ( !mask ) { - for( ; k < std::min(len0, j + 32764 * 4 * v_float64x2::nlanes); k += 4 * v_float64x2::nlanes ) + for( ; k < std::min(len0, j + 32764 * 4 * VTraits::vlanes()); k += 4 * VTraits::vlanes() ) { v_float64x2 data = v_load(src + k); - v_uint64x2 cmpMin = v_reinterpret_as_u64(data < valMin); - v_uint64x2 cmpMax = v_reinterpret_as_u64(data > valMax); + v_uint64x2 cmpMin = v_reinterpret_as_u64(v_lt(data, valMin)); + v_uint64x2 cmpMax = v_reinterpret_as_u64(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; - data = v_load(src + k + v_float64x2::nlanes); - cmpMin = v_reinterpret_as_u64(data < valMin); - cmpMax = v_reinterpret_as_u64(data > valMax); + idx = v_add(idx, inc); + data = v_load(src + k + VTraits::vlanes()); + cmpMin = v_reinterpret_as_u64(v_lt(data, valMin)); + cmpMax = v_reinterpret_as_u64(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; - data = v_load(src + k + 2 * v_float64x2::nlanes); - cmpMin = v_reinterpret_as_u64(data < valMin); - cmpMax = v_reinterpret_as_u64(data > valMax); + idx = v_add(idx, inc); + data = v_load(src + k + 2 * VTraits::vlanes()); + cmpMin = v_reinterpret_as_u64(v_lt(data, valMin)); + cmpMax = v_reinterpret_as_u64(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; - data = v_load(src + k + 3 * v_float64x2::nlanes); - cmpMin = v_reinterpret_as_u64(data < valMin); - cmpMax = v_reinterpret_as_u64(data > valMax); + idx = v_add(idx, inc); + data = v_load(src + k + 3 * VTraits::vlanes()); + cmpMin = v_reinterpret_as_u64(v_lt(data, valMin)); + cmpMax = v_reinterpret_as_u64(v_gt(data, valMax)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_min(data, valMin); valMax = v_max(data, valMax); - idx += inc; + idx = v_add(idx, inc); } } else { - for( ; k < std::min(len0, j + 32764 * 4 * v_float64x2::nlanes); k += 4 * v_float64x2::nlanes ) + for( ; k < std::min(len0, j + 32764 * 4 * VTraits::vlanes()); k += 4 * VTraits::vlanes() ) { v_float64x2 data = v_load(src + k); - v_uint16x8 maskVal = v_load_expand(mask + k) != v_setzero_u16(); + v_uint16x8 maskVal = v_ne(v_load_expand(mask + k), v_setzero_u16()); v_int32x4 maskVal1, maskVal2; v_expand(v_reinterpret_as_s16(maskVal), maskVal1, maskVal2); v_int64x2 maskVal3, maskVal4; v_expand(maskVal1, maskVal3, maskVal4); - v_uint64x2 cmpMin = v_reinterpret_as_u64(v_reinterpret_as_s64(data < valMin) & maskVal3); - v_uint64x2 cmpMax = v_reinterpret_as_u64(v_reinterpret_as_s64(data > valMax) & maskVal3); + v_uint64x2 cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal3)); + v_uint64x2 cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal3)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax); - idx += inc; - data = v_load(src + k + v_float64x2::nlanes); - cmpMin = v_reinterpret_as_u64(v_reinterpret_as_s64(data < valMin) & maskVal4); - cmpMax = v_reinterpret_as_u64(v_reinterpret_as_s64(data > valMax) & maskVal4); + idx = v_add(idx, inc); + data = v_load(src + k + VTraits::vlanes()); + cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal4)); + cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal4)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax); - idx += inc; - data = v_load(src + k + 2 * v_float64x2::nlanes); + idx = v_add(idx, inc); + data = v_load(src + k + 2 * VTraits::vlanes()); v_expand(maskVal2, maskVal3, maskVal4); - cmpMin = v_reinterpret_as_u64(v_reinterpret_as_s64(data < valMin) & maskVal3); - cmpMax = v_reinterpret_as_u64(v_reinterpret_as_s64(data > valMax) & maskVal3); + cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal3)); + cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal3)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax); - idx += inc; - data = v_load(src + k + 3 * v_float64x2::nlanes); - cmpMin = v_reinterpret_as_u64(v_reinterpret_as_s64(data < valMin) & maskVal4); - cmpMax = v_reinterpret_as_u64(v_reinterpret_as_s64(data > valMax) & maskVal4); + idx = v_add(idx, inc); + data = v_load(src + k + 3 * VTraits::vlanes()); + cmpMin = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_lt(data, valMin)), maskVal4)); + cmpMax = v_reinterpret_as_u64(v_and(v_reinterpret_as_s64(v_gt(data, valMax)), maskVal4)); idxMin = v_select(cmpMin, idx, idxMin); idxMax = v_select(cmpMax, idx, idxMax); valMin = v_select(v_reinterpret_as_f64(cmpMin), data, valMin); valMax = v_select(v_reinterpret_as_f64(cmpMax), data, valMax); - idx += inc; + idx = v_add(idx, inc); } } diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 4f26ff305e..0316ab375c 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -408,6 +408,8 @@ struct HWFeatures g_hwFeatureNames[CPU_NEON] = "NEON"; g_hwFeatureNames[CPU_NEON_DOTPROD] = "NEON_DOTPROD"; + g_hwFeatureNames[CPU_NEON_FP16] = "NEON_FP16"; + g_hwFeatureNames[CPU_NEON_BF16] = "NEON_BF16"; g_hwFeatureNames[CPU_VSX] = "VSX"; g_hwFeatureNames[CPU_VSX3] = "VSX3"; @@ -425,6 +427,7 @@ struct HWFeatures g_hwFeatureNames[CPU_RVV] = "RVV"; + g_hwFeatureNames[CPU_LSX] = "LSX"; g_hwFeatureNames[CPU_LASX] = "LASX"; } @@ -566,10 +569,15 @@ struct HWFeatures while ((size_t)read(cpufile, &auxv, size_auxv_t) == size_auxv_t) { + // see https://elixir.bootlin.com/linux/latest/source/arch/arm64/include/uapi/asm/hwcap.h if (auxv.a_type == AT_HWCAP) { - have[CV_CPU_NEON_DOTPROD] = (auxv.a_un.a_val & (1 << 20)) != 0; - break; + have[CV_CPU_NEON_DOTPROD] = (auxv.a_un.a_val & (1 << 20)) != 0; // HWCAP_ASIMDDP + have[CV_CPU_NEON_FP16] = (auxv.a_un.a_val & (1 << 10)) != 0; // HWCAP_ASIMDHP + } + else if (auxv.a_type == AT_HWCAP2) + { + have[CV_CPU_NEON_BF16] = (auxv.a_un.a_val & (1 << 14)) != 0; // HWCAP2_BF16 } } @@ -623,16 +631,23 @@ struct HWFeatures have[CV_CPU_NEON] = true; #endif #if (defined __ARM_FP && (((__ARM_FP & 0x2) != 0) && defined __ARM_NEON__)) - have[CV_CPU_FP16] = true; - #endif - #if (defined __ARM_FEATURE_DOTPROD) - int has_feat_dotprod = 0; - size_t has_feat_dotprod_size = sizeof(has_feat_dotprod); - sysctlbyname("hw.optional.arm.FEAT_DotProd", &has_feat_dotprod, &has_feat_dotprod_size, NULL, 0); - if (has_feat_dotprod) { - have[CV_CPU_NEON_DOTPROD] = true; - } + have[CV_CPU_FP16] = have[CV_CPU_NEON_FP16] = true; #endif + // system.cpp may be compiled w/o special -march=armv8...+dotprod, -march=armv8...+bf16 etc., + // so we check for the features in any case, no mater what are the compile flags. + // We check the real hardware capabilities here. + int has_feat_dotprod = 0; + size_t has_feat_dotprod_size = sizeof(has_feat_dotprod); + sysctlbyname("hw.optional.arm.FEAT_DotProd", &has_feat_dotprod, &has_feat_dotprod_size, NULL, 0); + if (has_feat_dotprod) { + have[CV_CPU_NEON_DOTPROD] = true; + } + int has_feat_bf16 = 0; + size_t has_feat_bf16_size = sizeof(has_feat_bf16); + sysctlbyname("hw.optional.arm.FEAT_BF16", &has_feat_bf16, &has_feat_bf16_size, NULL, 0); + if (has_feat_bf16) { + have[CV_CPU_NEON_BF16] = true; + } #elif (defined __clang__) #if (defined __ARM_NEON__ || (defined __ARM_NEON && defined __aarch64__)) have[CV_CPU_NEON] = true; @@ -689,6 +704,10 @@ struct HWFeatures have[CV_CPU_RVV] = true; #endif + #if defined __loongarch_sx + have[CV_CPU_LSX] = true; + #endif + #if defined __loongarch_asx have[CV_CPU_LASX] = true; #endif diff --git a/modules/core/test/test_intrin_utils.hpp b/modules/core/test/test_intrin_utils.hpp index 1ece6de82f..38b8d10f7b 100644 --- a/modules/core/test/test_intrin_utils.hpp +++ b/modules/core/test/test_intrin_utils.hpp @@ -1745,13 +1745,8 @@ template struct TheTest R a = dataA; R b = dataB; -#if CV_SIMD_SCALABLE Data dataEQ = v_eq(a, b); Data dataNE = v_ne(a, b); -#else - Data dataEQ = (a == b); - Data dataNE = (a != b); -#endif for (int i = 0; i < VTraits::vlanes(); ++i) { diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index 68075d15ab..c77831749f 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -602,7 +602,7 @@ static void setValue(SparseMat& M, const int* idx, double value, RNG& rng) CV_Error(CV_StsUnsupportedFormat, ""); } -#if defined(__GNUC__) && (__GNUC__ == 11 || __GNUC__ == 12) +#if defined(__GNUC__) && (__GNUC__ == 11 || __GNUC__ == 12 || __GNUC__ == 13) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Warray-bounds" #endif diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 681625816c..9122168c65 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -240,6 +240,12 @@ ocv_create_module(${libs} ${dnn_runtime_libs}) ocv_add_samples() ocv_add_accuracy_tests(${dnn_runtime_libs}) +if(NOT BUILD_PROTOBUF) + if(TARGET opencv_test_dnn) + ocv_target_compile_definitions(opencv_test_dnn PRIVATE "OPENCV_DNN_EXTERNAL_PROTOBUF=1") + endif() +endif() + set(perf_path "${CMAKE_CURRENT_LIST_DIR}/perf") file(GLOB_RECURSE perf_srcs "${perf_path}/*.cpp") file(GLOB_RECURSE perf_hdrs "${perf_path}/*.hpp" "${perf_path}/*.h") @@ -295,6 +301,13 @@ if(TARGET ocv.3rdparty.cann AND OPENCV_TEST_DNN_CANN) endif() endif() +ocv_option(OPENCV_TEST_DNN_TIMVX "Build test with TIM-VX" (HAVE_TIMVX)) +if(OPENCV_TEST_DNN_TIMVX) + if(TARGET opencv_test_dnn) + ocv_target_compile_definitions(opencv_test_dnn PRIVATE "HAVE_TIMVX=1") + endif() +endif() + ocv_option(OPENCV_TEST_DNN_TFLITE "Build test with TFLite" (OPENCV_DNN_TFLITE)) if(OPENCV_TEST_DNN_TFLITE) if(TARGET opencv_test_dnn) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 2f2bc6b548..fc48607f31 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -343,6 +343,22 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + /** @brief GatherElements layer + * GatherElements takes two inputs data and indices of the same rank r >= 1 and an optional attribute axis and works such that: + * output[i][j][k] = data[index[i][j][k]][j][k] if axis = 0 and r = 3 + * output[i][j][k] = data[i][index[i][j][k]][k] if axis = 1 and r = 3 + * output[i][j][k] = data[i][j][index[i][j][k]] if axis = 2 and r = 3 + * + * Gather, on the other hand, takes a data tensor of rank r >= 1, and indices tensor of rank q, and works such that: + * it gathers the enteries along axis dimension of the input data indexed by indices and concatenates them in an output tensor of rank q + (r - 1) + * e.g. If axis = 0, let k = indices[i_{0}, ..., i_{q-1}] then output[i_{0}, ..., i_{q-1}, j_{0}, ..., j_{r-2}] = input[k , j_{0}, ..., j_{r-2}]: + **/ + class CV_EXPORTS GatherElementsLayer : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS PoolingLayer : public Layer { public: @@ -1127,7 +1143,7 @@ CV__DNN_INLINE_NS_BEGIN class CV_EXPORTS LayerNormLayer : public Layer { public: - bool hasBias; + CV_DEPRECATED_EXTERNAL bool hasBias; // Deprecated, preserve for compatibility int axis; float epsilon; diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 76bf277bc1..88ba31410b 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -941,14 +941,14 @@ CV__DNN_INLINE_NS_BEGIN * * `*.caffemodel` (Caffe, http://caffe.berkeleyvision.org/) * * `*.pb` (TensorFlow, https://www.tensorflow.org/) * * `*.weights` (Darknet, https://pjreddie.com/darknet/) - * * `*.bin` (DLDT, https://software.intel.com/openvino-toolkit) + * * `*.bin` | `*.onnx` (OpenVINO, https://software.intel.com/openvino-toolkit) * * `*.onnx` (ONNX, https://onnx.ai/) * @param[in] config Text file contains network configuration. It could be a * file with the following extensions: * * `*.prototxt` (Caffe, http://caffe.berkeleyvision.org/) * * `*.pbtxt` (TensorFlow, https://www.tensorflow.org/) * * `*.cfg` (Darknet, https://pjreddie.com/darknet/) - * * `*.xml` (DLDT, https://software.intel.com/openvino-toolkit) + * * `*.xml` (OpenVINO, https://software.intel.com/openvino-toolkit) * @param[in] framework Explicit framework name tag to determine a format. * @returns Net object. * @@ -979,7 +979,7 @@ CV__DNN_INLINE_NS_BEGIN * backend. */ CV_EXPORTS_W - Net readNetFromModelOptimizer(const String &xml, const String &bin); + Net readNetFromModelOptimizer(const String &xml, const String &bin = ""); /** @brief Load a network from Intel's Model Optimizer intermediate representation. * @param[in] bufferModelConfig Buffer contains XML configuration with network's topology. diff --git a/modules/dnn/perf/perf_layer.cpp b/modules/dnn/perf/perf_layer.cpp index 519a477509..38015a0103 100644 --- a/modules/dnn/perf/perf_layer.cpp +++ b/modules/dnn/perf/perf_layer.cpp @@ -633,6 +633,56 @@ PERF_TEST_P_(Layer_LayerNormExpanded, DISABLED_LayerNormExpanded) test_layer({N, H ,W}); } +struct Layer_GatherElements : public TestBaseWithParam > +{ + void test_layer(const std::vector& data_shape, const std::vector& indices_shape, int axis = 0) + { + int backendId = get<0>(GetParam()); + int targetId = get<1>(GetParam()); + + Mat data(data_shape, CV_32FC1); + Mat indices(indices_shape, CV_32FC1); + + randu(data, 0.f, 1.f); + randu(indices, 0, data_shape[axis]); + + Net net; + LayerParams lp; + lp.type = "GatherElements"; + lp.name = "testLayer"; + lp.set("axis", axis); + int id = net.addLayerToPrev(lp.name, lp.type, lp); + net.connect(0, 0, id, 0); + net.connect(0, 1, id, 1); + + // warmup + { + std::vector inpNames(3); + inpNames[0] = "data"; + inpNames[1] = "indices"; + net.setInputsNames(inpNames); + net.setInput(data, inpNames[0]); + net.setInput(indices, inpNames[1]); + + net.setPreferableBackend(backendId); + net.setPreferableTarget(targetId); + Mat out = net.forward(); + } + + TEST_CYCLE() + { + Mat res = net.forward(); + } + + SANITY_CHECK_NOTHING(); + } +}; + +PERF_TEST_P_(Layer_GatherElements, GatherElements) +{ + test_layer({2700, 1, 2914}, {2700, 1, 81}, 2); +} + INSTANTIATE_TEST_CASE_P(/**/, Layer_Slice, dnnBackendsAndTargets(/* withInferenceEngine = */false, /* obsolete_withHalide = */false)); INSTANTIATE_TEST_CASE_P(/**/, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); #ifdef HAVE_CUDA @@ -642,6 +692,7 @@ INSTANTIATE_TEST_CASE_P(/**/, Layer_Scatter, testing::Values(std::make_tuple(DNN INSTANTIATE_TEST_CASE_P(/**/, Layer_ScatterND, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNormExpanded, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); +INSTANTIATE_TEST_CASE_P(/**/, Layer_GatherElements, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU))); typedef TestBaseWithParam > > Layer_FullyConnected; diff --git a/modules/dnn/src/dnn_read.cpp b/modules/dnn/src/dnn_read.cpp index 437d19559c..89aef857d5 100644 --- a/modules/dnn/src/dnn_read.cpp +++ b/modules/dnn/src/dnn_read.cpp @@ -39,9 +39,11 @@ Net readNet(const String& _model, const String& _config, const String& _framewor std::swap(model, config); return readNetFromDarknet(config, model); } - if (framework == "dldt" || modelExt == "bin" || configExt == "bin" || modelExt == "xml" || configExt == "xml") + if (framework == "dldt" || framework == "openvino" || + modelExt == "bin" || configExt == "bin" || + modelExt == "xml" || configExt == "xml") { - if (modelExt == "xml" || configExt == "bin") + if (modelExt == "xml" || configExt == "bin" || modelExt == "onnx") std::swap(model, config); return readNetFromModelOptimizer(config, model); } @@ -62,7 +64,7 @@ Net readNet(const String& _framework, const std::vector& bufferModel, return readNetFromTensorflow(bufferModel, bufferConfig); else if (framework == "darknet") return readNetFromDarknet(bufferConfig, bufferModel); - else if (framework == "dldt") + else if (framework == "dldt" || framework == "openvino") return readNetFromModelOptimizer(bufferConfig, bufferModel); else if (framework == "tflite") return readNetFromTFLite(bufferModel); diff --git a/modules/dnn/src/dnn_utils.cpp b/modules/dnn/src/dnn_utils.cpp index d4d7dda008..f25da7458a 100644 --- a/modules/dnn/src/dnn_utils.cpp +++ b/modules/dnn/src/dnn_utils.cpp @@ -22,6 +22,48 @@ Image2BlobParams::Image2BlobParams(const Scalar& scalefactor_, const Size& size_ datalayout(datalayout_), paddingmode(mode_) {} +void getVector(InputArrayOfArrays images_, std::vector& images) { + images_.getMatVector(images); +} + +void getVector(InputArrayOfArrays images_, std::vector& images) { + images_.getUMatVector(images); +} + +void getMat(UMat& blob, InputArray blob_, AccessFlag flag) { + if(blob_.kind() == _InputArray::UMAT) + blob = blob_.getUMat(); + else if(blob_.kind() == _InputArray::MAT) { + blob = blob_.getUMat(); + } +} + +void getMat(Mat& blob, InputArray blob_, AccessFlag flag) { + if(blob_.kind() == _InputArray::UMAT) + blob = blob_.getMat(); + else if(blob_.kind() == _InputArray::MAT) { + blob = blob_.getMat(); + } +} + +void getChannelFromBlob(Mat& m, InputArray blob, int i, int j, int rows, int cols, int type) { + m = Mat(rows, cols, type, blob.getMat().ptr(i, j)); +} + +void getChannelFromBlob(UMat& m, InputArray blob, int i, int j, int rows, int cols, int type) { + UMat ublob = blob.getUMat(); + int offset = (i * ublob.step.p[0] + j * ublob.step.p[1]) / ublob.elemSize(); + int length = 1; + for(int i = 0; i < ublob.dims; ++i) { + length *= ublob.size[i]; + } + + const int newShape[1] { length }; + UMat reshaped = ublob.reshape(1, 1, newShape); + UMat roi = reshaped(Rect(0, offset, 1, rows * cols)); + m = roi.reshape(CV_MAT_CN(type), rows); +} + Mat blobFromImage(InputArray image, const double scalefactor, const Size& size, const Scalar& mean, bool swapRB, bool crop, int ddepth) { @@ -35,8 +77,13 @@ void blobFromImage(InputArray image, OutputArray blob, double scalefactor, const Size& size, const Scalar& mean, bool swapRB, bool crop, int ddepth) { CV_TRACE_FUNCTION(); - std::vector images(1, image.getMat()); - blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth); + if (image.kind() == _InputArray::UMAT) { + std::vector images(1, image.getUMat()); + blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth); + } else { + std::vector images(1, image.getMat()); + blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth); + } } Mat blobFromImages(InputArrayOfArrays images, double scalefactor, Size size, @@ -52,9 +99,9 @@ void blobFromImages(InputArrayOfArrays images_, OutputArray blob_, double scalef Size size, const Scalar& mean_, bool swapRB, bool crop, int ddepth) { CV_TRACE_FUNCTION(); - if (images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT && + if (images_.kind() != _InputArray::STD_VECTOR_UMAT && images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT && images_.kind() != _InputArray::STD_VECTOR_VECTOR) { - String error_message = "The data is expected as vectors of vectors or vectors of matrices."; + String error_message = "The data is expected as vectors of vectors, vectors of Mats or vectors of UMats."; CV_Error(Error::StsBadArg, error_message); } Image2BlobParams param(Scalar::all(scalefactor), size, mean_, swapRB, ddepth); @@ -71,13 +118,6 @@ Mat blobFromImageWithParams(InputArray image, const Image2BlobParams& param) return blob; } -void blobFromImageWithParams(InputArray image, OutputArray blob, const Image2BlobParams& param) -{ - CV_TRACE_FUNCTION(); - std::vector images(1, image.getMat()); - blobFromImagesWithParams(images, blob, param); -} - Mat blobFromImagesWithParams(InputArrayOfArrays images, const Image2BlobParams& param) { CV_TRACE_FUNCTION(); @@ -86,19 +126,22 @@ Mat blobFromImagesWithParams(InputArrayOfArrays images, const Image2BlobParams& return blob; } -void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, const Image2BlobParams& param) +template +void blobFromImagesWithParamsImpl(InputArrayOfArrays images_, Tmat& blob_, const Image2BlobParams& param) { CV_TRACE_FUNCTION(); - if (images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT && - images_.kind() != _InputArray::STD_VECTOR_VECTOR) { - String error_message = "The data is expected as vectors of vectors or vectors of matrices."; + if(!std::is_same::value && !std::is_same::value) { + String error_message = "The template parameter is expected to be either a cv::Mat or a cv::UMat"; CV_Error(Error::StsBadArg, error_message); } + CV_CheckType(param.ddepth, param.ddepth == CV_32F || param.ddepth == CV_8U, "Blob depth should be CV_32F or CV_8U"); Size size = param.size; - std::vector images; - images_.getMatVector(images); + + std::vector images; + getVector(images_, images); + CV_Assert(!images.empty()); if (param.ddepth == CV_8U) @@ -164,12 +207,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con if (images[i].depth() == CV_8U && param.ddepth == CV_32F) images[i].convertTo(images[i], CV_32F); - images[i] -= mean; + subtract(images[i], mean, images[i]); multiply(images[i], scalefactor, images[i]); } size_t nimages = images.size(); - Mat image0 = images[0]; + Tmat image0 = images[0]; CV_Assert(image0.dims == 2); if (param.datalayout == DNN_LAYOUT_NCHW) @@ -178,21 +221,22 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con { int sz[] = { (int)nimages, nch, image0.rows, image0.cols }; blob_.create(4, sz, param.ddepth); - Mat blob = blob_.getMat(); - Mat ch[4]; + std::vector ch(4); for (size_t i = 0; i < nimages; i++) { - const Mat& image = images[i]; + const Tmat& image = images[i]; CV_Assert(image.depth() == blob_.depth()); nch = image.channels(); CV_Assert(image.dims == 2 && (nch == 3 || nch == 4)); CV_Assert(image.size() == image0.size()); - for (int j = 0; j < nch; j++) - ch[j] = Mat(image.rows, image.cols, param.ddepth, blob.ptr((int)i, j)); + for (int j = 0; j < nch; j++) { + getChannelFromBlob(ch[j], blob_, i, j ,image.rows, image.cols, param.ddepth); + } if (param.swapRB) std::swap(ch[0], ch[2]); + split(image, ch); } } @@ -201,11 +245,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con CV_Assert(nch == 1); int sz[] = { (int)nimages, 1, image0.rows, image0.cols }; blob_.create(4, sz, param.ddepth); - Mat blob = blob_.getMat(); + Mat blob; + getMat(blob, blob_, ACCESS_RW); for (size_t i = 0; i < nimages; i++) { - const Mat& image = images[i]; + const Tmat& image = images[i]; CV_Assert(image.depth() == blob_.depth()); nch = image.channels(); CV_Assert(image.dims == 2 && (nch == 1)); @@ -219,11 +264,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con { int sz[] = { (int)nimages, image0.rows, image0.cols, nch}; blob_.create(4, sz, param.ddepth); - Mat blob = blob_.getMat(); + Mat blob; + getMat(blob, blob_, ACCESS_RW); int subMatType = CV_MAKETYPE(param.ddepth, nch); for (size_t i = 0; i < nimages; i++) { - const Mat& image = images[i]; + const Tmat& image = images[i]; CV_Assert(image.depth() == blob_.depth()); CV_Assert(image.channels() == image0.channels()); CV_Assert(image.size() == image0.size()); @@ -243,6 +289,72 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con { CV_Error(Error::StsUnsupportedFormat, "Unsupported data layout in blobFromImagesWithParams function."); } + CV_Assert(blob_.total()); +} + +void blobFromImagesWithParams(InputArrayOfArrays images, OutputArray blob, const Image2BlobParams& param) { + CV_TRACE_FUNCTION(); + + if (images.kind() == _InputArray::STD_VECTOR_UMAT) { + if(blob.kind() == _InputArray::UMAT) { + UMat& u = blob.getUMatRef(); + blobFromImagesWithParamsImpl(images, u, param); + return; + } else if(blob.kind() == _InputArray::MAT) { + UMat u = blob.getMatRef().getUMat(ACCESS_WRITE); + blobFromImagesWithParamsImpl(images, u, param); + u.copyTo(blob); + return; + } + } else if (images.kind() == _InputArray::STD_VECTOR_MAT) { + if(blob.kind() == _InputArray::UMAT) { + Mat m = blob.getUMatRef().getMat(ACCESS_WRITE); + blobFromImagesWithParamsImpl(images, m, param); + m.copyTo(blob); + return; + } else if(blob.kind() == _InputArray::MAT) { + Mat& m = blob.getMatRef(); + blobFromImagesWithParamsImpl(images, m, param); + return; + } + } + + CV_Error(Error::StsBadArg, "Images are expected to be a vector of either a Mat or UMat and Blob is expected to be either a Mat or UMat"); +} + +void blobFromImageWithParams(InputArray image, OutputArray blob, const Image2BlobParams& param) +{ + CV_TRACE_FUNCTION(); + + if (image.kind() == _InputArray::UMAT) { + if(blob.kind() == _InputArray::UMAT) { + UMat& u = blob.getUMatRef(); + std::vector images(1, image.getUMat()); + blobFromImagesWithParamsImpl(images, u, param); + return; + } else if(blob.kind() == _InputArray::MAT) { + UMat u = blob.getMatRef().getUMat(ACCESS_RW); + std::vector images(1, image.getUMat()); + blobFromImagesWithParamsImpl(images, u, param); + u.copyTo(blob); + return; + } + } else if (image.kind() == _InputArray::MAT) { + if(blob.kind() == _InputArray::UMAT) { + Mat m = blob.getUMatRef().getMat(ACCESS_RW); + std::vector images(1, image.getMat()); + blobFromImagesWithParamsImpl(images, m, param); + m.copyTo(blob); + return; + } else if(blob.kind() == _InputArray::MAT) { + Mat& m = blob.getMatRef(); + std::vector images(1, image.getMat()); + blobFromImagesWithParamsImpl(images, m, param); + return; + } + } + + CV_Error(Error::StsBadArg, "Image an Blob are expected to be either a Mat or UMat"); } void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_) diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 3183d71f0b..e70d5dad47 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -157,6 +157,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Arg, ArgLayer); CV_DNN_REGISTER_LAYER_CLASS(Reciprocal, ReciprocalLayer); CV_DNN_REGISTER_LAYER_CLASS(Gather, GatherLayer); + CV_DNN_REGISTER_LAYER_CLASS(GatherElements, GatherElementsLayer); CV_DNN_REGISTER_LAYER_CLASS(LayerNormalization, LayerNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Expand, ExpandLayer); diff --git a/modules/dnn/src/int8layers/convolution_layer.cpp b/modules/dnn/src/int8layers/convolution_layer.cpp index 60301a406c..ba9b31fe35 100644 --- a/modules/dnn/src/int8layers/convolution_layer.cpp +++ b/modules/dnn/src/int8layers/convolution_layer.cpp @@ -29,10 +29,10 @@ static inline void v_expand_mul_add(const v_int8x16& a, const v_int8x16& b, v_int32x4 t0, t1; v_mul_expand(a0, b0, t0, t1); - out0 += t0; out1 += t1; + out0 = v_add(out0, t0); out1 = v_add(out1, t1); v_mul_expand(a1, b1, t0, t1); - out2 += t0; out3 += t1; + out2 = v_add(out2, t0); out3 = v_add(out3, t1); } #endif @@ -1055,10 +1055,10 @@ public: v_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); v_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); - vout0 = voutzp + v_round(v_cvt_f32(vout0)*vmult); - vout1 = voutzp + v_round(v_cvt_f32(vout1)*vmult); - vout2 = voutzp + v_round(v_cvt_f32(vout2)*vmult); - vout3 = voutzp + v_round(v_cvt_f32(vout3)*vmult); + vout0 = v_add(voutzp, v_round(v_mul(v_cvt_f32(vout0), vmult))); + vout1 = v_add(voutzp, v_round(v_mul(v_cvt_f32(vout1), vmult))); + vout2 = v_add(voutzp, v_round(v_mul(v_cvt_f32(vout2), vmult))); + vout3 = v_add(voutzp, v_round(v_mul(v_cvt_f32(vout3), vmult))); vout0 = v_min(v_max(vout0, outmin), outmax); vout1 = v_min(v_max(vout1, outmin), outmax); @@ -1408,12 +1408,12 @@ public: vs12 = v_dotprod_expand_fast(w1, r2, vs12); vs13 = v_dotprod_expand_fast(w1, r3, vs13); } - s0 += v_int32x4(v_reduce_sum(vs00), v_reduce_sum(vs01), v_reduce_sum(vs02), v_reduce_sum(vs03)); - s1 += v_int32x4(v_reduce_sum(vs10), v_reduce_sum(vs11), v_reduce_sum(vs12), v_reduce_sum(vs13)); + s0 = v_add(s0, v_int32x4(v_reduce_sum(vs00), v_reduce_sum(vs01), v_reduce_sum(vs02), v_reduce_sum(vs03))); + s1 = v_add(s1, v_int32x4(v_reduce_sum(vs10), v_reduce_sum(vs11), v_reduce_sum(vs12), v_reduce_sum(vs13))); if( cn1 == inpCn ) { - s0 = voutzp + v_round(v_cvt_f32(s0)*vmult0); - s1 = voutzp + v_round(v_cvt_f32(s1)*vmult1); + s0 = v_add(voutzp, v_round(v_mul(v_cvt_f32(s0), vmult0))); + s1 = v_add(voutzp, v_round(v_mul(v_cvt_f32(s1), vmult1))); s0 = v_min(v_max(s0, outmin), outmax); s1 = v_min(v_max(s1, outmin), outmax); diff --git a/modules/dnn/src/int8layers/fully_connected_layer.cpp b/modules/dnn/src/int8layers/fully_connected_layer.cpp index b8e3bd6ee5..ba5b0d79c1 100644 --- a/modules/dnn/src/int8layers/fully_connected_layer.cpp +++ b/modules/dnn/src/int8layers/fully_connected_layer.cpp @@ -323,8 +323,8 @@ public: vs3 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep*3 + k), vs3); } - s += v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3)); - v_int32x4 out = outzp + v_round(v_cvt_f32(s)*mult); + s = v_add(s, v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3))); + v_int32x4 out = v_add(outzp, v_round(v_mul(v_cvt_f32(s), mult))); v_store(dptr + i, v_min(v_max(out, outmin), outmax)); } #endif diff --git a/modules/dnn/src/int8layers/pooling_layer.cpp b/modules/dnn/src/int8layers/pooling_layer.cpp index bfff3d34c5..b321d730f7 100644 --- a/modules/dnn/src/int8layers/pooling_layer.cpp +++ b/modules/dnn/src/int8layers/pooling_layer.cpp @@ -631,17 +631,17 @@ public: (int)srcData[index + stride_w*10], (int)srcData[index + stride_w*11]); v_int32x4 v3((int)srcData[index + stride_w*12], (int)srcData[index + stride_w*13], (int)srcData[index + stride_w*14], (int)srcData[index + stride_w*15]); - sum_val0 += v0; - sum_val1 += v1; - sum_val2 += v2; - sum_val3 += v3; + sum_val0 = v_add(sum_val0, v0); + sum_val1 = v_add(sum_val1, v1); + sum_val2 = v_add(sum_val2, v2); + sum_val3 = v_add(sum_val3, v3); } } - sum_val0 = v_round(v_cvt_f32(sum_val0)*ikarea) + voutzp; - sum_val1 = v_round(v_cvt_f32(sum_val1)*ikarea) + voutzp; - sum_val2 = v_round(v_cvt_f32(sum_val2)*ikarea) + voutzp; - sum_val3 = v_round(v_cvt_f32(sum_val3)*ikarea) + voutzp; + sum_val0 = v_add(v_round(v_mul(v_cvt_f32(sum_val0), ikarea)), voutzp); + sum_val1 = v_add(v_round(v_mul(v_cvt_f32(sum_val1), ikarea)), voutzp); + sum_val2 = v_add(v_round(v_mul(v_cvt_f32(sum_val2), ikarea)), voutzp); + sum_val3 = v_add(v_round(v_mul(v_cvt_f32(sum_val3), ikarea)), voutzp); v_store(dstData + x0, v_pack(v_pack(sum_val0, sum_val1), v_pack(sum_val2, sum_val3))); x0 += 15; diff --git a/modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp b/modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp index 3e969336ad..59f069eeaa 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_depthwise.cpp @@ -236,13 +236,11 @@ void depthWiseBlockConv2D(const float* wptr, v21 = v_load(imgptr2 + in_j + dilation_w), v22 = v_load(imgptr2 + in_j + dilation_w*2); - v_float32x4 vout = v00*vw00 + v01*vw01 + v02*vw02 + - v10*vw10 + v11*vw11 + v12*vw12 + - v20*vw20 + v21*vw21 + v22*vw22 + vbias; + v_float32x4 vout = v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_mul(v00, vw00), v_mul(v01, vw01)), v_mul(v02, vw02)), v_mul(v10, vw10)), v_mul(v11, vw11)), v_mul(v12, vw12)), v_mul(v20, vw20)), v_mul(v21, vw21)), v_mul(v22, vw22)), vbias); if (fusedAdd) - vout = v_load(outptr + out_j) + vout; + vout = v_add(v_load(outptr + out_j), vout); if (relu) - vout = v_select(vout > z, vout, vout*vrc); + vout = v_select(v_gt(vout, z), vout, v_mul(vout, vrc)); v_store(outptr + out_j, vout); } } @@ -268,14 +266,12 @@ void depthWiseBlockConv2D(const float* wptr, v_load_deinterleave(imgptr2 + in_j, v20, v21); v_load_deinterleave(imgptr2 + in_j + 2, v22, unused); - v_float32x4 vout = v00 * vw00 + v01 * vw01 + v02 * vw02 + - v10 * vw10 + v11 * vw11 + v12 * vw12 + - v20 * vw20 + v21 * vw21 + v22 * vw22 + vbias; + v_float32x4 vout = v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_add(v_mul(v00, vw00), v_mul(v01, vw01)), v_mul(v02, vw02)), v_mul(v10, vw10)), v_mul(v11, vw11)), v_mul(v12, vw12)), v_mul(v20, vw20)), v_mul(v21, vw21)), v_mul(v22, vw22)), vbias); if (fusedAdd) - vout = v_load(outptr + out_j) + vout; + vout = v_add(v_load(outptr + out_j), vout); if (relu) - vout = v_select(vout > z, vout, vout*vrc); + vout = v_select(v_gt(vout, z), vout, v_mul(vout, vrc)); v_store(outptr + out_j, vout); } } @@ -381,11 +377,11 @@ void depthWiseBlockConv1D(const float* wptr, v01 = v_load(imgptr0 + in_j + dilation_w), v02 = v_load(imgptr0 + in_j + dilation_w*2); - v_float32x4 vout = v00*vw00 + v01*vw01 + v02*vw02 + vbias; + v_float32x4 vout = v_add(v_add(v_add(v_mul(v00, vw00), v_mul(v01, vw01)), v_mul(v02, vw02)), vbias); if (fusedAdd) - vout = v_load(outptr + out_j) + vout; + vout = v_add(v_load(outptr + out_j), vout); if (relu) - vout = v_select(vout > z, vout, vout*vrc); + vout = v_select(v_gt(vout, z), vout, v_mul(vout, vrc)); v_store(outptr + out_j, vout); } } @@ -407,13 +403,13 @@ void depthWiseBlockConv1D(const float* wptr, v_load_deinterleave(imgptr0 + in_j, v00, v01); v_load_deinterleave(imgptr0 + in_j + 2, v02, unused); - v_float32x4 vout = v00 * vw00 + v01 * vw01 + v02 * vw02 + vbias; + v_float32x4 vout = v_add(v_add(v_add(v_mul(v00, vw00), v_mul(v01, vw01)), v_mul(v02, vw02)), vbias); if (fusedAdd) - vout = v_load(outptr + out_j) + vout; + vout = v_add(v_load(outptr + out_j), vout); if (relu) - vout = v_select(vout > z, vout, vout*vrc); + vout = v_select(v_gt(vout, z), vout, v_mul(vout, vrc)); v_store(outptr + out_j, vout); } } diff --git a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp index a18943994c..605cf37949 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_winograd_f63.cpp @@ -430,32 +430,32 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, /* Y[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*X */ /* Y[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*X */ v_float32x4 q5_25 = v_setall_f32(5.25f), t00, t01, t10, t11; - t00 = x40 - x20; - t01 = x41 - x21; - t10 = x30 - x50; - t11 = x31 - x51; - v_float32x4 y00 = v_fma(t00, q5_25, x00 - x60); - v_float32x4 y01 = v_fma(t01, q5_25, x01 - x61); - v_float32x4 y70 = v_fma(t10, q5_25, x70 - x10); - v_float32x4 y71 = v_fma(t11, q5_25, x71 - x11); + t00 = v_sub(x40, x20); + t01 = v_sub(x41, x21); + t10 = v_sub(x30, x50); + t11 = v_sub(x31, x51); + v_float32x4 y00 = v_fma(t00, q5_25, v_sub(x00, x60)); + v_float32x4 y01 = v_fma(t01, q5_25, v_sub(x01, x61)); + v_float32x4 y70 = v_fma(t10, q5_25, v_sub(x70, x10)); + v_float32x4 y71 = v_fma(t11, q5_25, v_sub(x71, x11)); /* Y[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*X */ /* Y[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*X */ v_float32x4 qm4_25 = v_setall_f32(-4.25f); - t00 = v_fma(x30, qm4_25, x10 + x50); - t01 = v_fma(x31, qm4_25, x11 + x51); - t10 = v_fma(x40, qm4_25, x20 + x60); - t11 = v_fma(x41, qm4_25, x21 + x61); + t00 = v_fma(x30, qm4_25, v_add(x10, x50)); + t01 = v_fma(x31, qm4_25, v_add(x11, x51)); + t10 = v_fma(x40, qm4_25, v_add(x20, x60)); + t11 = v_fma(x41, qm4_25, v_add(x21, x61)); - v_float32x4 y10 = t00 + t10, y11 = t01 + t11; - v_float32x4 y20 = t10 - t00, y21 = t11 - t01; + v_float32x4 y10 = v_add(t00, t10), y11 = v_add(t01, t11); + v_float32x4 y20 = v_sub(t10, t00), y21 = v_sub(t11, t01); /* Y[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*X */ /* Y[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*X */ v_float32x4 q0_5 = v_setall_f32(0.5f), q0_25 = v_setall_f32(0.25f); v_float32x4 qm2_5 = v_setall_f32(-2.5f), qm1_25 = v_setall_f32(-1.25f); - t00 = v_fma(x10, q0_5, x50 + x50); - t01 = v_fma(x11, q0_5, x51 + x51); + t00 = v_fma(x10, q0_5, v_add(x50, x50)); + t01 = v_fma(x11, q0_5, v_add(x51, x51)); t10 = v_fma(x20, q0_25, x60); t11 = v_fma(x21, q0_25, x61); t00 = v_fma(x30, qm2_5, t00); @@ -463,14 +463,14 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, t10 = v_fma(x40, qm1_25, t10); t11 = v_fma(x41, qm1_25, t11); - v_float32x4 y30 = t00 + t10, y31 = t01 + t11; - v_float32x4 y40 = t10 - t00, y41 = t11 - t01; + v_float32x4 y30 = v_add(t00, t10), y31 = v_add(t01, t11); + v_float32x4 y40 = v_sub(t10, t00), y41 = v_sub(t11, t01); /* Y[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*X */ /* Y[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*X */ v_float32x4 q4 = v_setall_f32(4.f), qm5 = v_setall_f32(-5.f); - t00 = v_fma(x50, q0_5, x10 + x10); - t01 = v_fma(x51, q0_5, x11 + x11); + t00 = v_fma(x50, q0_5, v_add(x10, x10)); + t01 = v_fma(x51, q0_5, v_add(x11, x11)); t10 = v_fma(x20, q4 , x60); t11 = v_fma(x21, q4 , x61); t00 = v_fma(x30, qm2_5, t00); @@ -478,8 +478,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, t10 = v_fma(x40, qm5 , t10); t11 = v_fma(x41, qm5 , t11); - v_float32x4 y50 = t00 + t10, y51 = t01 + t11; - v_float32x4 y60 = t10 - t00, y61 = t11 - t01; + v_float32x4 y50 = v_add(t00, t10), y51 = v_add(t01, t11); + v_float32x4 y60 = v_sub(t10, t00), y61 = v_sub(t11, t01); /* transpose 8x8 matrix with v_transpose4x4 */ @@ -491,29 +491,29 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, /* Z[0] = [1.f, 0.f, -5.25f, 0.f, 5.25f, 0.f, -1.f, 0.f]*Y */ /* Z[7] = [0.f, -1.f, 0.f, 5.25f, 0.f, -5.25f, 0.f, 1.f]*Y */ - t00 = y010 - y200; - t01 = y410 - y600; - t10 = y300 - y110; - t11 = y700 - y510; - z00 = v_fma(t00, q5_25, y000 - y210); - z01 = v_fma(t01, q5_25, y400 - y610); - z70 = v_fma(t10, q5_25, y310 - y100); - z71 = v_fma(t11, q5_25, y710 - y500); + t00 = v_sub(y010, y200); + t01 = v_sub(y410, y600); + t10 = v_sub(y300, y110); + t11 = v_sub(y700, y510); + z00 = v_fma(t00, q5_25, v_sub(y000, y210)); + z01 = v_fma(t01, q5_25, v_sub(y400, y610)); + z70 = v_fma(t10, q5_25, v_sub(y310, y100)); + z71 = v_fma(t11, q5_25, v_sub(y710, y500)); /* Z[1] = [0.f, 1.f, 1.f, -4.25f, -4.25f, 1.f, 1.f, 0.f]*Y */ /* Z[2] = [0.f, -1.f, 1.f, 4.25f, -4.25f, -1.f, 1.f, 0.f]*Y */ - t00 = v_fma(y300, qm4_25, y100 + y110); - t01 = v_fma(y700, qm4_25, y500 + y510); - t10 = v_fma(y010, qm4_25, y200 + y210); - t11 = v_fma(y410, qm4_25, y600 + y610); + t00 = v_fma(y300, qm4_25, v_add(y100, y110)); + t01 = v_fma(y700, qm4_25, v_add(y500, y510)); + t10 = v_fma(y010, qm4_25, v_add(y200, y210)); + t11 = v_fma(y410, qm4_25, v_add(y600, y610)); - z10 = t00 + t10; z11 = t01 + t11; - z20 = t10 - t00; z21 = t11 - t01; + z10 = v_add(t00, t10); z11 = v_add(t01, t11); + z20 = v_sub(t10, t00); z21 = v_sub(t11, t01); /* Z[3] = [0.f, 0.5f, 0.25f, -2.5f, -1.25f, 2.f, 1.f, 0.f]*Y */ /* Z[4] = [0.f, -0.5f, 0.25f, 2.5f, -1.25f, -2.f, 1.f, 0.f]*Y */ - t00 = v_fma(y100, q0_5, y110 + y110); - t01 = v_fma(y500, q0_5, y510 + y510); + t00 = v_fma(y100, q0_5, v_add(y110, y110)); + t01 = v_fma(y500, q0_5, v_add(y510, y510)); t10 = v_fma(y200, q0_25, y210); t11 = v_fma(y600, q0_25, y610); t00 = v_fma(y300, qm2_5, t00); @@ -521,13 +521,13 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, t10 = v_fma(y010, qm1_25, t10); t11 = v_fma(y410, qm1_25, t11); - z30 = t00 + t10; z31 = t01 + t11; - z40 = t10 - t00; z41 = t11 - t01; + z30 = v_add(t00, t10); z31 = v_add(t01, t11); + z40 = v_sub(t10, t00); z41 = v_sub(t11, t01); /* Z[5] = [0.f, 2.f, 4.f, -2.5f, -5.f, 0.5f, 1.f, 0.f]*Y */ /* Z[6] = [0.f, -2.f, 4.f, 2.5f, -5.f, -0.5f, 1.f, 0.f]*Y */ - t00 = v_fma(y110, q0_5, y100 + y100); - t01 = v_fma(y510, q0_5, y500 + y500); + t00 = v_fma(y110, q0_5, v_add(y100, y100)); + t01 = v_fma(y510, q0_5, v_add(y500, y500)); t10 = v_fma(y200, q4, y210); t11 = v_fma(y600, q4, y610); t00 = v_fma(y300, qm2_5, t00); @@ -535,8 +535,8 @@ void winofunc_BtXB_8x8_f32(const float* inptr, int inpstep, t10 = v_fma(y010, qm5, t10); t11 = v_fma(y410, qm5, t11); - z50 = t00 + t10; z51 = t01 + t11; - z60 = t10 - t00; z61 = t11 - t01; + z50 = v_add(t00, t10); z51 = v_add(t01, t11); + z60 = v_sub(t10, t00); z61 = v_sub(t11, t01); } const int outstep = winoIblock*winoAtomF32*Cg; @@ -601,12 +601,12 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, { v_float32x4 s12_0, s12_1, s34_0, s34_1, s56_0, s56_1; - s12_0 = x10 + x20; s12_1 = x11 + x21; - s34_0 = x30 + x40; s34_1 = x31 + x41; - s56_0 = x50 + x60; s56_1 = x51 + x61; + s12_0 = v_add(x10, x20); s12_1 = v_add(x11, x21); + s34_0 = v_add(x30, x40); s34_1 = v_add(x31, x41); + s56_0 = v_add(x50, x60); s56_1 = v_add(x51, x61); - v_float32x4 y00 = x00 + s12_0 + s34_0 + s56_0; - v_float32x4 y01 = x01 + s12_1 + s34_1 + s56_1; + v_float32x4 y00 = v_add(v_add(v_add(x00, s12_0), s34_0), s56_0); + v_float32x4 y01 = v_add(v_add(v_add(x01, s12_1), s34_1), s56_1); v_float32x4 a0 = v_setall_f32(0.25f), a1 = v_setall_f32(4.0f); v_float32x4 y20 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0)); @@ -616,13 +616,13 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, v_float32x4 y40 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0)); v_float32x4 y41 = v_fma(s56_1, a0, v_fma(s34_1, a1, s12_1)); - s12_0 = x10 - x20; s12_1 = x11 - x21; - s34_0 = x30 - x40; s34_1 = x31 - x41; - s56_0 = x50 - x60; s56_1 = x51 - x61; + s12_0 = v_sub(x10, x20); s12_1 = v_sub(x11, x21); + s34_0 = v_sub(x30, x40); s34_1 = v_sub(x31, x41); + s56_0 = v_sub(x50, x60); s56_1 = v_sub(x51, x61); a0 = v_setall_f32(1.f/32), a1 = v_setall_f32(32.f); - v_float32x4 y50 = v_fma(s56_0, a0, v_fma(s34_0, a1, x70 + s12_0)); - v_float32x4 y51 = v_fma(s56_1, a0, v_fma(s34_1, a1, x71 + s12_1)); + v_float32x4 y50 = v_fma(s56_0, a0, v_fma(s34_0, a1, v_add(x70, s12_0))); + v_float32x4 y51 = v_fma(s56_1, a0, v_fma(s34_1, a1, v_add(x71, s12_1))); a0 = v_setall_f32(0.5f), a1 = v_setall_f32(2.f); v_float32x4 y10 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0)); @@ -642,12 +642,12 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, v_transpose4x4(y40, y50, y60, y70, y400, y500, y600, y700); v_transpose4x4(y41, y51, y61, y71, y410, y510, y610, y710); - s12_0 = y100 + y200; s12_1 = y500 + y600; - s34_0 = y300 + y010; s34_1 = y700 + y410; - s56_0 = y110 + y210; s56_1 = y510 + y610; + s12_0 = v_add(y100, y200); s12_1 = v_add(y500, y600); + s34_0 = v_add(y300, y010); s34_1 = v_add(y700, y410); + s56_0 = v_add(y110, y210); s56_1 = v_add(y510, y610); - z00 = y000 + s12_0 + s34_0 + s56_0; - z01 = y400 + s12_1 + s34_1 + s56_1; + z00 = v_add(v_add(v_add(y000, s12_0), s34_0), s56_0); + z01 = v_add(v_add(v_add(y400, s12_1), s34_1), s56_1); a0 = v_setall_f32(0.25f), a1 = v_setall_f32(4.0f); z20 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0)); @@ -657,13 +657,13 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, z40 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0)); z41 = v_fma(s56_1, a0, v_fma(s34_1, a1, s12_1)); - s12_0 = y100 - y200; s12_1 = y500 - y600; - s34_0 = y300 - y010; s34_1 = y700 - y410; - s56_0 = y110 - y210; s56_1 = y510 - y610; + s12_0 = v_sub(y100, y200); s12_1 = v_sub(y500, y600); + s34_0 = v_sub(y300, y010); s34_1 = v_sub(y700, y410); + s56_0 = v_sub(y110, y210); s56_1 = v_sub(y510, y610); a0 = v_setall_f32(1.f/32), a1 = v_setall_f32(32.0f); - z50 = v_fma(s56_0, a0, v_fma(s34_0, a1, y310 + s12_0)); - z51 = v_fma(s56_1, a0, v_fma(s34_1, a1, y710 + s12_1)); + z50 = v_fma(s56_0, a0, v_fma(s34_0, a1, v_add(y310, s12_0))); + z51 = v_fma(s56_1, a0, v_fma(s34_1, a1, v_add(y710, s12_1))); a0 = v_setall_f32(0.5f), a1 = v_setall_f32(2.0f); z10 = v_fma(s56_0, a0, v_fma(s34_0, a1, s12_0)); z11 = v_fma(s56_1, a0, v_fma(s34_1, a1, s12_1)); @@ -673,34 +673,34 @@ void winofunc_AtXA_8x8_f32(const float* inptr, int inpstep, z31 = v_fma(s56_1, a0, v_fma(s34_1, a1, s12_1)); v_float32x4 vbias = v_setall_f32(bias); - z00 += vbias; - z01 += vbias; - z10 += vbias; - z11 += vbias; - z20 += vbias; - z21 += vbias; - z30 += vbias; - z31 += vbias; - z40 += vbias; - z41 += vbias; - z50 += vbias; - z51 += vbias; + z00 = v_add(z00, vbias); + z01 = v_add(z01, vbias); + z10 = v_add(z10, vbias); + z11 = v_add(z11, vbias); + z20 = v_add(z20, vbias); + z21 = v_add(z21, vbias); + z30 = v_add(z30, vbias); + z31 = v_add(z31, vbias); + z40 = v_add(z40, vbias); + z41 = v_add(z41, vbias); + z50 = v_add(z50, vbias); + z51 = v_add(z51, vbias); } if (bpptr) { - z00 += v_load(bpptr); - z01 += v_load_low(bpptr + 4); - z10 += v_load(bpptr + bpstep); - z11 += v_load_low(bpptr + bpstep + 4); - z20 += v_load(bpptr + bpstep*2); - z21 += v_load_low(bpptr + bpstep*2 + 4); - z30 += v_load(bpptr + bpstep*3); - z31 += v_load_low(bpptr + bpstep*3 + 4); - z40 += v_load(bpptr + bpstep*4); - z41 += v_load_low(bpptr + bpstep*4 + 4); - z50 += v_load(bpptr + bpstep*5); - z51 += v_load_low(bpptr + bpstep*5 + 4); + z00 = v_add(z00, v_load(bpptr)); + z01 = v_add(z01, v_load_low(bpptr + 4)); + z10 = v_add(z10, v_load(bpptr + bpstep)); + z11 = v_add(z11, v_load_low(bpptr + bpstep + 4)); + z20 = v_add(z20, v_load(bpptr + bpstep * 2)); + z21 = v_add(z21, v_load_low(bpptr + bpstep * 2 + 4)); + z30 = v_add(z30, v_load(bpptr + bpstep * 3)); + z31 = v_add(z31, v_load_low(bpptr + bpstep * 3 + 4)); + z40 = v_add(z40, v_load(bpptr + bpstep * 4)); + z41 = v_add(z41, v_load_low(bpptr + bpstep * 4 + 4)); + z50 = v_add(z50, v_load(bpptr + bpstep * 5)); + z51 = v_add(z51, v_load_low(bpptr + bpstep * 5 + 4)); } if (ifMinMaxAct) diff --git a/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp b/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp new file mode 100644 index 0000000000..60b503513f --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp @@ -0,0 +1,160 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../../precomp.hpp" +#include "fast_norm.hpp" + +namespace cv { namespace dnn { + +void fastNorm(const Mat &input, Mat &output, float epsilon, size_t normalized_axis, bool normalize_variance) { + const auto input_shape = shape(input); + CV_CheckLT(normalized_axis, input_shape.size(), "fastNorm: axis out of range"); + + size_t loops = static_cast(total(input_shape, 0, static_cast(normalized_axis))), + norm_size = static_cast(total(input_shape, static_cast(normalized_axis))); + float inv_norm_size = 1.0 / norm_size; + + auto fn = [&](const Range &r) { + const auto *input_data = input.ptr(); + auto *output_data = output.ptr(); + for (int i = r.start; i < r.end; i++) { + const auto *x = input_data + norm_size * i; + auto *y = output_data + norm_size * i; + + float mean = 0.f, mean_square = 0.f; + for (int j = 0; j < norm_size; j++) { + float v = x[j]; + mean += v; + mean_square += v * v; + } + + mean *= inv_norm_size; + mean_square = std::sqrt(std::max(0.f, mean_square * inv_norm_size - mean * mean) + epsilon); + float inv_stdev = normalize_variance ? 1.f / mean_square : 1.f; + + for (size_t j = 0; j < norm_size; j++) { + y[j] = (x[j] - mean) * inv_stdev; + } + } + }; + double nstripes = loops * norm_size * (1 / 1024.0); + parallel_for_(Range(0, loops), fn, nstripes); +} + +void fastNorm(const Mat &input, const Mat &scale, Mat &output, float epsilon, size_t normalized_axis) { + const auto input_shape = shape(input); + CV_CheckLT(normalized_axis, input_shape.size(), "fastNorm: axis out of range"); + + size_t loops = static_cast(total(input_shape, 0, static_cast(normalized_axis))), + norm_size = static_cast(total(input_shape, static_cast(normalized_axis))); + float inv_norm_size = 1.0 / norm_size; + + auto fn = [&](const Range &r) { + const auto *input_data = input.ptr(); + const auto *scale_data = scale.ptr(); + auto *output_data = output.ptr(); + for (int i = r.start; i < r.end; i++) { + const auto *x = input_data + norm_size * i; + auto *y = output_data + norm_size * i; + + float mean = 0.f, mean_square = 0.f; + for (int j = 0; j < norm_size; j++) { + float v = x[j]; + mean += v; + mean_square += v * v; + } + + mean *= inv_norm_size; + mean_square = std::sqrt(std::max(0.f, mean_square * inv_norm_size - mean * mean) + epsilon); + float inv_stdev = 1.f / mean_square; + + for (size_t j = 0; j < norm_size; j++) { + y[j] = scale_data[j] * (x[j] - mean) * inv_stdev; + } + } + }; + double nstripes = loops * norm_size * (1 / 1024.0); + parallel_for_(Range(0, loops), fn, nstripes); +} + +void fastNorm(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon, size_t normalized_axis) { + const auto input_shape = shape(input); + CV_CheckLT(normalized_axis, input_shape.size(), "fastNorm: axis out of range"); + CV_CheckEQ(scale.total(), bias.total(), "fastNorm: scale and bias should have the same shape"); + + size_t loops = static_cast(total(input_shape, 0, static_cast(normalized_axis))), + norm_size = static_cast(total(input_shape, static_cast(normalized_axis))); + float inv_norm_size = 1.0 / norm_size; + + auto fn = [&](const Range &r) { + const auto *input_data = input.ptr(); + const auto *scale_data = scale.ptr(); + const auto *bias_data = bias.ptr(); + auto *output_data = output.ptr(); + for (int i = r.start; i < r.end; i++) { + const auto *x = input_data + norm_size * i; + auto *y = output_data + norm_size * i; + + float mean = 0.f, mean_square = 0.f; + for (int j = 0; j < norm_size; j++) { + float v = x[j]; + mean += v; + mean_square += v * v; + } + + mean *= inv_norm_size; + mean_square = std::sqrt(std::max(0.f, mean_square * inv_norm_size - mean * mean) + epsilon); + float inv_stdev = 1.f / mean_square; + + for (size_t j = 0; j < norm_size; j++) { + y[j] = scale_data[j] * (x[j] - mean) * inv_stdev + bias_data[j]; + } + } + }; + double nstripes = loops * norm_size * (1 / 1024.0); + parallel_for_(Range(0, loops), fn, nstripes); +} + +void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon) { + const auto input_shape = shape(input); + CV_CheckEQ(scale.total(), bias.total(), "fastNormChannel: scale and bias should have the same shape"); + CV_CheckGE(input.dims, 3, "fastNormChannel: input dimension >= 3"); + + size_t N = input_shape[0], C = input_shape[1]; + size_t loops = N * C, + norm_size = static_cast(total(input_shape, 2)); + float inv_norm_size = 1.0 / norm_size; + + auto fn = [&](const Range &r) { + const auto *input_data = input.ptr(); + const auto *scale_data = scale.ptr(); + const auto *bias_data = bias.ptr(); + auto *output_data = output.ptr(); + for (int i = r.start; i < r.end; i++) { + const auto *x = input_data + norm_size * i; + auto *y = output_data + norm_size * i; + + float mean = 0.f, mean_square = 0.f; + for (int j = 0; j < norm_size; j++) { + float v = x[j]; + mean += v; + mean_square += v * v; + } + + mean *= inv_norm_size; + mean_square = std::sqrt(std::max(0.f, mean_square * inv_norm_size - mean * mean) + epsilon); + float inv_stdev = 1.f / mean_square; + + size_t c = i % C; + float s = scale_data[c], b = bias_data[c]; + for (size_t j = 0; j < norm_size; j++) { + y[j] = s * (x[j] - mean) * inv_stdev + b; + } + } + }; + double nstripes = loops * norm_size * (1 / 1024.0); + parallel_for_(Range(0, loops), fn, nstripes); +} + +}} // cv::dnn diff --git a/modules/dnn/src/layers/cpu_kernels/fast_norm.hpp b/modules/dnn/src/layers/cpu_kernels/fast_norm.hpp new file mode 100644 index 0000000000..61316542d3 --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/fast_norm.hpp @@ -0,0 +1,26 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_DNN_FAST_NORM_HPP +#define OPENCV_DNN_FAST_NORM_HPP + +#include + +namespace cv { namespace dnn { + +// Normalization speedup by multi-threading, mainly for Caffe MVN layer which has normalize_variance parameter. +void fastNorm(const Mat &input, Mat &output, float epsilon, size_t normalized_axis = 0, bool normalize_variance = true); + +// Normalization speedup by multi-threading with absent bias. Mainly for LayerNormalization. +void fastNorm(const Mat &input, const Mat &scale, Mat &output, float epsilon, size_t normalized_axis = 0); + +// Normalization speedup by multi-threading with scale and bias. Mainly for LayerNormalization. +void fastNorm(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon, size_t normalized_axis = 0); + +// Channel-wise Normalization speedup by multi-threading. Scale and bias should have the same shape (C). Input should have dimension >= 3. +void fastNormChannel(const Mat &input, const Mat &scale, const Mat &bias, Mat &output, float epsilon); + +}} // cv::dnn + +#endif // OPENCV_DNN_FAST_NORM_HPP diff --git a/modules/dnn/src/layers/cumsum_layer.cpp b/modules/dnn/src/layers/cumsum_layer.cpp index 0104b2d568..f94fa2750e 100644 --- a/modules/dnn/src/layers/cumsum_layer.cpp +++ b/modules/dnn/src/layers/cumsum_layer.cpp @@ -29,7 +29,7 @@ public: std::vector &internals) const CV_OVERRIDE { Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); - return true; + return exclusive_raw == 0; } void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE diff --git a/modules/dnn/src/layers/einsum_layer.cpp b/modules/dnn/src/layers/einsum_layer.cpp index 2cfb36da13..baf4297c0e 100644 --- a/modules/dnn/src/layers/einsum_layer.cpp +++ b/modules/dnn/src/layers/einsum_layer.cpp @@ -32,15 +32,14 @@ static bool IsTransposeReshapeForEinsum(const std::vector& perm, return true; } -Mat batchwiseMatMul( +static Mat batchwiseMatMul( const Mat& input1, const MatShape& input1ShapeOverride, const Mat& input2, const MatShape& input2ShapeOverride) { // Sanity checks before the actual MatMul - //input_1.DataType() == input_2.DataType(), "Data types of the inputs must match for MatMul"); - + CV_CheckType(input1.type(), input2.type(), "Data types of the inputs must match for MatMul"); CV_CheckEQ(input1ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul"); CV_CheckEQ(input2ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul"); CV_CheckEQ((size_t) input1ShapeOverride[0], (size_t) input2ShapeOverride[0], "Batch dimension should match for MatMul;"); @@ -51,8 +50,6 @@ Mat batchwiseMatMul( size_t K = input1ShapeOverride[2]; size_t N = input2ShapeOverride[2]; - //TODO: deal with dynamic shapes - //TODO: deal with reshaping operation (it might not always be needed) std::vector output; if (batches > 1) { @@ -141,26 +138,19 @@ Mat batchwiseMatMul( return output_buffer; }; -Mat Transpose( - const cv::Mat& input, +static Mat Transpose( + const Mat& input, const MatShape& input_shape_override, const std::vector permutation) { int input_rank = input_shape_override.size(); - CV_Assert(input_rank == permutation.size()); - // TODO: ouptimize - bool reshape = false; - if (input.dims != input_shape_override.size()) - { - reshape = true; - } + bool reshape = input.dims != input_rank; Mat input_reshaped; - if(reshape) - { + if(reshape){ input_reshaped = input.reshape(1, input_shape_override.size(), input_shape_override.data()); } @@ -170,13 +160,9 @@ Mat Transpose( outputDims.emplace_back(input_shape_override[dim]); Mat output; - // TODO: ouptimize - MatShape tmp_perm; - tmp_perm.reserve(permutation.size()); - for (int i = 0; i < permutation.size(); i++) - tmp_perm.emplace_back(static_cast(permutation[i])); + MatShape order(permutation.begin(), permutation.end()); - cv::transposeND((reshape ? input_reshaped : input), tmp_perm, output); + cv::transposeND((reshape ? input_reshaped : input), order, output); return output; } @@ -201,12 +187,183 @@ bool IsTransposeRequired(size_t input_rank, const std::vector& permutati return transpose_required; } -Mat Diagonal( - const cv::Mat& input, - int subscriptIndicesToInputIndex, - int dimIndexInIreprocessedInput) + +bool IsTransposeRequiredForDiagonal(int dim1, int dim2, int rank) { + // If the input is 2D, we don't need a transpose + if (rank == 2) + return false; + + // If the two dims are the innermost dims, no transpose is required + if ((dim1 == rank - 1 && dim2 == rank - 2) || + (dim1 == rank - 2 && dim2 == rank - 1)) + return false; + + // Transpose is required + return true; +} + +template +Mat DiagonalDataAssignment(Mat input) { + + int rank = input.dims; + CV_Assert(rank >= 2); + CV_Assert(input.size[rank - 1] == input.size[rank - 2]); + MatShape original_dims = shape(input); + + if (rank > 3){ + //reshape to 3D mat + int collapsed_size = 1; + for (int i = 0; i < rank - 2; ++i) { + collapsed_size *= input.size[i]; + } + std::vector reshaped_dims = {collapsed_size, input.size[rank - 2], input.size[rank - 1]}; + input = input.reshape(1, reshaped_dims); + } + + // Compute total number of higher-dimensional slices + int total_slices = input.size[0]; + + original_dims[rank - 1] = 1; // Set the last dimension to 1, as we have extracted the diagonal + Mat output = Mat(original_dims, input.type()); + + int inner_stride = input.size[input.dims - 1]; + auto inputPtr = input.ptr(); + auto outputPtr = output.ptr(); + for (int slice = 0; slice < total_slices; ++slice) { + for (int j = 0; j < inner_stride; ++j) { + // Direct memory access using raw pointers + outputPtr[slice * inner_stride + j] = inputPtr[slice * inner_stride * inner_stride + j * inner_stride + j]; + } + } + return output; +} + +/* Extract the diagonal elements from the last two dimensions of the tensor. +For instance, given an input_shape of [1, 2, 3, 3]: + +The flexibility in this implementation allows one to choose which of the two +last dimensions retains its value, determined by the `preserve_innermost_dim_val` parameter. + +When preserve_innermost_dim_val == true: + The resulting shape is [1, 2, 1, 3], indicating the diagonal has 3 elements, + and it keeps the dimension value of the innermost dimension. + +When preserve_innermost_dim_val == false: + The resulting shape is [1, 2, 3, 1], indicating the diagonal also has 3 elements, + but it retains the dimension value of the penultimate dimension. */ +Mat DiagonalInnermostDims(const Mat& input, bool preserve_innermost_dim_val) { + const MatShape input_dims = shape(input); + int rank = input_dims.size(); + + // This is an internal method and we already have finished all validations in the calling method. + // We proceed without duplicating all validations again here. + + // We have a minimalistic check here to make sure the innermost dims have the same dim value + // as the calling method may have done a transpose before calling this method + CV_CheckEQ(input.size[rank - 1], input.size[rank - 2], + "innermost dims should have the same dim value to parse the diagonal elements"); + + MatShape output_dims = input_dims; // Copy the original dims + if (preserve_innermost_dim_val) { + output_dims[rank - 2] = 1; + } else { + output_dims[rank - 1] = 1; + } + + // TODO: hande different types + Mat output = DiagonalDataAssignment(input); + + if (output_dims != shape(output)){ + CV_Error(Error::StsError, "Output shape does not match with calculated shape"); + } + return output; +} + +Mat Diagonal(const Mat& input, int dim1, int dim2) { - CV_Error(Error::StsNotImplemented, "Diagonal Not Implemented Yet"); + const MatShape input_dims = shape(input); + int rank = input_dims.size(); + + if (!(rank >= 2 && dim1 != dim2 && input_dims[dim1] == input_dims[dim2])){ + std::string input_dims_str = std::accumulate(std::next(input_dims.begin()), input_dims.end(), std::to_string(input_dims[0]), + [](const std::string& a, int b) { + return a + ' ' + std::to_string(b); + }); + CV_Error(Error::StsError, cv::format("Cannot parse the diagonal elements along dims %d and %d for input shape %s",dim1, dim2, input_dims_str.c_str())); + } + + int first_dim = std::min(dim1, dim2); + int second_dim = std::max(dim1, dim2); + + Mat output; + bool preserve_innermost_dim_val = false; + + bool is_transpose_required = IsTransposeRequiredForDiagonal(dim1, dim2, rank); + if (is_transpose_required) + { + std::vector permutation(rank, 0); + int first_dim_axis = -1; // This is the axis eventually occupied by the first_dim + + // If one of the diagonal dimensions is one of the 2 innermost dims, then leave it as such + // so as to avoid transpose overhead + if (first_dim == rank - 2) { // If rank - 2 is occupied by first_dim, keep it there + permutation[rank - 2] = first_dim; + first_dim_axis = rank - 2; + } else { + if (second_dim != rank - 2) { // If rank - 2 is not occupied by second_dim, then put first_dim there + permutation[rank - 2] = first_dim; + first_dim_axis = rank - 2; + } else { // If rank - 2 is occupied by second_dim, then put first_dim in rank - 1 + permutation[rank - 1] = first_dim; + first_dim_axis = rank - 1; + preserve_innermost_dim_val = true; // We always want to preserve the dim value of the first_dim + } + } + + // Put the second_dim in the dim not occupied by the first_dim + if (first_dim_axis != rank - 1) { + permutation[rank - 1] = second_dim; + } else { + permutation[rank - 2] = second_dim; + } + + size_t iter = 0; + for (int i = 0; i < rank; ++i) { + if (i != first_dim && i != second_dim) { + permutation[iter++] = i; + } + } + + // Permutate the input so that the dims from which we need the diagonal forms the innermost dims + Mat transposed = Transpose(input, input_dims, permutation); + + // Parse the diagonal from the innermost dims + output = DiagonalInnermostDims(transposed, preserve_innermost_dim_val); + + // Swap back the dimensions to the original axes ordering using a "reverse permutation" + // Find the "reverse" permutation + iter = 0; + std::vector reverse_permutation(rank, 0); + for (const auto& perm : permutation) { + reverse_permutation[perm] = iter++; + } + + // Permutate using the reverse permutation to get back the original axes ordering + // (Pass in CPU Transpose function here as this Diagonal method will only be used for CPU based diagonal parsing) + output = Transpose(output, shape(output), reverse_permutation); + } else { + // No transposing required + output = DiagonalInnermostDims(input, preserve_innermost_dim_val); + } + + // Make copy of the output dims + MatShape output_dims = shape(output); + + // Unsqueeze the reduced dim + auto iter = output_dims.begin() + second_dim; + output_dims.erase(iter); + output = output.reshape(1, output_dims); + return output; } /** @@ -299,7 +456,7 @@ public: void parseEquation(String equation); void processEquation(const std::vector& inputs); void processBroadcastedDims(); - void createOutputSubsctipt(); + void validateOutputSubscript(); void calculateOutputShape(); void preProcessInputs(InputArrayOfArrays& inputs); Mat reduceSum(Mat& src, MatShape& reduceAxis); @@ -358,7 +515,7 @@ public: processBroadcastedDims(); // calculate output shape - createOutputSubsctipt(); + validateOutputSubscript(); calculateOutputShape(); } @@ -624,7 +781,7 @@ void LayerEinsumImpl::calculateOutputShape() { // Traverse through each of the subscript labels within the output subscript. bool middleOfEllipsis = false; - // int64_t ellipsisCharCount = 0; + int ellipsisCharCount = 0; subscriptIndicesToOutputIndices.resize(numLetterIndices, -1); @@ -636,7 +793,21 @@ void LayerEinsumImpl::calculateOutputShape() { if(letter == '.') { - CV_Error(Error::StsNotImplemented, "Ellipsis are not supported yet"); + middleOfEllipsis = true; + // Make sure there aren't more than 3 '.'s in the current subscript + if (++ellipsisCharCount > 3) { + CV_Error(Error::StsError, "Found a '.' not part of an ellipsis in the output subscript provided"); + } + + if (ellipsisCharCount == 3) { // Ellipsis is complete. Process it. + middleOfEllipsis = false; + for (size_t i = 0; i < numOfEllipsisDims; ++i) { + einsumOutDims.emplace_back(subscriptIndicesToDimValue[i]); + // The ellipsis is seen in the output and hence the corresponding dims are to not be reduced + subscriptIndicesToLastInput[i] = -1; + subscriptIndicesToOutputIndices[i] = outputDimCounter++; + } + } } else { CV_CheckEQ(middleOfEllipsis, false, "Encountered '.' character that is not part of output subscript"); @@ -666,7 +837,7 @@ void LayerEinsumImpl::calculateOutputShape() } } -void LayerEinsumImpl::createOutputSubsctipt() +void LayerEinsumImpl::validateOutputSubscript() { // The explicit form requires no operation, as the output // would have already been parsed during the input parsing process. @@ -679,8 +850,6 @@ void LayerEinsumImpl::createOutputSubsctipt() { CV_Error(Error::StsError, "Provided output subscript does not include ellipsis while Inputs subscrits constain ellipsis"); - } else { - CV_Error(Error::StsNotImplemented, "Ellipsis are not yet supported"); } } } @@ -689,9 +858,84 @@ void LayerEinsumImpl::createOutputSubsctipt() void LayerEinsumImpl::processBroadcastedDims() { // Only compute this function if ellipsis "..." was found in the equation - if (numOfEllipsisDims > 0){ - // add assert inplace of return bool - CV_Error(Error::StsError, "Ellipsis are not supperted currenly"); + if (numOfEllipsisDims > 0) + { + // extend the number of subscript labels to include each ellipsis dim as + // theoretically each ellipsis dim does correspond to a "virtual" subscript label + numLetterIndices += numOfEllipsisDims; + + // We are going to assign the broadcasted dims outermost subscript indices (i.e.) 0 -> numOfEllipsisDims - 1 + // as most likely bradcasted dims will be batch dimensions (i.e.) outermost dimensions and hence we don't have to pay + // transposing while "homogenizing" the input + + // Hence offset all subscript indices by numOfEllipsisDims + for (size_t i = 0; i < numOfLetters; ++i){ + if (letter2count[i] != -1){ + letter2index[i] += numOfEllipsisDims; + } + } + + std::vector tempIndex2LastInput(numLetterIndices, -1); + for (int i = 0; i < subscriptIndicesToLastInput.size(); ++i){ + tempIndex2LastInput[i + numOfEllipsisDims] = subscriptIndicesToLastInput[i]; + } + subscriptIndicesToLastInput = std::move(tempIndex2LastInput); + + std::vector tempIndexToDimValue(numLetterIndices, -1); + for (int i = 0; i < subscriptIndicesToDimValue.size(); ++i){ + tempIndexToDimValue[i + numOfEllipsisDims] = subscriptIndicesToDimValue[i]; + } + subscriptIndicesToDimValue = std::move(tempIndexToDimValue); + + for (size_t i = 0; i < inputSubscriptIndices.size(); ++i) + { + auto& currentInputDimIndicesToSubscriptIndices = inputSubscriptIndices[i]; + std::vector tempCurrentInputDimIndicesToSubscriptIndices; + tempCurrentInputDimIndicesToSubscriptIndices.reserve(currentInputDimIndicesToSubscriptIndices.size()); + + // make sure it is correct + const auto& dims = einsumInpShapes[i]; + auto rank = dims.size(); + + size_t dimIter = 0; + size_t numBroadcastedIndices = 0; + while (dimIter < currentInputDimIndicesToSubscriptIndices.size()) + { + auto value = currentInputDimIndicesToSubscriptIndices[dimIter]; + if (value == numOfLetters) + { // This is a broadcasted dim + // Shouldn't hit this error - just a sanity check + CV_Assert(numBroadcastedIndices < numOfEllipsisDims); + tempCurrentInputDimIndicesToSubscriptIndices.push_back(static_cast(numBroadcastedIndices)); + subscriptIndicesToLastInput[numBroadcastedIndices] = i; + + // This is the first time we are seeing this broadcasted dim + if (subscriptIndicesToDimValue[numBroadcastedIndices] == -1) + { + subscriptIndicesToDimValue[numBroadcastedIndices] = dims[dimIter]; + } else { // We have seen this broadcasted dim before + // Check if the previous value is equal to the current value + if (subscriptIndicesToDimValue[numBroadcastedIndices] != dims[dimIter]) + { + // If they are not equal, one of them needs to be 1 + if (subscriptIndicesToDimValue[numBroadcastedIndices] == 1) + { + subscriptIndicesToDimValue[numBroadcastedIndices] = dims[dimIter]; + } else { + CV_CheckEQ(dims[dimIter], 1, "The broadcasted dimensions of the inputs are incompatible"); + } + } + } + ++numBroadcastedIndices; + } else { // This is a regular dim - offset it by number of broadcasted dims + tempCurrentInputDimIndicesToSubscriptIndices.push_back(value + static_cast(numOfEllipsisDims)); + } + ++dimIter; + } + // Shouldn't hit this error - just a sanity check + CV_Assert(dimIter == rank); + currentInputDimIndicesToSubscriptIndices = std::move(tempCurrentInputDimIndicesToSubscriptIndices); + } } } @@ -718,18 +962,58 @@ void LayerEinsumImpl::processEquation(const std::vector& inputs) // Variable to deal with "ellipsis" - '...' in the input bool middleOfellipsis = false; + int ellipsisCharCount = 0; for (auto letter : token) { - // Broadcasting based tokens are not implemented yet if (letter == '.') { - CV_Error(Error::StsNotImplemented, - "Broad casting based indices are not supported currently"); - } else - { + middleOfellipsis = true; - if (middleOfellipsis) + // there should not be more than 3 '.'s in the current subscript + if (++ellipsisCharCount > 3) { + CV_Error(Error::StsError, cv::format("Found a '.' not part of an ellipsis in input: %d", inputIdx)); + } + + // We have seen all 3 '.'s. We can safely process the ellipsis now. + if (ellipsisCharCount == 3) + { + middleOfellipsis = false; + + // Example for the following line of code + // Subscript "...ij" for an input of rank 6 + // numOfEllipsisDims = 6 - 5 + 3 = 4 + int currentNumOfEllipsisDims = static_cast(rank) - token.length() + 3; + CV_CheckGE(currentNumOfEllipsisDims, 0, + "Einsum subscripts string contains too many subscript labels when compared to the rank of the input"); + + // Theoretically, currentNumOfEllipsisDims could be 0 + // Example: For an input of rank 2 paired with a subscript "...ij" + if (currentNumOfEllipsisDims != 0) + { + // We have seen a ellipsis before - make sure ranks align as per the ONNX spec - + // "Ellipsis must indicate a fixed number of dimensions." + if (numOfEllipsisDims != 0){ + CV_CheckEQ(numOfEllipsisDims, static_cast(currentNumOfEllipsisDims), + "Ellipsis must indicate a fixed number of dimensions across all inputs"); + } else { + numOfEllipsisDims = static_cast(currentNumOfEllipsisDims); + } + + // We reserve 'numOfLetters' for broadcasted dims as we only allow 'a' - 'z' + // and 'A' - 'Z' (0 - 51) for non-broadcasted dims. + // We will assign appropriate indices (based on number of dimensions the ellipsis corresponds to) + // during broadcasting related post-processing. + for (size_t i = 0; i < numOfEllipsisDims; ++i){ + currTokenIndices.push_back(numOfLetters); + } + + // Offset 'dim_count' by number of dimensions the ellipsis corresponds to + dim_count += numOfEllipsisDims; + } + } + } else { + if (middleOfellipsis){ CV_Error(Error::StsAssert, cv::format( "Encountered '.' character that is not part of an ellipsis in the input: [%d]", @@ -744,8 +1028,7 @@ void LayerEinsumImpl::processEquation(const std::vector& inputs) // The subscript label was not found in the global subscript label array // Therefore, it is added to both the local and global subscript arrays - if(letter2count[letterIdx] == 0) - { + if(letter2count[letterIdx] == 0){ letter2index[letterIdx] = numLetterIndices++; subscriptIndicesToDimValue.push_back(dimValue); subscriptIndicesToLastInput.push_back(inputIdx); @@ -756,20 +1039,12 @@ void LayerEinsumImpl::processEquation(const std::vector& inputs) auto mappedIndx = letter2index[letterIdx]; subscriptIndicesToLastInput[mappedIndx] = inputIdx; - if (subscriptIndicesToDimValue[mappedIndx] != dimValue) - { - if(subscriptIndicesToDimValue[mappedIndx] == 1){ - //TODO: uncomment later on - // subscriptIndicesToDimValue[mappedIndx] == dimValue; - } else - { - if (dimValue != 1) - { - CV_Error(Error::StsError, cv::format("Einsum operands can not be broadcasted." - "Check input shapes/equation passed." - "Input shape of operand [%d]", inputIdx) + - cv::format(" is incompatible in the dimention [%zu].", static_cast(dim_count))); - } + if (subscriptIndicesToDimValue[mappedIndx] != dimValue) { + if (dimValue != 1) { + CV_Error(Error::StsError, cv::format("Einsum operands can not be broadcasted." + "Check input shapes/equation passed." + "Input shape of operand [%d]", inputIdx) + + cv::format(" is incompatible in the dimention [%zu].", static_cast(dim_count))); } } } diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 58c2cf3998..3d96a7bfc6 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -336,10 +336,10 @@ struct ReLUFunctor : public BaseFunctor v_float32x4 x1 = v_load(srcptr + i + 4); v_float32x4 x2 = v_load(srcptr + i + 8); v_float32x4 x3 = v_load(srcptr + i + 12); - x0 = v_select(x0 >= z, x0, x0*s4); - x1 = v_select(x1 >= z, x1, x1*s4); - x2 = v_select(x2 >= z, x2, x2*s4); - x3 = v_select(x3 >= z, x3, x3*s4); + x0 = v_select(v_ge(x0, z), x0, v_mul(x0, s4)); + x1 = v_select(v_ge(x1, z), x1, v_mul(x1, s4)); + x2 = v_select(v_ge(x2, z), x2, v_mul(x2, s4)); + x3 = v_select(v_ge(x3, z), x3, v_mul(x3, s4)); v_store(dstptr + i, x0); v_store(dstptr + i + 4, x1); v_store(dstptr + i + 8, x2); @@ -2288,10 +2288,10 @@ struct ChannelsPReLUFunctor : public BaseFunctor v_float32x4 x1 = v_load(srcptr + i + 4); v_float32x4 x2 = v_load(srcptr + i + 8); v_float32x4 x3 = v_load(srcptr + i + 12); - x0 = v_select(x0 >= z, x0, x0*s4); - x1 = v_select(x1 >= z, x1, x1*s4); - x2 = v_select(x2 >= z, x2, x2*s4); - x3 = v_select(x3 >= z, x3, x3*s4); + x0 = v_select(v_ge(x0, z), x0, v_mul(x0, s4)); + x1 = v_select(v_ge(x1, z), x1, v_mul(x1, s4)); + x2 = v_select(v_ge(x2, z), x2, v_mul(x2, s4)); + x3 = v_select(v_ge(x3, z), x3, v_mul(x3, s4)); v_store(dstptr + i, x0); v_store(dstptr + i + 4, x1); v_store(dstptr + i + 8, x2); @@ -2435,10 +2435,10 @@ struct PReLUFunctor : public ChannelsPReLUFunctor v_float32x4 s1 = v_load(scaleptr + i + 4); v_float32x4 s2 = v_load(scaleptr + i + 8); v_float32x4 s3 = v_load(scaleptr + i + 12); - x0 = v_select(x0 >= z, x0, x0*s0); - x1 = v_select(x1 >= z, x1, x1*s1); - x2 = v_select(x2 >= z, x2, x2*s2); - x3 = v_select(x3 >= z, x3, x3*s3); + x0 = v_select(v_ge(x0, z), x0, v_mul(x0, s0)); + x1 = v_select(v_ge(x1, z), x1, v_mul(x1, s1)); + x2 = v_select(v_ge(x2, z), x2, v_mul(x2, s2)); + x3 = v_select(v_ge(x3, z), x3, v_mul(x3, s3)); v_store(dstptr + i, x0); v_store(dstptr + i + 4, x1); v_store(dstptr + i + 8, x2); diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index 2f7e11f57e..1d655a50f6 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -306,7 +306,7 @@ public: } v_float32x4 s = v_reduce_sum4(vs0, vs1, vs2, vs3); - s += v_load(biasptr + i); + s = v_add(s, v_load(biasptr + i)); v_store(dptr + i, s); } #endif diff --git a/modules/dnn/src/layers/gather_elements_layer.cpp b/modules/dnn/src/layers/gather_elements_layer.cpp new file mode 100644 index 0000000000..c7bc43ea1a --- /dev/null +++ b/modules/dnn/src/layers/gather_elements_layer.cpp @@ -0,0 +1,160 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include + +namespace cv { namespace dnn { + +static inline int calculateOffset(int outer_dim, const MatShape &shape_indices, int axis_skip, const MatStep &step_data) { + int offset = 0; + for (int axis = static_cast(shape_indices.size()) - 2; axis >= 0; axis--) { + int dim = shape_indices[axis]; + if (axis != axis_skip) { + offset += (outer_dim % dim) * step_data[axis]; + } + outer_dim /= dim; + } + return offset; +} + +class GatherElementsLayerImpl CV_FINAL : public GatherElementsLayer +{ +public: + GatherElementsLayerImpl(const LayerParams& params) + { + setParamsFrom(params); + axis = params.get("axis", 0); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_CheckEQ(inputs.size(), 2ull, "GatherElements: requires two inputs"); + + const auto &data = inputs[0]; + const auto &indices = inputs[1]; + CV_CheckEQ(data.size(), indices.size(), "GatherElements: data and indices should have the same dimension"); + + int normalized_axis = normalize_axis(axis, static_cast(data.size())); + CV_CheckGE(normalized_axis, 0, "GatherElements: axis out of range"); + CV_CheckLT(normalized_axis, static_cast(data.size()), "GatherElements: axis out of range"); + for (size_t i = 0; i < data.size(); i++) { + if (i != normalized_axis) { + CV_CheckEQ(data[i], indices[i], "GatherElements: shape mismatched"); + } + } + + outputs.assign(1, inputs[1]); // shape of output is same as indices + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + const auto &data = inputs[0]; + axis = normalize_axis(axis, data.dims); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + const Mat& data = inputs[0]; + const Mat& indices = inputs[1]; + Mat& out = outputs[0]; + + typeDispatch(outputs[0].type(), data, indices, out); + } + + template + void forward_impl(const Mat& data_, const Mat& indices_, Mat& out_) + { + const auto *ptr_data = data_.ptr(); + const auto *ptr_indices = indices_.ptr(); + auto *ptr_out = out_.ptr(); + + const auto shape_data = shape(data_); + const auto &step_data = data_.step; + const auto shape_indices = shape(indices_); + + int inner_most_dim = shape_indices.back(); + int axis_dim = shape_data[axis]; + size_t axis_step = static_cast(step_data[axis] / sizeof(T)); + + bool innermost_axis = axis == static_cast(shape_data.size() - 1); + + auto fn = [&](const Range &r) { + for (int i = r.start; i < r.end; i++) { + auto *data = ptr_data + static_cast(calculateOffset(i, shape_indices, axis, step_data) / sizeof(T)); + auto *indices = ptr_indices + i * inner_most_dim; + auto *out = ptr_out + i * inner_most_dim; + + if (innermost_axis) { + for (int j = 0; j < inner_most_dim; j++) { + int index = static_cast((indices[j] + axis_dim)) % axis_dim; // TODO: Check out-of-range index + out[j] = data[index]; + } + } else { + for (int j = 0; j < inner_most_dim; j++) { + int index = static_cast(indices[j] + axis_dim) % axis_dim; // TODO: Check out-of-range index + out[j] = data[index * axis_step + j]; + } + } + } + }; + + int outer_dims = total(shape_indices, 0, shape_indices.size() - 1); + double nstripes = static_cast(outer_dims * inner_most_dim * (1 / 1024.0)); + parallel_for_(Range(0, outer_dims), fn, nstripes); + } + + template + inline void typeDispatch(const int type, Args&&... args) + { + switch (type) + { + case CV_8U: + forward_impl(std::forward(args)...); + break; + case CV_32S: + forward_impl(std::forward(args)...); + break; + case CV_32F: + forward_impl(std::forward(args)...); + break; + default: + CV_Error(cv::Error::BadDepth, "DNN/GatherElements: Unsupported type."); + }; + } + +private: + int axis; +}; + +Ptr GatherElementsLayer::create(const LayerParams& params) +{ + return makePtr(params); +} + +}} // namespace cv::dnn diff --git a/modules/dnn/src/layers/layer_norm.cpp b/modules/dnn/src/layers/layer_norm.cpp index a760766a3f..9c16d19e41 100644 --- a/modules/dnn/src/layers/layer_norm.cpp +++ b/modules/dnn/src/layers/layer_norm.cpp @@ -4,6 +4,7 @@ #include "../precomp.hpp" #include "layers_common.hpp" +#include "cpu_kernels/fast_norm.hpp" namespace cv { namespace dnn { @@ -15,11 +16,8 @@ public: setParamsFrom(params); // standard attr - axis = params.get("axis", 0); + axis = params.get("axis", -1); epsilon = params.get("epsilon", 1e-5); - - // opencv attr - hasBias = params.get("hasBias", false); } virtual bool supportBackend(int backendId) CV_OVERRIDE @@ -46,104 +44,25 @@ public: CV_CheckEQ(x_ndims - axis, w_ndims, "LayerNorm: shape of weight does not match with given axis and shape of input"); for (int i = 0; i < w_ndims; ++i) CV_CheckEQ(x_shape[axis+i], w_shape[i], "LayerNorm: weight dimensions does not match with input dimensions"); - if (hasBias) + if (inputs.size() == static_cast(3)) { - CV_CheckEQ(inputs.size(), (size_t)3, ""); auto b_shape = inputs[2]; CV_CheckEQ(w_shape.size(), b_shape.size(), "LayerNorm: shape of weight does not match with shape of bias"); for (size_t i = 0; i < w_shape.size(); ++i) CV_CheckEQ(w_shape[i], b_shape[i], "LayerNorm: bias dimensions does not match with weight dimensions"); } - // only one output is needed; Mean & InvStdDev are not needed - // in inference and should beomitted in onnx importer outputs.assign(1, inputs[0]); return false; } - template - class LayerNormInvoker : public ParallelLoopBody - { - public: - const Mat& src; - const float* scaleData; - const float* biasData; - Mat& dst; + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { + std::vector inputs; + inputs_arr.getMatVector(inputs); - float epsilon; - - int total; - int normSize; - float invNormSize; - - LayerNormInvoker(const Mat& src_, const Mat& scale, const Mat* b, Mat& dst_, int axis, float epsilon_) - : src(src_), scaleData(scale.ptr()), biasData(nullptr), dst(dst_), epsilon(epsilon_) - { - if (hasBias) - { - CV_Assert(b != nullptr); - CV_Assert(b->isContinuous()); - biasData = (const float*)b->ptr(); - } - - auto dstShape = shape(dst); - total = std::accumulate(dstShape.begin(), dstShape.begin() + axis, 1, std::multiplies()); - normSize = std::accumulate(dstShape.begin() + axis, dstShape.end(), 1, std::multiplies()); - invNormSize = 1.0f / normSize; - } - - static void run(const Mat& src, const Mat& scale, const Mat* b, Mat& dst, int axis, float epsilon) - { - CV_Assert(src.isContinuous()); - CV_Assert(dst.isContinuous()); - CV_CheckTypeEQ(src.type(), CV_32F, "DNN/LayerNorm: only support float32"); - CV_CheckTypeEQ(src.type(), dst.type(), ""); - CV_Assert(scale.isContinuous()); - - CV_CheckGE(epsilon, 0.0f, ""); - - LayerNormInvoker p(src, scale, b, dst, axis, epsilon); - - double nstripes = ((size_t)p.total * p.normSize) * (1 / 1024.0); - // double nstripes = ((size_t)p.total) * (1 / 1024.0); - parallel_for_(Range(0, p.total), p, nstripes); - } - - void operator()(const Range& r) const CV_OVERRIDE - { - int stripeStart = r.start; - int stripeEnd = r.end; - - const float* srcData = src.ptr(); - float* dstData = dst.ptr(); - - for (int ofs = stripeStart; ofs < stripeEnd; ++ofs) - { - const float* first = srcData + ofs * normSize; - float* dstFirst = dstData + ofs * normSize; - - float mean = 0; - float meanSquare = 0; - for (int h = 0; h < normSize; ++h) - { - float v = first[h]; - mean += v; - meanSquare += v * v; - } - mean *= invNormSize; - meanSquare = std::sqrt(std::max(0.f, meanSquare * invNormSize - mean * mean) + epsilon); - float invMeanSquare = 1.0f / meanSquare; - for (int h = 0; h < normSize; ++h) - { - float v = (first[h] - mean) * invMeanSquare * scaleData[h]; - if (hasBias) { - v = v + biasData[h]; - } - dstFirst[h] = v; - } - } - } - }; + const auto input_shape = shape(inputs[0]); + axis = normalize_axis(axis, static_cast(input_shape.size())); + } void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { @@ -160,10 +79,15 @@ public: inputs_arr.getMatVector(inputs); outputs_arr.getMatVector(outputs); - if (hasBias) { - LayerNormInvoker::run(inputs[0], inputs[1], &inputs[2], outputs[0], axis, epsilon); + const auto &input = inputs[0]; + const auto &scale = inputs[1]; + auto &output = outputs[0]; + + if (inputs.size() == 3) { + const auto &bias = inputs[2]; + fastNorm(input, scale, bias, output, epsilon, static_cast(axis)); } else { - LayerNormInvoker::run(inputs[0], inputs[1], nullptr, outputs[0], axis, epsilon); + fastNorm(input, scale, output, epsilon, static_cast(axis)); } } }; diff --git a/modules/dnn/src/layers/layers_common.cpp b/modules/dnn/src/layers/layers_common.cpp index b128872817..48401893f0 100644 --- a/modules/dnn/src/layers/layers_common.cpp +++ b/modules/dnn/src/layers/layers_common.cpp @@ -149,10 +149,11 @@ void getPoolingKernelParams(const LayerParams ¶ms, std::vector& kern std::vector& strides, cv::String &padMode) { bool is_global = params.get("global_pooling", false); - globalPooling.resize(3); - globalPooling[0] = params.get("global_pooling_d", is_global); - globalPooling[1] = params.get("global_pooling_h", is_global); - globalPooling[2] = params.get("global_pooling_w", is_global); + globalPooling.assign({ + params.get("global_pooling_d", is_global), + params.get("global_pooling_h", is_global), + params.get("global_pooling_w", is_global) + }); if (globalPooling[0] || globalPooling[1] || globalPooling[2]) { diff --git a/modules/dnn/src/layers/mvn_layer.cpp b/modules/dnn/src/layers/mvn_layer.cpp index aae53fa327..43c33d82af 100644 --- a/modules/dnn/src/layers/mvn_layer.cpp +++ b/modules/dnn/src/layers/mvn_layer.cpp @@ -46,6 +46,8 @@ #include "../ie_ngraph.hpp" #include "../op_cuda.hpp" +#include "./cpu_kernels/fast_norm.hpp" + #include #ifdef HAVE_OPENCL @@ -69,9 +71,12 @@ public: MVNLayerImpl(const LayerParams& params) { setParamsFrom(params); + + // Caffe params normVariance = params.get("normalize_variance", true); acrossChannels = params.get("across_channels", false); eps = params.get("eps", 1e-9); + fuse_batch_norm = false; fuse_relu = false; relu_slope = 0.f; @@ -310,73 +315,18 @@ public: return; } - std::vector inputs, outputs, internals; - inputs_arr.getMatVector(inputs); + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); // assume only one input outputs_arr.getMatVector(outputs); - internals_arr.getMatVector(internals); - for (size_t inpIdx = 0; inpIdx < inputs.size(); inpIdx++) - { - Mat &inpBlob = inputs[inpIdx]; - Mat &outBlob = outputs[inpIdx]; + const auto &input = inputs[0]; - int splitDim = (acrossChannels) ? 1 : 2; - int i, newRows = 1; - for( i = 0; i < splitDim; i++ ) - newRows *= inpBlob.size[i]; - - Mat inpMat = inpBlob.reshape(1, newRows); - Mat outMat = outBlob.reshape(1, newRows); - - if ( inpBlob.total() == newRows ) - { - // MVN is applied to single values at an every row. - if (shift.empty()) - { - outBlob.setTo(0); - } - else - { - for ( i = 0; i < newRows; i++ ) - { - outMat.row(i).setTo(((float*)shift.data)[i]); - } - } - return; - } - - Scalar mean, dev; - for ( i = 0; i < newRows; i++) - { - Mat inpRow = inpMat.row(i); - Mat outRow = outMat.row(i); - float weight = 1.f; - float bias = 0.f; - if (fuse_batch_norm) - { - weight = i < scale.cols ? ((float*)scale.data)[i] : weight; - bias = i < shift.cols ? ((float*)shift.data)[i] : bias; - } - cv::meanStdDev(inpRow, mean, (normVariance) ? dev : noArray()); - double alpha = 1; - if (normVariance) - { - alpha = 1 / std::sqrt(eps + dev[0]*dev[0]); - } - double normalizationScale = 1.0; - double normalizationShift = 0.0; - if (fuse_batch_norm) - { - normalizationScale = alpha * weight; - normalizationShift = -mean[0] * normalizationScale + bias; - } - else - { - normalizationScale = alpha; - normalizationShift = -mean[0] * alpha; - } - inpRow.convertTo(outRow, outRow.type(), normalizationScale, normalizationShift); - } + if (fuse_batch_norm) { // channel-wise scale/bias of shape (C) + CV_CheckTrue(normVariance, "DNN/MVN: not supported"); + fastNormChannel(input, scale, shift, outputs[0], eps); + } else { + size_t axis = acrossChannels ? 1 : 2; + fastNorm(input, outputs[0], eps, axis, normVariance); } } diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index 10902304e7..6d3158b575 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -873,25 +873,25 @@ public: v_float32x4 max_idx0 = v_setall_f32(-1.f); v_float32x4 max_idx1 = max_idx0; int index0 = ystart * inp_width + xstart; - v_float32x4 idx0 = idx00 + v_setall_f32((float)index0); - v_float32x4 idx1 = idx0 + v_setall_f32((float)(stride_w*4)); + v_float32x4 idx0 = v_add(idx00, v_setall_f32((float)index0)); + v_float32x4 idx1 = v_add(idx0, v_setall_f32((float)(stride_w * 4))); for (int y = ystart; y < yend; ++y) { - for (int x = xstart; x < xend; ++x, idx0 += ones, idx1 += ones) + for (int x = xstart; x < xend; ++x, idx0 = v_add(idx0, ones), idx1 = v_add(idx1, ones)) { const int index = y * inp_width + x; v_float32x4 v0(srcData[index], srcData[index + stride_w], srcData[index + stride_w*2], srcData[index + stride_w*3]); v_float32x4 v1(srcData[index + stride_w*4], srcData[index + stride_w*5], srcData[index + stride_w*6], srcData[index + stride_w*7]); - max_idx0 = v_select(v0 > max_val0, idx0, max_idx0); - max_idx1 = v_select(v1 > max_val1, idx1, max_idx1); + max_idx0 = v_select(v_gt(v0, max_val0), idx0, max_idx0); + max_idx1 = v_select(v_gt(v1, max_val1), idx1, max_idx1); max_val0 = v_max(max_val0, v0); max_val1 = v_max(max_val1, v1); } - idx0 += idx_delta; - idx1 += idx_delta; + idx0 = v_add(idx0, idx_delta); + idx1 = v_add(idx1, idx_delta); } v_store(dstData + x0, max_val0); v_store(dstData + x0 + 4, max_val1); @@ -1044,12 +1044,12 @@ public: srcData[index + stride_w*2], srcData[index + stride_w*3]); v_float32x4 v1(srcData[index + stride_w*4], srcData[index + stride_w*5], srcData[index + stride_w*6], srcData[index + stride_w*7]); - sum_val0 += v0; - sum_val1 += v1; + sum_val0 = v_add(sum_val0, v0); + sum_val1 = v_add(sum_val1, v1); } } - v_store(dstData + x0, sum_val0*ikarea); - v_store(dstData + x0 + 4, sum_val1*ikarea); + v_store(dstData + x0, v_mul(sum_val0, ikarea)); + v_store(dstData + x0 + 4, v_mul(sum_val1, ikarea)); x0 += 7; } else diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 9cec94c1fc..3031c599b1 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -179,6 +179,7 @@ private: void parseCast (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseConstantFill (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseGather (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGatherElements (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -2555,6 +2556,53 @@ void ONNXImporter::parseGather(LayerParams& layerParams, const opencv_onnx::Node addLayer(layerParams, node_proto); } +void ONNXImporter::parseGatherElements(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_CheckEQ(node_proto.input_size(), 2, "GatherElements: two inputs are required"); + + size_t num_const = 0; + for (size_t i = 0; i < node_proto.input_size(); ++i){ + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + ++num_const; + } + + if (num_const == node_proto.input_size()) + { + std::vector inputs, output; + for (size_t i = 0; i < node_proto.input_size(); i++) { + Mat blob = getBlob(node_proto, i); + if (i == 1) { // indices, from int32/int64 to float32 for compatibility + blob.convertTo(blob, CV_32F); + } + inputs.push_back(blob); + } + runLayer(layerParams, inputs, output); + CV_Assert(output.size() == 1); + addConstant(node_proto.output(0), output[0]); + return; + } else if (num_const > 0) { + for (size_t i = 0; i < node_proto.input_size(); i++) { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) { + Mat blob = getBlob(node_proto, i); + if (i == 1) { // indices, from int32/int64 to float32 for compatibility + blob.convertTo(blob, CV_32F); + } + + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "Const"; + constParams.blobs.push_back(blob); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } + + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { bool hasVariableInps = false; @@ -3113,12 +3161,6 @@ void ONNXImporter::parseLayerNorm(LayerParams& layerParams, const opencv_onnx::N axis = (axis + inputDims) % inputDims; layerParams.set("axis", axis); - // check if bias existed - bool hasBias = false; - if (node_proto.input_size() > 2) - hasBias = true; - layerParams.set("hasBias", hasBias); - // constants as constant inputs for (size_t i = 1; i < node_proto.input_size(); i++) { @@ -3902,6 +3944,7 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version) dispatch["Cast"] = &ONNXImporter::parseCast; dispatch["ConstantFill"] = dispatch["ConstantOfShape"] = &ONNXImporter::parseConstantFill; dispatch["Gather"] = &ONNXImporter::parseGather; + dispatch["GatherElements"] = &ONNXImporter::parseGatherElements; dispatch["Concat"] = &ONNXImporter::parseConcat; dispatch["Resize"] = &ONNXImporter::parseResize; dispatch["Upsample"] = &ONNXImporter::parseUpsample; diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 28744c586a..9ebd2a6b1d 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -260,6 +260,11 @@ const tensorflow::AttrValue& getLayerAttr(const tensorflow::NodeDef &layer, cons return layer.attr().at(name); } +#if defined(__GNUC__) && (__GNUC__ == 13) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdangling-reference" +#endif + static DataLayout getDataLayout(const tensorflow::NodeDef& layer) { if (hasLayerAttr(layer, "data_format")) @@ -2978,6 +2983,10 @@ static void addConstNodes(tensorflow::GraphDef& net, std::map& cons CV_LOG_DEBUG(NULL, "DNN/TF: layers_to_ignore.size() = " << layers_to_ignore.size()); } +#if defined(__GNUC__) && (__GNUC__ == 13) +#pragma GCC diagnostic pop +#endif + // If all inputs of specific layer have the same data layout we can say that // this layer's output has this data layout too. Returns DNN_LAYOUT_UNKNOWN otherwise. DataLayout TFImporter::predictOutputDataLayout(const tensorflow::NodeDef& layer) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 924fe6a823..f07b0c0e1b 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -455,6 +455,11 @@ TEST_P(DNNTestNetwork, FastNeuralStyle_eccv16) l1 = 0.4; lInf = 7.46; } + else if (target == DNN_TARGET_CUDA) + { + l1 = 7e-4; + lInf = 2e-2; + } else if (target == DNN_TARGET_CUDA_FP16) { l1 = 0.9; diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 536bc7dcc4..09a45718a4 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -764,11 +764,15 @@ TEST_F(Layer_RNN_Test, get_set_test) TEST_P(Test_Caffe_layers, Accum) { +#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF + throw SkipTestException("Requires patched protobuf"); +#else if (backend == DNN_BACKEND_OPENCV && target != DNN_TARGET_CPU) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); testLayerUsingCaffeModels("accum", false, false, 0.0, 0.0, 2); testLayerUsingCaffeModels("accum_ref", false, false, 0.0, 0.0, 2); +#endif } TEST_P(Test_Caffe_layers, FlowWarp) @@ -788,29 +792,41 @@ TEST_P(Test_Caffe_layers, ChannelNorm) TEST_P(Test_Caffe_layers, DataAugmentation) { +#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF + throw SkipTestException("Requires patched protobuf"); +#else if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); testLayerUsingCaffeModels("data_augmentation", true, false); testLayerUsingCaffeModels("data_augmentation_2x1", true, false); testLayerUsingCaffeModels("data_augmentation_8x6", true, false); +#endif } TEST_P(Test_Caffe_layers, Resample) { +#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF + throw SkipTestException("Requires patched protobuf"); +#else #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2023000000) if (backend != DNN_BACKEND_OPENCV) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); #endif testLayerUsingCaffeModels("nearest_2inps", false, false, 0.0, 0.0, 2); testLayerUsingCaffeModels("nearest", false, false); +#endif } TEST_P(Test_Caffe_layers, Correlation) { +#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF + throw SkipTestException("Requires patched protobuf"); +#else if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); testLayerUsingCaffeModels("correlation", false, false, 0.0, 0.0, 2); +#endif } TEST_P(Test_Caffe_layers, Convolution2Inputs) @@ -1651,12 +1667,11 @@ private: int outWidth, outHeight, zoomFactor; }; -#ifndef OPENCV_DNN_EXTERNAL_PROTOBUF TEST_P(Test_Caffe_layers, Interp) -#else -TEST_P(Test_Caffe_layers, DISABLED_Interp) // requires patched protobuf (available in OpenCV source tree only) -#endif { +#ifdef OPENCV_DNN_EXTERNAL_PROTOBUF + throw SkipTestException("Requires patched protobuf"); +#else #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception @@ -1680,6 +1695,7 @@ TEST_P(Test_Caffe_layers, DISABLED_Interp) // requires patched protobuf (availa // Test an implemented layer. testLayerUsingCaffeModels("layer_interp", false, false); +#endif } INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_Caffe_layers, dnnBackendsAndTargets()); diff --git a/modules/dnn/test/test_misc.cpp b/modules/dnn/test/test_misc.cpp index 2691df40c6..2ac683ad55 100644 --- a/modules/dnn/test/test_misc.cpp +++ b/modules/dnn/test/test_misc.cpp @@ -6,6 +6,7 @@ // Third party copyrights are property of their respective owners. #include "test_precomp.hpp" +#include "npy_blob.hpp" #include #include #include // CV_DNN_REGISTER_LAYER_CLASS @@ -869,6 +870,35 @@ TEST_P(Test_Model_Optimizer, flexible_inputs) normAssert(ref, out, 0, 0); } +TEST_P(Test_Model_Optimizer, readONNX) +{ + const Backend backendId = get<0>(GetParam()); + const Target targetId = get<1>(GetParam()); + + ASSERT_EQ(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, backendId); + + const std::string& model = findDataFile("dnn/onnx/models/convolution.onnx"); + + std::vector nets = { + // Old API + readNetFromModelOptimizer(model, ""), + readNet("", model, "dldt"), + // New API + readNetFromModelOptimizer(model), + readNet(model, "", "openvino") + }; + + Mat inp = blobFromNPY(findDataFile("dnn/onnx/data/input_convolution.npy")); + Mat ref = blobFromNPY(findDataFile("dnn/onnx/data/output_convolution.npy")); + + for (int i = 0; i < nets.size(); ++i) { + nets[i].setPreferableTarget(targetId); + nets[i].setInput(inp); + Mat out = nets[i].forward(); + normAssert(out, ref, format("Index: %d", i).c_str()); + } +} + INSTANTIATE_TEST_CASE_P(/**/, Test_Model_Optimizer, dnnBackendsAndTargetsIE() ); diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp index 6f8d7aef20..f87e16a42f 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__vulkan_denylist.inl.hpp @@ -55,6 +55,9 @@ "test_flatten_negative_axis1", "test_flatten_negative_axis2", "test_flatten_negative_axis4", +"test_gather_elements_0", +"test_gather_elements_1", +"test_gather_elements_negative_indices", "test_logsoftmax_default_axis", "test_maxpool_2d_dilations", "test_maxpool_2d_same_lower", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 00dd9191c5..52e709ebeb 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -103,7 +103,6 @@ "test_dynamicquantizelinear_min_adjusted", "test_dynamicquantizelinear_min_adjusted_expanded", "test_edge_pad", -"test_einsum_batch_diagonal", "test_einsum_inner_prod", "test_equal", "test_equal_bcast", @@ -115,9 +114,6 @@ "test_gather_0", "test_gather_1", "test_gather_2d_indices", -"test_gather_elements_0", -"test_gather_elements_1", -"test_gather_elements_negative_indices", "test_gather_negative_indices", "test_gathernd_example_float32", "test_gathernd_example_int32", diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 30bc9568df..525da26ed5 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -9,6 +9,7 @@ #include "test_precomp.hpp" #include "npy_blob.hpp" #include +#include namespace opencv_test { namespace { template @@ -791,6 +792,11 @@ TEST_P(Test_ONNX_layers, Concatenation) testONNXModels("concat_const_blobs"); } +TEST_P(Test_ONNX_layers, CumSumExclusiveInplace) +{ + testONNXModels("cumsum_exclusive_inplace"); +} + TEST_P(Test_ONNX_layers, Eltwise3D) { #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) @@ -1456,6 +1462,11 @@ TEST_P(Test_ONNX_layers, Einsum_2D) testONNXModels("einsum_2d", npy, 0, 0, false, false, 2); } +TEST_P(Test_ONNX_layers, Einsum_2D_Ellipses) +{ + testONNXModels("einsum_2d_ellipses", npy, 0, 0, false, false, 2); +} + TEST_P(Test_ONNX_layers, Einsum_3D) { testONNXModels("einsum_3d", npy, 0, 0, false, false, 2); @@ -1481,7 +1492,7 @@ TEST_P(Test_ONNX_layers, DISABLED_Einsum_HadamardProduct) testONNXModels("einsum_hadamard", npy, 0, 0, false, false, 2); } -TEST_P(Test_ONNX_layers, DISABLED_Einsum_Batch_Diagonal) +TEST_P(Test_ONNX_layers, Einsum_Batch_Diagonal) { testONNXModels("einsum_batch_diagonal", npy, 0, 0, false, false, 1); } @@ -2135,6 +2146,34 @@ TEST_P(Test_ONNX_nets, Alexnet) expectNoFallbacksFromIE(net); } +TEST_P(Test_ONNX_nets, RAFT) +{ + applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_DEBUG_VERYLONG, CV_TEST_TAG_MEMORY_2GB); + + std::string weight_path = _tf("models/optical_flow_estimation_raft_2023aug.onnx", false); + std::string img0_path = findDataFile(std::string("gpu/opticalflow/frame0.png")); + std::string img1_path = findDataFile(std::string("gpu/opticalflow/frame1.png")); + + Size target_size{480, 360}; + auto img0 = imread(img0_path); + auto img1 = imread(img1_path); + auto blob0 = blobFromImage(img0, 1.0, target_size, 0, true); + auto blob1 = blobFromImage(img1, 1.0, target_size, 0, true); + + auto net = readNet(weight_path); + net.setInput(blob0, "0"); + net.setInput(blob1, "1"); + std::vector outnames{"12007", "12006"}; + std::vector outs; + net.forward(outs, outnames); + + // output 12006 is not checked to save space in opencv_extra since its ref is > 1MB, + // and output 12006 is calculated from 12007 so checking 12007 is sufficient. + std::string ref_12700_path = _tf("data/output_optical_flow_estimation_raft_2023aug.npy"); + auto ref0 = blobFromNPY(ref_12700_path); + normAssert(ref0, outs[0], "", 1e-5, 1.8e-4); +} + TEST_P(Test_ONNX_nets, Squeezenet) { testONNXModels("squeezenet", pb); diff --git a/modules/features2d/src/fast.cpp b/modules/features2d/src/fast.cpp index 3e88f3e3f5..ef1fd9378c 100644 --- a/modules/features2d/src/fast.cpp +++ b/modules/features2d/src/fast.cpp @@ -120,8 +120,8 @@ void FAST_t(InputArray _img, std::vector& keypoints, int threshold, bo for (; j < img.cols - 16 - 3; j += 16, ptr += 16) { v_uint8x16 v = v_load(ptr); - v_int8x16 v0 = v_reinterpret_as_s8((v + t) ^ delta); - v_int8x16 v1 = v_reinterpret_as_s8((v - t) ^ delta); + v_int8x16 v0 = v_reinterpret_as_s8(v_xor(v_add(v, t), delta)); + v_int8x16 v1 = v_reinterpret_as_s8(v_xor(v_sub(v, t), delta)); v_int8x16 x0 = v_reinterpret_as_s8(v_sub_wrap(v_load(ptr + pixel[0]), delta)); v_int8x16 x1 = v_reinterpret_as_s8(v_sub_wrap(v_load(ptr + pixel[quarterPatternSize]), delta)); @@ -129,15 +129,15 @@ void FAST_t(InputArray _img, std::vector& keypoints, int threshold, bo v_int8x16 x3 = v_reinterpret_as_s8(v_sub_wrap(v_load(ptr + pixel[3*quarterPatternSize]), delta)); v_int8x16 m0, m1; - m0 = (v0 < x0) & (v0 < x1); - m1 = (x0 < v1) & (x1 < v1); - m0 = m0 | ((v0 < x1) & (v0 < x2)); - m1 = m1 | ((x1 < v1) & (x2 < v1)); - m0 = m0 | ((v0 < x2) & (v0 < x3)); - m1 = m1 | ((x2 < v1) & (x3 < v1)); - m0 = m0 | ((v0 < x3) & (v0 < x0)); - m1 = m1 | ((x3 < v1) & (x0 < v1)); - m0 = m0 | m1; + m0 = v_and(v_lt(v0, x0), v_lt(v0, x1)); + m1 = v_and(v_lt(x0, v1), v_lt(x1, v1)); + m0 = v_or(m0, v_and(v_lt(v0, x1), v_lt(v0, x2))); + m1 = v_or(m1, v_and(v_lt(x1, v1), v_lt(x2, v1))); + m0 = v_or(m0, v_and(v_lt(v0, x2), v_lt(v0, x3))); + m1 = v_or(m1, v_and(v_lt(x2, v1), v_lt(x3, v1))); + m0 = v_or(m0, v_and(v_lt(v0, x3), v_lt(v0, x0))); + m1 = v_or(m1, v_and(v_lt(x3, v1), v_lt(x0, v1))); + m0 = v_or(m0, m1); if( !v_check_any(m0) ) continue; @@ -154,18 +154,18 @@ void FAST_t(InputArray _img, std::vector& keypoints, int threshold, bo v_uint8x16 max1 = v_setzero_u8(); for( k = 0; k < N; k++ ) { - v_int8x16 x = v_reinterpret_as_s8(v_load((ptr + pixel[k])) ^ delta); - m0 = v0 < x; - m1 = x < v1; + v_int8x16 x = v_reinterpret_as_s8(v_xor(v_load((ptr + pixel[k])), delta)); + m0 = v_lt(v0, x); + m1 = v_lt(x, v1); - c0 = v_sub_wrap(c0, m0) & m0; - c1 = v_sub_wrap(c1, m1) & m1; + c0 = v_and(v_sub_wrap(c0, m0), m0); + c1 = v_and(v_sub_wrap(c1, m1), m1); max0 = v_max(max0, v_reinterpret_as_u8(c0)); max1 = v_max(max1, v_reinterpret_as_u8(c1)); } - max0 = K16 < v_max(max0, max1); + max0 = v_lt(K16, v_max(max0, max1)); unsigned int m = v_signmask(v_reinterpret_as_s8(max0)); for( k = 0; m > 0 && k < 16; k++, m >>= 1 ) @@ -190,7 +190,7 @@ void FAST_t(InputArray _img, std::vector& keypoints, int threshold, bo a1 = v_min(a1, v_nms); b1 = v_max(b1, v_nms); } - curr[j + k] = (uchar)(v_reduce_max(v_max(v_max(a0, a1), v_setzero_s16() - v_min(b0, b1))) - 1); + curr[j + k] = (uchar)(v_reduce_max(v_max(v_max(a0, a1), v_sub(v_setzero_s16(), v_min(b0, b1)))) - 1); } } } diff --git a/modules/features2d/src/fast_score.cpp b/modules/features2d/src/fast_score.cpp index 0bc011af49..0c43ad5552 100644 --- a/modules/features2d/src/fast_score.cpp +++ b/modules/features2d/src/fast_score.cpp @@ -160,7 +160,7 @@ int cornerScore<16>(const uchar* ptr, const int pixel[], int threshold) q0 = v_max(q0, v_min(a, v0)); q1 = v_min(q1, v_max(b, v0)); } - q0 = v_max(q0, v_setzero_s16() - q1); + q0 = v_max(q0, v_sub(v_setzero_s16(), q1)); threshold = v_reduce_max(q0) - 1; } else @@ -251,7 +251,7 @@ int cornerScore<12>(const uchar* ptr, const int pixel[], int threshold) q0 = v_max(q0, v_min(a, v0)); q1 = v_min(q1, v_max(b, v0)); } - q0 = v_max(q0, v_setzero_s16() - q1); + q0 = v_max(q0, v_sub(v_setzero_s16(), q1)); threshold = v_reduce_max(q0) - 1; } else @@ -323,7 +323,7 @@ int cornerScore<8>(const uchar* ptr, const int pixel[], int threshold) v0 = v_load(d + 5); q0 = v_max(q0, v_min(a, v0)); q1 = v_min(q1, v_max(b, v0)); - q0 = v_max(q0, v_setzero_s16() - q1); + q0 = v_max(q0, v_sub(v_setzero_s16(), q1)); threshold = v_reduce_max(q0) - 1; } else diff --git a/modules/gapi/cmake/DownloadADE.cmake b/modules/gapi/cmake/DownloadADE.cmake index e7a145b14a..871f99b419 100644 --- a/modules/gapi/cmake/DownloadADE.cmake +++ b/modules/gapi/cmake/DownloadADE.cmake @@ -1,7 +1,7 @@ set(ade_src_dir "${OpenCV_BINARY_DIR}/3rdparty/ade") -set(ade_filename "v0.1.2c.zip") -set(ade_subdir "ade-0.1.2c") -set(ade_md5 "4e996f545f9dddf2348cf593cbb2726f") +set(ade_filename "v0.1.2d.zip") +set(ade_subdir "ade-0.1.2d") +set(ade_md5 "dbb095a8bf3008e91edbbf45d8d34885") ocv_download(FILENAME ${ade_filename} HASH ${ade_md5} URL diff --git a/modules/gapi/src/backends/fluid/gfluidimgproc_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidimgproc_func.simd.hpp index 6c517b1f57..927f08d30a 100644 --- a/modules/gapi/src/backends/fluid/gfluidimgproc_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidimgproc_func.simd.hpp @@ -335,7 +335,7 @@ void run_rgb2hsv_impl(uchar out[], const uchar in[], const int sdiv_table[], // divide and calculate s according to above feature v_uint32x4 ss[4]; - v_uint32x4 vadd = v_setall_u32(1) << (hsv_shift - 1); + v_uint32x4 vadd = v_shl(v_setall_u32(1), (hsv_shift - 1)); v_uint32x4 v_diff_exp[4]; v_diff_exp[0] = v_reinterpret_as_u32(v_and(v_reinterpret_as_u8(v_diff), mask1)); @@ -406,16 +406,16 @@ void run_rgb2hsv_impl(uchar out[], const uchar in[], const int sdiv_table[], // start computing H-ch //h = (_vr & (g - b)) + (~_vr & ((_vg & (b - r + 2 * diff)) + ((~_vg) & (r - g + 4 * diff)))); v_int32x4 hh[4]; - hh[0] = v_reinterpret_as_s32(v_select(e[0], v_reinterpret_as_s32(gg[0] - bb[0]), + hh[0] = v_reinterpret_as_s32(v_select(e[0], v_reinterpret_as_s32(v_sub(gg[0], bb[0])), v_select(p[0], v_reinterpret_as_s32(v_add(v_sub(bb[0], rr[0]), v_mul(v_setall_u32(2), vdd[0]))), v_reinterpret_as_s32(v_add(v_sub(rr[0], gg[0]), v_mul(v_setall_u32(4), vdd[0])))))); - hh[1] = v_reinterpret_as_s32(v_select(e[1], v_reinterpret_as_s32(gg[1] - bb[1]), + hh[1] = v_reinterpret_as_s32(v_select(e[1], v_reinterpret_as_s32(v_sub(gg[1], bb[1])), v_select(p[1], v_reinterpret_as_s32(v_add(v_sub(bb[1], rr[1]), v_mul(v_setall_u32(2), vdd[1]))), v_reinterpret_as_s32(v_add(v_sub(rr[1], gg[1]), v_mul(v_setall_u32(4), vdd[1])))))); - hh[2] = v_reinterpret_as_s32(v_select(e[2], v_reinterpret_as_s32(gg[2] - bb[2]), + hh[2] = v_reinterpret_as_s32(v_select(e[2], v_reinterpret_as_s32(v_sub(gg[2], bb[2])), v_select(p[2], v_reinterpret_as_s32(v_add(v_sub(bb[2], rr[2]), v_mul(v_setall_u32(2), vdd[2]))), v_reinterpret_as_s32(v_add(v_sub(rr[2], gg[2]), v_mul(v_setall_u32(4), vdd[2])))))); - hh[3] = v_reinterpret_as_s32(v_select(e[3], v_reinterpret_as_s32(gg[3] - bb[3]), + hh[3] = v_reinterpret_as_s32(v_select(e[3], v_reinterpret_as_s32(v_sub(gg[3], bb[3])), v_select(p[3], v_reinterpret_as_s32(v_add(v_sub(bb[3], rr[3]), v_mul(v_setall_u32(2), vdd[3]))), v_reinterpret_as_s32(v_add(v_sub(rr[3], gg[3]), v_mul(v_setall_u32(4), vdd[3])))))); @@ -433,16 +433,16 @@ void run_rgb2hsv_impl(uchar out[], const uchar in[], const int sdiv_table[], // check for negative H v_int32x4 v_h_less_0[4]; - v_h_less_0[0] = (hh[0] < v_setall_s32(0)); - v_h_less_0[1] = (hh[1] < v_setall_s32(0)); - v_h_less_0[2] = (hh[2] < v_setall_s32(0)); - v_h_less_0[3] = (hh[3] < v_setall_s32(0)); + v_h_less_0[0] = (v_lt(hh[0], v_setall_s32(0))); + v_h_less_0[1] = (v_lt(hh[1], v_setall_s32(0))); + v_h_less_0[2] = (v_lt(hh[2], v_setall_s32(0))); + v_h_less_0[3] = (v_lt(hh[3], v_setall_s32(0))); v_int32x4 v_h_180[4]; - v_h_180[0] = hh[0] + v_setall_s32(180); - v_h_180[1] = hh[1] + v_setall_s32(180); - v_h_180[2] = hh[2] + v_setall_s32(180); - v_h_180[3] = hh[3] + v_setall_s32(180); + v_h_180[0] = v_add(hh[0], v_setall_s32(180)); + v_h_180[1] = v_add(hh[1], v_setall_s32(180)); + v_h_180[2] = v_add(hh[2], v_setall_s32(180)); + v_h_180[3] = v_add(hh[3], v_setall_s32(180)); hh[0] = v_select(v_h_less_0[0], v_h_180[0], hh[0]); hh[1] = v_select(v_h_less_0[1], v_h_180[1], hh[1]); diff --git a/modules/gapi/src/backends/fluid/gfluidimgproc_simd_avx2.hpp b/modules/gapi/src/backends/fluid/gfluidimgproc_simd_avx2.hpp index e246f0613b..f7a502f150 100644 --- a/modules/gapi/src/backends/fluid/gfluidimgproc_simd_avx2.hpp +++ b/modules/gapi/src/backends/fluid/gfluidimgproc_simd_avx2.hpp @@ -64,7 +64,7 @@ CV_ALWAYS_INLINE void calcRowLinear32FC1Impl(float *dst[], bool xRatioEq1 = inSz.width == outSz.width; bool yRatioEq1 = inSz.height == outSz.height; - constexpr int nlanes = v_float32x8::nlanes; + const int nlanes = VTraits::vlanes(); if (!xRatioEq1 && !yRatioEq1) { diff --git a/modules/gapi/src/compiler/passes/exec.cpp b/modules/gapi/src/compiler/passes/exec.cpp index 93d833d602..24dcef7bc1 100644 --- a/modules/gapi/src/compiler/passes/exec.cpp +++ b/modules/gapi/src/compiler/passes/exec.cpp @@ -142,6 +142,11 @@ namespace std::unordered_set cycle_causers; }; +#if defined(__GNUC__) && (__GNUC__ == 13) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdangling-reference" +#endif + bool canMerge(const GIslandModel::Graph &g, const ade::NodeHandle &a_nh, const ade::NodeHandle &slot_nh, @@ -191,6 +196,10 @@ namespace return true; } +#if defined(__GNUC__) && (__GNUC__ == 13) +#pragma GCC diagnostic pop +#endif + inline bool isProducedBy(const ade::NodeHandle &slot, const ade::NodeHandle &island) { diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.hpp b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp index 10171999a0..7e5e49b0e8 100644 --- a/modules/gapi/src/streaming/onevpl/file_data_provider.hpp +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp @@ -18,6 +18,14 @@ namespace cv { namespace gapi { namespace wip { namespace onevpl { + +// With gcc13, std::unique_ptr(FILE, decltype(&fclose)> causes ignored-attributes warning. +// See https://stackoverflow.com/questions/76849365/can-we-add-attributes-to-standard-function-declarations-without-breaking-standar +#if defined(__GNUC__) && (__GNUC__ == 13) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wignored-attributes" +#endif + struct GAPI_EXPORTS FileDataProvider : public IDataProvider { using file_ptr = std::unique_ptr; @@ -34,6 +42,11 @@ private: mfx_codec_id_type codec; const uint32_t bitstream_data_size; }; + +#if defined(__GNUC__) && (__GNUC__ == 13) +#pragma GCC diagnostic pop +#endif + } // namespace onevpl } // namespace wip } // namespace gapi diff --git a/modules/imgcodecs/include/opencv2/imgcodecs.hpp b/modules/imgcodecs/include/opencv2/imgcodecs.hpp index 2d3bc4e6f2..89bd6e1c1b 100644 --- a/modules/imgcodecs/include/opencv2/imgcodecs.hpp +++ b/modules/imgcodecs/include/opencv2/imgcodecs.hpp @@ -337,8 +337,9 @@ See cv::imreadmulti for the list of supported formats and flags description. @param buf Input array or vector of bytes. @param flags The same flags as in cv::imread, see cv::ImreadModes. @param mats A vector of Mat objects holding each page, if more than one. +@param range A continuous selection of pages. */ -CV_EXPORTS_W bool imdecodemulti(InputArray buf, int flags, CV_OUT std::vector& mats); +CV_EXPORTS_W bool imdecodemulti(InputArray buf, int flags, CV_OUT std::vector& mats, const cv::Range& range = Range::all()); /** @brief Encodes an image into a memory buffer. diff --git a/modules/imgcodecs/src/grfmt_avif.cpp b/modules/imgcodecs/src/grfmt_avif.cpp index b2830b755a..4b39ada60a 100644 --- a/modules/imgcodecs/src/grfmt_avif.cpp +++ b/modules/imgcodecs/src/grfmt_avif.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include "opencv2/imgproc.hpp" @@ -147,18 +148,6 @@ AvifDecoder::~AvifDecoder() { size_t AvifDecoder::signatureLength() const { return kAvifSignatureSize; } -bool AvifDecoder::checkSignature(const String &signature) const { - avifDecoder *decoder = avifDecoderCreate(); - if (!decoder) return false; - avifDecoderSetIOMemory(decoder, - reinterpret_cast(signature.c_str()), - signature.size()); - decoder->io->sizeHint = 1e9; - const avifResult status = avifDecoderParse(decoder); - avifDecoderDestroy(decoder); - return (status == AVIF_RESULT_OK || status == AVIF_RESULT_TRUNCATED_DATA); -} - #define OPENCV_AVIF_CHECK_STATUS(X, ENCDEC) \ { \ const avifResult status = (X); \ @@ -170,6 +159,20 @@ bool AvifDecoder::checkSignature(const String &signature) const { } \ } +bool AvifDecoder::checkSignature(const String &signature) const { + std::unique_ptr decoder( + avifDecoderCreate(), avifDecoderDestroy); + if (!decoder) return false; + OPENCV_AVIF_CHECK_STATUS( + avifDecoderSetIOMemory( + decoder.get(), reinterpret_cast(signature.c_str()), + signature.size()), + decoder); + decoder->io->sizeHint = 1e9; + const avifResult status = avifDecoderParse(decoder.get()); + return (status == AVIF_RESULT_OK || status == AVIF_RESULT_TRUNCATED_DATA); +} + ImageDecoder AvifDecoder::newDecoder() const { return makePtr(); } bool AvifDecoder::readHeader() { diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index 617463c339..692c924480 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -1077,12 +1077,21 @@ imdecodemulti_(const Mat& buf, int flags, std::vector& mats, int start, int return !mats.empty(); } -bool imdecodemulti(InputArray _buf, int flags, CV_OUT std::vector& mats) +bool imdecodemulti(InputArray _buf, int flags, CV_OUT std::vector& mats, const Range& range) { CV_TRACE_FUNCTION(); Mat buf = _buf.getMat(); - return imdecodemulti_(buf, flags, mats, 0, -1); + if (range == Range::all()) + { + return imdecodemulti_(buf, flags, mats, 0, -1); + } + else + { + CV_CheckGE(range.start, 0, "Range start cannot be negative."); + CV_CheckGT(range.size(), 0, "Range cannot be empty."); + return imdecodemulti_(buf, flags, mats, range.start, range.size()); + } } bool imencode( const String& ext, InputArray _image, diff --git a/modules/imgcodecs/test/test_tiff.cpp b/modules/imgcodecs/test/test_tiff.cpp index 82da0cdf42..db27a305e5 100644 --- a/modules/imgcodecs/test/test_tiff.cpp +++ b/modules/imgcodecs/test/test_tiff.cpp @@ -965,7 +965,7 @@ TEST_P(Imgcodecs_Tiff_Modes, decode_multipage) } } -TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer) +TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer_all_pages) { const int mode = GetParam(); const string root = cvtest::TS::ptr()->get_data_path(); @@ -984,13 +984,14 @@ TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer) FILE* fp = fopen(filename.c_str(), "rb"); ASSERT_TRUE(fp != NULL); fseek(fp, 0, SEEK_END); - long pos = ftell(fp); - - std::vector buf; - buf.resize((size_t)pos); + const size_t file_size = ftell(fp); fseek(fp, 0, SEEK_SET); - buf.resize(fread(&buf[0], 1, buf.size(), fp)); + + std::vector buf(file_size); + const size_t actual_read = fread(&buf[0], 1, file_size, fp); fclose(fp); + ASSERT_EQ(file_size, actual_read); + ASSERT_EQ(file_size, static_cast(buf.size())); bool res = imdecodemulti(buf, mode, pages); ASSERT_TRUE(res == true); @@ -1002,6 +1003,60 @@ TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer) } } +TEST_P(Imgcodecs_Tiff_Modes, decode_multipage_use_memory_buffer_selected_pages) +{ + const int mode = GetParam(); + const string root = cvtest::TS::ptr()->get_data_path(); + const string filename = root + "readwrite/multipage.tif"; + const string page_files[] = { + "readwrite/multipage_p1.tif", + "readwrite/multipage_p2.tif", + "readwrite/multipage_p3.tif", + "readwrite/multipage_p4.tif", + "readwrite/multipage_p5.tif", + "readwrite/multipage_p6.tif" + }; + const size_t page_count = sizeof(page_files) / sizeof(page_files[0]); + + FILE* fp = fopen(filename.c_str(), "rb"); + ASSERT_TRUE(fp != NULL); + fseek(fp, 0, SEEK_END); + const size_t file_size = ftell(fp); + fseek(fp, 0, SEEK_SET); + + std::vector buf(file_size); + const size_t actual_read = fread(&buf[0], 1, file_size, fp); + fclose(fp); + ASSERT_EQ(file_size, actual_read); + ASSERT_EQ(file_size, static_cast(buf.size())); + + const Range range(1, page_count - 1); + ASSERT_GE(range.size(), 1); + + vector middle_pages_from_imread; + for (int page_i = range.start; page_i < range.end; page_i++) + { + const Mat page = imread(root + page_files[page_i], mode); + middle_pages_from_imread.push_back(page); + } + ASSERT_EQ( + static_cast(range.size()), + static_cast(middle_pages_from_imread.size()) + ); + + vector middle_pages_from_imdecodemulti; + const bool res = imdecodemulti(buf, mode, middle_pages_from_imdecodemulti, range); + ASSERT_TRUE(res == true); + EXPECT_EQ(middle_pages_from_imread.size(), middle_pages_from_imdecodemulti.size()); + + for (int i = 0, e = range.size(); i < e; i++) + { + EXPECT_PRED_FORMAT2(cvtest::MatComparator(0, 0), + middle_pages_from_imread[i], + middle_pages_from_imdecodemulti[i]); + } +} + const int all_modes[] = { IMREAD_UNCHANGED, diff --git a/modules/imgproc/perf/perf_resize.cpp b/modules/imgproc/perf/perf_resize.cpp index a0ea0804cc..0f470a5f81 100644 --- a/modules/imgproc/perf/perf_resize.cpp +++ b/modules/imgproc/perf/perf_resize.cpp @@ -202,9 +202,9 @@ typedef TestBaseWithParam > MatInfo_Size_Scale_Area PERF_TEST_P(MatInfo_Size_Scale_Area, ResizeArea, testing::Combine( - testing::Values(CV_8UC1, CV_8UC4), - testing::Values(szVGA, szqHD, sz720p), - testing::Values(2.4, 3.4, 1.3) + testing::Values(CV_8UC1, CV_8UC3, CV_8UC4), + testing::Values(szVGA, szqHD, sz720p, sz1080p, sz2160p), + testing::Values(0.1, 0.25, 0.81) ) ) { diff --git a/modules/imgproc/src/bilateral_filter.simd.hpp b/modules/imgproc/src/bilateral_filter.simd.hpp index 332b36646c..77e0328678 100644 --- a/modules/imgproc/src/bilateral_filter.simd.hpp +++ b/modules/imgproc/src/bilateral_filter.simd.hpp @@ -140,9 +140,9 @@ public: #if CV_SIMD128 v_uint32x4 rval = v_setall_u32(sptr[j]); v_uint32x4 val(ksptr0[j], ksptr1[j], ksptr2[j], ksptr3[j]); - v_float32x4 w = kweight4 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); + v_float32x4 w = v_mul(kweight4, v_lut(this->color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); wsum[j] += v_reduce_sum(w); - sum[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(val)) * w); + sum[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(val)), w)); #else int rval = sptr[j]; @@ -407,11 +407,11 @@ public: v_uint32x4 b(ksptr0[0], ksptr1[0], ksptr2[0], ksptr3[0]); v_uint32x4 g(ksptr0[1], ksptr1[1], ksptr2[1], ksptr3[1]); v_uint32x4 r(ksptr0[2], ksptr1[2], ksptr2[2], ksptr3[2]); - v_float32x4 w = kweight4 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(b, rb) + v_absdiff(g, rg) + v_absdiff(r, rr))); + v_float32x4 w = v_mul(kweight4, v_lut(this->color_weight, v_reinterpret_as_s32(v_add(v_add(v_absdiff(b, rb), v_absdiff(g, rg)), v_absdiff(r, rr))))); wsum[j] += v_reduce_sum(w); - sum_b[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(b)) * w); - sum_g[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(g)) * w); - sum_r[j] += v_reduce_sum(v_cvt_f32(v_reinterpret_as_s32(r)) * w); + sum_b[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(b)), w)); + sum_g[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(g)), w)); + sum_r[j] += v_reduce_sum(v_mul(v_cvt_f32(v_reinterpret_as_s32(r)), w)); #else int rb = rsptr[0], rg = rsptr[1], rr = rsptr[2]; @@ -661,12 +661,12 @@ public: v_float32x4 rval = v_setall_f32(sptr[j]); v_float32x4 val(ksptr0[j], ksptr1[j], ksptr2[j], ksptr3[j]); v_float32x4 knan = v_not_nan(val); - v_float32x4 alpha = (v_absdiff(val, rval) * sindex4) & v_not_nan(rval) & knan; + v_float32x4 alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex4), v_not_nan(rval)), knan); v_int32x4 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - v_float32x4 w = (kweight4 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one4 - alpha))) & knan; + alpha = v_sub(alpha, v_cvt_f32(idx)); + v_float32x4 w = v_and(v_mul(kweight4, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one4, alpha)))), knan); wsum[j] += v_reduce_sum(w); - sum[j] += v_reduce_sum((val & knan) * w); + sum[j] += v_reduce_sum(v_mul(v_and(val, knan), w)); #else float rval = sptr[j]; @@ -862,15 +862,15 @@ public: v_float32x4 kb(ksptr0[0], ksptr1[0], ksptr2[0], ksptr3[0]); v_float32x4 kg(ksptr0[1], ksptr1[1], ksptr2[1], ksptr3[1]); v_float32x4 kr(ksptr0[2], ksptr1[2], ksptr2[2], ksptr3[2]); - v_float32x4 knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - v_float32x4 alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex4) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + v_float32x4 knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + v_float32x4 alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex4), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); v_int32x4 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - v_float32x4 w = (kweight4 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one4 - alpha))) & knan; + alpha = v_sub(alpha, v_cvt_f32(idx)); + v_float32x4 w = v_and(v_mul(kweight4, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one4, alpha)))), knan); wsum[j] += v_reduce_sum(w); - sum_b[j] += v_reduce_sum((kb & knan) * w); - sum_g[j] += v_reduce_sum((kg & knan) * w); - sum_r[j] += v_reduce_sum((kr & knan) * w); + sum_b[j] += v_reduce_sum(v_mul(v_and(kb, knan), w)); + sum_g[j] += v_reduce_sum(v_mul(v_and(kg, knan), w)); + sum_r[j] += v_reduce_sum(v_mul(v_and(kr, knan), w)); #else float rb = rsptr[0], rg = rsptr[1], rr = rsptr[2]; bool r_NAN = cvIsNaN(rb) || cvIsNaN(rg) || cvIsNaN(rr); diff --git a/modules/imgproc/src/box_filter.simd.hpp b/modules/imgproc/src/box_filter.simd.hpp index 735935c04f..f7c8f66a35 100644 --- a/modules/imgproc/src/box_filter.simd.hpp +++ b/modules/imgproc/src/box_filter.simd.hpp @@ -315,7 +315,7 @@ struct ColumnSum : v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } @@ -357,10 +357,10 @@ struct ColumnSum : } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); - for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); - v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits::vlanes()), v_load(Sp + i + VTraits::vlanes())); v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale))); v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale))); @@ -369,7 +369,7 @@ struct ColumnSum : v_pack_store(D + i, v_dst); v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); - v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits::vlanes()))); } #endif #endif @@ -396,16 +396,16 @@ struct ColumnSum : v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); - v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits::vlanes()), v_load(Sp + i + VTraits::vlanes())); v_uint16x8 v_dst = v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01)); v_pack_store(D + i, v_dst); v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); - v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits::vlanes()))); } #endif #endif @@ -486,7 +486,7 @@ public BaseColumnFilter v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width - v_uint16x8::nlanes; i += v_uint16x8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } @@ -546,13 +546,13 @@ public BaseColumnFilter v_uint32x4 ds4 = v_setall_u32((unsigned)ds); v_uint16x8 dd8 = v_setall_u16((ushort)dd); - for( ; i <= width-v_uint8x16::nlanes; i+=v_uint8x16::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_uint16x8 _sm0 = v_load(Sm + i); - v_uint16x8 _sm1 = v_load(Sm + i + v_uint16x8::nlanes); + v_uint16x8 _sm1 = v_load(Sm + i + VTraits::vlanes()); v_uint16x8 _s0 = v_add_wrap(v_load(SUM + i), v_load(Sp + i)); - v_uint16x8 _s1 = v_add_wrap(v_load(SUM + i + v_uint16x8::nlanes), v_load(Sp + i + v_uint16x8::nlanes)); + v_uint16x8 _s1 = v_add_wrap(v_load(SUM + i + VTraits::vlanes()), v_load(Sp + i + VTraits::vlanes())); v_uint32x4 _s00, _s01, _s10, _s11; @@ -572,7 +572,7 @@ public BaseColumnFilter v_store(D + i, v_pack_u(r0, r1)); v_store(SUM + i, _s0); - v_store(SUM + i + v_uint16x8::nlanes, _s1); + v_store(SUM + i + VTraits::vlanes(), _s1); } #endif #endif @@ -649,7 +649,7 @@ struct ColumnSum : v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } @@ -689,17 +689,17 @@ struct ColumnSum : } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); - for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); - v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits::vlanes()), v_load(Sp + i + VTraits::vlanes())); v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale)); v_int32x4 v_s01d = v_round(v_mul(v_cvt_f32(v_s01), v_scale)); v_store(D + i, v_pack(v_s0d, v_s01d)); v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); - v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits::vlanes()))); } #endif #endif @@ -725,15 +725,15 @@ struct ColumnSum : v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); - v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits::vlanes()), v_load(Sp + i + VTraits::vlanes())); v_store(D + i, v_pack(v_s0, v_s01)); v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); - v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits::vlanes()))); } #endif #endif @@ -798,7 +798,7 @@ struct ColumnSum : v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } @@ -838,17 +838,17 @@ struct ColumnSum : } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); - for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); - v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits::vlanes()), v_load(Sp + i + VTraits::vlanes())); v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale))); v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale))); v_store(D + i, v_pack(v_s0d, v_s01d)); v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); - v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits::vlanes()))); } #endif #endif @@ -874,15 +874,15 @@ struct ColumnSum : v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); - v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + VTraits::vlanes()), v_load(Sp + i + VTraits::vlanes())); v_store(D + i, v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01))); v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); - v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, v_load(Sm + i + VTraits::vlanes()))); } #endif #endif @@ -945,7 +945,7 @@ struct ColumnSum : v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } @@ -981,7 +981,7 @@ struct ColumnSum : } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); - for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale)); @@ -1010,7 +1010,7 @@ struct ColumnSum : v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); @@ -1079,7 +1079,7 @@ struct ColumnSum : v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } @@ -1115,7 +1115,7 @@ struct ColumnSum : } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); - for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); v_store(D + i, v_mul(v_cvt_f32(v_s0), v_scale)); @@ -1142,7 +1142,7 @@ struct ColumnSum : v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } #if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 - for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); v_store(D + i, v_cvt_f32(v_s0)); diff --git a/modules/imgproc/src/color_lab.cpp b/modules/imgproc/src/color_lab.cpp index d111efdc47..6a8a0ea7f9 100644 --- a/modules/imgproc/src/color_lab.cpp +++ b/modules/imgproc/src/color_lab.cpp @@ -66,7 +66,7 @@ template static inline cv::v_float32 splineInterpolate(const cv::v ix = v_shl<2>(ix); v_float32 t0, t1, t2, t3; - // assume that v_float32::nlanes == v_int32::nlanes + // assume that VTraits::vlanes() == VTraits::vlanes() if(VTraits::vlanes() == 4) { int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) idx[4]; @@ -1388,16 +1388,16 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin v_uint16x8& outA, v_uint16x8& outB, v_uint16x8& outC) { //LUT idx of origin pt of cube - v_uint16x8 idxsX = inX >> (lab_base_shift - lab_lut_shift); - v_uint16x8 idxsY = inY >> (lab_base_shift - lab_lut_shift); - v_uint16x8 idxsZ = inZ >> (lab_base_shift - lab_lut_shift); + v_uint16x8 idxsX = v_shr(inX); + v_uint16x8 idxsY = v_shr(inY); + v_uint16x8 idxsZ = v_shr(inZ); //x, y, z are [0; TRILINEAR_BASE) const uint16_t bitMask = (1 << trilinear_shift) - 1; v_uint16x8 bitMaskReg = v_setall_u16(bitMask); - v_uint16x8 fracX = (inX >> (lab_base_shift - 8 - 1)) & bitMaskReg; - v_uint16x8 fracY = (inY >> (lab_base_shift - 8 - 1)) & bitMaskReg; - v_uint16x8 fracZ = (inZ >> (lab_base_shift - 8 - 1)) & bitMaskReg; + v_uint16x8 fracX = v_and(v_shr(inX), bitMaskReg); + v_uint16x8 fracY = v_and(v_shr(inY), bitMaskReg); + v_uint16x8 fracZ = v_and(v_shr(inZ), bitMaskReg); //load values to interpolate for pix0, pix1, .., pix7 v_int16x8 a0, a1, a2, a3, a4, a5, a6, a7; @@ -1407,9 +1407,9 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin v_uint32x4 addrDw0, addrDw1, addrDw10, addrDw11; v_mul_expand(v_setall_u16(3*8), idxsX, addrDw0, addrDw1); v_mul_expand(v_setall_u16(3*8*LAB_LUT_DIM), idxsY, addrDw10, addrDw11); - addrDw0 += addrDw10; addrDw1 += addrDw11; + addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11); v_mul_expand(v_setall_u16(3*8*LAB_LUT_DIM*LAB_LUT_DIM), idxsZ, addrDw10, addrDw11); - addrDw0 += addrDw10; addrDw1 += addrDw11; + addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11); uint32_t CV_DECL_ALIGNED(16) addrofs[8]; v_store_aligned(addrofs, addrDw0); @@ -1431,9 +1431,9 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin v_int16x8 w0, w1, w2, w3, w4, w5, w6, w7; v_mul_expand(v_setall_u16(8), fracX, addrDw0, addrDw1); v_mul_expand(v_setall_u16(8*TRILINEAR_BASE), fracY, addrDw10, addrDw11); - addrDw0 += addrDw10; addrDw1 += addrDw11; + addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11); v_mul_expand(v_setall_u16(8*TRILINEAR_BASE*TRILINEAR_BASE), fracZ, addrDw10, addrDw11); - addrDw0 += addrDw10; addrDw1 += addrDw11; + addrDw0 = v_add(addrDw0, addrDw10); addrDw1 = v_add(addrDw1, addrDw11); v_store_aligned(addrofs, addrDw0); v_store_aligned(addrofs + 4, addrDw1); @@ -1476,7 +1476,8 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 const int16_t* LUT, v_uint16& outA, v_uint16& outB, v_uint16& outC) { - const int vsize = VTraits::max_nlanes; + const int vsize = VTraits::vlanes(); + const int vsize_max = VTraits::max_nlanes; // LUT idx of origin pt of cube v_uint16 tx = v_shr(inX); @@ -1492,7 +1493,7 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 baseIdx0 = v_add(v_add(btmp00, btmp10), btmp20); baseIdx1 = v_add(v_add(btmp01, btmp11), btmp21); - uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vbaseIdx[vsize]; + uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vbaseIdx[vsize_max]; v_store_aligned(vbaseIdx + 0*vsize/2, baseIdx0); v_store_aligned(vbaseIdx + 1*vsize/2, baseIdx1); @@ -1513,13 +1514,13 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 trilinearIdx0 = v_add(v_add(v_shl<3>(fracX0), v_shl<3 + trilinear_shift>(fracY0)), v_shl<3 + trilinear_shift * 2>(fracZ0)); trilinearIdx1 = v_add(v_add(v_shl<3>(fracX1), v_shl<3 + trilinear_shift>(fracY1)), v_shl<3 + trilinear_shift * 2>(fracZ1)); - uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vtrilinearIdx[vsize]; + uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vtrilinearIdx[vsize_max]; v_store_aligned(vtrilinearIdx + 0*vsize/2, trilinearIdx0); v_store_aligned(vtrilinearIdx + 1*vsize/2, trilinearIdx1); v_uint32 a0, a1, b0, b1, c0, c1; - uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) va[vsize], vb[vsize], vc[vsize]; + uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) va[vsize_max], vb[vsize_max], vc[vsize_max]; for(int j = 0; j < vsize; j++) { const int16_t* baseLUT = LUT + vbaseIdx[j]; @@ -1649,11 +1650,11 @@ struct RGB2Lab_b vL = v_shr(vL); /* int a = CV_DESCALE( 500*(fX - fY) + 128*(1 << lab_shift2), lab_shift2 );*/ - va = v_fma(vfX - vfY, v_setall_s32(500), v_setall_s32(abShift+labDescaleShift)); + va = v_fma(v_sub(vfX, vfY), v_setall_s32(500), v_setall_s32(abShift+labDescaleShift)); va = v_shr(va); /* int b = CV_DESCALE( 200*(fY - fZ) + 128*(1 << lab_shift2), lab_shift2 );*/ - vb = v_fma(vfY - vfZ, v_setall_s32(200), v_setall_s32(abShift+labDescaleShift)); + vb = v_fma(v_sub(vfY, vfZ), v_setall_s32(200), v_setall_s32(abShift+labDescaleShift)); vb = v_shr(vb); } #endif // CV_NEON @@ -1675,8 +1676,8 @@ struct RGB2Lab_b #if CV_NEON // On each loop, we load nlanes of RGB/A v_uint8s and store nlanes of // Lab v_uint8s - for(; i <= n - v_uint8::nlanes; i += v_uint8::nlanes, - src += scn*v_uint8::nlanes, dst += 3*v_uint8::nlanes ) + for(; i <= n - VTraits::vlanes(); i += VTraits::vlanes(), + src += scn*VTraits::vlanes(), dst += 3*VTraits::vlanes() ) { // Load 4 batches of 4 src v_uint8 vRi, vGi, vBi; @@ -1712,7 +1713,7 @@ struct RGB2Lab_b #endif // CV_NEON #if CV_SIMD - const int vsize = v_uint8::nlanes; + const int vsize = VTraits::vlanes(); const int xyzDescaleShift = 1 << (lab_shift - 1); v_int16 vXYZdescale = vx_setall_s16(xyzDescaleShift); v_int16 cxrg, cxb1, cyrg, cyb1, czrg, czb1; @@ -1752,7 +1753,7 @@ struct RGB2Lab_b v_expand(drgb[k], qrgb[k*2+0], qrgb[k*2+1]); } - uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vdrgb[vsize*3]; + uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vdrgb[VTraits::max_nlanes*3]; for(int k = 0; k < 12; k++) { v_store_aligned(vdrgb + k*vsize/4, qrgb[k]); @@ -1784,14 +1785,14 @@ struct RGB2Lab_b v_uint32 x[4], y[4], z[4]; for(int j = 0; j < 4; j++) { - x[j] = v_reinterpret_as_u32(v_dotprod(rg[j], cxrg) + v_dotprod(bd[j], cxb1)) >> lab_shift; - y[j] = v_reinterpret_as_u32(v_dotprod(rg[j], cyrg) + v_dotprod(bd[j], cyb1)) >> lab_shift; - z[j] = v_reinterpret_as_u32(v_dotprod(rg[j], czrg) + v_dotprod(bd[j], czb1)) >> lab_shift; + x[j] = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], cxrg), v_dotprod(bd[j], cxb1)))); + y[j] = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], cyrg), v_dotprod(bd[j], cyb1)))); + z[j] = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(rg[j], czrg), v_dotprod(bd[j], czb1)))); } // [fX, fY, fZ] = LabCbrtTab_b[vx, vy, vz] // [4 per X, 4 per Y, 4 per Z] - uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vxyz[vsize*3]; + uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vxyz[VTraits::max_nlanes*3]; for(int j = 0; j < 4; j++) { v_store_aligned(vxyz + (0*4+j)*vsize/4, x[j]); @@ -1822,7 +1823,7 @@ struct RGB2Lab_b v_uint32 vLshift = vx_setall_u32((uint32_t)(Lshift + labDescaleShift)); for(int k = 0; k < 4; k++) { - vL[k] = (vL[k] + vLshift) >> lab_shift2; + vL[k] = v_shr(v_add(vL[k], vLshift)); } v_uint16 L0, L1; L0 = v_pack(vL[0], vL[1]); @@ -1846,7 +1847,7 @@ struct RGB2Lab_b v_int32 abShift = vx_setall_s32(128*(1 << lab_shift2) + labDescaleShift); for(int k = 0; k < 8; k++) { - ab[k] = (ab[k] + abShift) >> lab_shift2; + ab[k] = v_shr(v_add(ab[k], abShift)); } v_int16 a0, a1, b0, b1; a0 = v_pack(ab[0], ab[1]); a1 = v_pack(ab[2], ab[3]); @@ -1941,7 +1942,7 @@ struct RGB2Lab_f #if CV_SIMD if(enablePackedLab) { - const int vsize = v_float32::nlanes; + const int vsize = VTraits::vlanes(); static const int nPixels = vsize*2; for(; i < n - 3*nPixels; i += 3*nPixels, src += scn*nPixels) { @@ -1973,8 +1974,8 @@ struct RGB2Lab_f #undef clipv /* int iR = R*LAB_BASE, iG = G*LAB_BASE, iB = B*LAB_BASE, iL, ia, ib; */ v_float32 basef = vx_setall_f32(LAB_BASE); - rvec0 *= basef, gvec0 *= basef, bvec0 *= basef; - rvec1 *= basef, gvec1 *= basef, bvec1 *= basef; + rvec0 = v_mul(rvec0, basef), gvec0 = v_mul(gvec0, basef), bvec0 = v_mul(bvec0, basef); + rvec1 = v_mul(rvec1, basef), gvec1 = v_mul(gvec1, basef), bvec1 = v_mul(bvec1, basef); v_int32 irvec0, igvec0, ibvec0, irvec1, igvec1, ibvec1; irvec0 = v_round(rvec0); irvec1 = v_round(rvec1); @@ -2004,8 +2005,8 @@ struct RGB2Lab_f /* dst[i] = L*100.0f */ v_float32 v100dBase = vx_setall_f32(100.0f/LAB_BASE); - l_vec0 = l_vec0*v100dBase; - l_vec1 = l_vec1*v100dBase; + l_vec0 = v_mul(l_vec0, v100dBase); + l_vec1 = v_mul(l_vec1, v100dBase); /* dst[i + 1] = a*256.0f - 128.0f; dst[i + 2] = b*256.0f - 128.0f; @@ -2043,8 +2044,8 @@ struct RGB2Lab_f static const float _a = (softfloat(16) / softfloat(116)); int i = 0; #if CV_SIMD - const int vsize = v_float32::nlanes; - const int nrepeats = vsize == 4 ? 2 : 1; + const int vsize = VTraits::vlanes(); + const int nrepeats = VTraits::nlanes == 4 ? 2 : 1; v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2); v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4), vc5 = vx_setall_f32(C5); v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8); @@ -2080,9 +2081,9 @@ struct RGB2Lab_f v_float32 vgscale = vx_setall_f32(gscale); for (int k = 0; k < nrepeats; k++) { - R[k] = splineInterpolate(R[k]*vgscale, gammaTab, GAMMA_TAB_SIZE); - G[k] = splineInterpolate(G[k]*vgscale, gammaTab, GAMMA_TAB_SIZE); - B[k] = splineInterpolate(B[k]*vgscale, gammaTab, GAMMA_TAB_SIZE); + R[k] = splineInterpolate(v_mul(R[k], vgscale), gammaTab, GAMMA_TAB_SIZE); + G[k] = splineInterpolate(v_mul(G[k], vgscale), gammaTab, GAMMA_TAB_SIZE); + B[k] = splineInterpolate(v_mul(B[k], vgscale), gammaTab, GAMMA_TAB_SIZE); } } @@ -2090,26 +2091,26 @@ struct RGB2Lab_f v_float32 FX[nrepeats], FY[nrepeats], FZ[nrepeats]; for (int k = 0; k < nrepeats; k++) { - X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, B[k]*vc2)); - Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, B[k]*vc5)); - Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, B[k]*vc8)); + X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, v_mul(B[k], vc2))); + Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, v_mul(B[k], vc5))); + Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, v_mul(B[k], vc8))); // use spline interpolation instead of direct calculation v_float32 vTabScale = vx_setall_f32(LabCbrtTabScale); - FX[k] = splineInterpolate(X[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE); - FY[k] = splineInterpolate(Y[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE); - FZ[k] = splineInterpolate(Z[k]*vTabScale, LabCbrtTab, LAB_CBRT_TAB_SIZE); + FX[k] = splineInterpolate(v_mul(X[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE); + FY[k] = splineInterpolate(v_mul(Y[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE); + FZ[k] = splineInterpolate(v_mul(Z[k], vTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE); } v_float32 L[nrepeats], a[nrepeats], b[nrepeats]; for (int k = 0; k < nrepeats; k++) { // 7.787f = (29/3)^3/(29*4), 0.008856f = (6/29)^3, 903.3 = (29/3)^3 - v_float32 mask = Y[k] > (vx_setall_f32(0.008856f)); + v_float32 mask = v_gt(Y[k], (vx_setall_f32(0.008856f))); v_float32 v116 = vx_setall_f32(116.f), vm16 = vx_setall_f32(-16.f); - L[k] = v_select(mask, v_fma(v116, FY[k], vm16), vx_setall_f32(903.3f)*Y[k]); - a[k] = vx_setall_f32(500.f) * (FX[k] - FY[k]); - b[k] = vx_setall_f32(200.f) * (FY[k] - FZ[k]); + L[k] = v_select(mask, v_fma(v116, FY[k], vm16), v_mul(vx_setall_f32(903.3f),Y[k])); + a[k] = v_mul(vx_setall_f32(500.F), v_sub(FX[k], FY[k])); + b[k] = v_mul(vx_setall_f32(200.F), v_sub(FY[k], FZ[k])); v_store_interleave(dst + k*3*vsize, L[k], a[k], b[k]); } @@ -2204,7 +2205,7 @@ struct Lab2RGBfloat float alpha = ColorChannel::max(); #if CV_SIMD - const int vsize = v_float32::nlanes; + const int vsize = VTraits::vlanes(); const int nrepeats = 2; v_float32 v16_116 = vx_setall_f32(16.0f / 116.0f); for( ; i <= n-vsize*nrepeats; @@ -2221,14 +2222,14 @@ struct Lab2RGBfloat v_float32 vlThresh = vx_setall_f32(lThresh); for(int k = 0; k < nrepeats; k++) { - limask[k] = li[k] <= vlThresh; + limask[k] = v_le(li[k], vlThresh); } v_float32 ylo[nrepeats], yhi[nrepeats], fylo[nrepeats], fyhi[nrepeats]; // 903.3 = (29/3)^3, 7.787 = (29/3)^3/(29*4) v_float32 vinv903 = vx_setall_f32(1.f/903.3f); for(int k = 0; k < nrepeats; k++) { - ylo[k] = li[k] * vinv903; + ylo[k] = v_mul(li[k], vinv903); } v_float32 v7787 = vx_setall_f32(7.787f); for(int k = 0; k < nrepeats; k++) @@ -2238,11 +2239,11 @@ struct Lab2RGBfloat v_float32 v16 = vx_setall_f32(16.0f), vinv116 = vx_setall_f32(1.f/116.0f); for(int k = 0; k < nrepeats; k++) { - fyhi[k] = (li[k] + v16) * vinv116; + fyhi[k] = v_mul(v_add(li[k], v16), vinv116); } for(int k = 0; k < nrepeats; k++) { - yhi[k] = fyhi[k] * fyhi[k] * fyhi[k]; + yhi[k] = v_mul(fyhi[k], fyhi[k], fyhi[k]); } for(int k = 0; k < nrepeats; k++) { @@ -2265,9 +2266,9 @@ struct Lab2RGBfloat for (int j = 0; j < 2; j++) { v_float32 f = fxz[k*2+j]; - v_float32 fmask = f <= vfTresh; - v_float32 flo = (f - v16_116) * vinv7787; - v_float32 fhi = f*f*f; + v_float32 fmask = v_le(f, vfTresh); + v_float32 flo = v_mul(v_sub(f, v16_116), vinv7787); + v_float32 fhi = v_mul(v_mul(f, f), f); fxz[k*2+j] = v_select(fmask, flo, fhi); } } @@ -2281,9 +2282,9 @@ struct Lab2RGBfloat v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8); for(int k = 0; k < nrepeats; k++) { - ro[k] = v_fma(vc0, x[k], v_fma(vc1, y[k], vc2 * z[k])); - go[k] = v_fma(vc3, x[k], v_fma(vc4, y[k], vc5 * z[k])); - bo[k] = v_fma(vc6, x[k], v_fma(vc7, y[k], vc8 * z[k])); + ro[k] = v_fma(vc0, x[k], v_fma(vc1, y[k], v_mul(vc2, z[k]))); + go[k] = v_fma(vc3, x[k], v_fma(vc4, y[k], v_mul(vc5, z[k]))); + bo[k] = v_fma(vc6, x[k], v_fma(vc7, y[k], v_mul(vc8, z[k]))); } v_float32 one = vx_setall_f32(1.f), zero = vx_setzero_f32(); for(int k = 0; k < nrepeats; k++) @@ -2298,9 +2299,9 @@ struct Lab2RGBfloat v_float32 vgscale = vx_setall_f32(gscale); for(int k = 0; k < nrepeats; k++) { - ro[k] *= vgscale; - go[k] *= vgscale; - bo[k] *= vgscale; + ro[k] = v_mul(ro[k], vgscale); + go[k] = v_mul(go[k], vgscale); + bo[k] = v_mul(bo[k], vgscale); } for(int k = 0; k < nrepeats; k++) @@ -2500,8 +2501,8 @@ struct Lab2RGBinteger for(int k = 0; k < 4; k++) { yf[k] = v_lut((const int*)LabToYF_b, lq[k]); - y[k] = yf[k] & mask16; - ify[k] = v_reinterpret_as_s32(v_reinterpret_as_u32(yf[k]) >> 16); + y[k] = v_and(yf[k], mask16); + ify[k] = v_reinterpret_as_s32(v_shr(v_reinterpret_as_u32(yf[k]), 16)); } v_int16 ify0, ify1; @@ -2516,18 +2517,18 @@ struct Lab2RGBinteger v_uint16 mulA = vx_setall_u16(53687); v_uint32 ma[4]; v_uint32 addA = vx_setall_u32(1 << 7); - v_mul_expand((a0 + (a0 << 2)), mulA, ma[0], ma[1]); - v_mul_expand((a1 + (a1 << 2)), mulA, ma[2], ma[3]); - adiv0 = v_reinterpret_as_s16(v_pack(((ma[0] + addA) >> 13), ((ma[1] + addA) >> 13))); - adiv1 = v_reinterpret_as_s16(v_pack(((ma[2] + addA) >> 13), ((ma[3] + addA) >> 13))); + v_mul_expand((v_add(a0, v_shl<2>(a0))), mulA, ma[0], ma[1]); + v_mul_expand((v_add(a1, v_shl<2>(a1))), mulA, ma[2], ma[3]); + adiv0 = v_reinterpret_as_s16(v_pack((v_shr<13>(v_add(ma[0], addA))), (v_shr<13>(v_add(ma[1], addA))))); + adiv1 = v_reinterpret_as_s16(v_pack((v_shr<13>(v_add(ma[2], addA))), (v_shr<13>(v_add(ma[3], addA))))); v_uint16 mulB = vx_setall_u16(41943); v_uint32 mb[4]; v_uint32 addB = vx_setall_u32(1 << 4); v_mul_expand(b0, mulB, mb[0], mb[1]); v_mul_expand(b1, mulB, mb[2], mb[3]); - bdiv0 = v_reinterpret_as_s16(v_pack((mb[0] + addB) >> 9, (mb[1] + addB) >> 9)); - bdiv1 = v_reinterpret_as_s16(v_pack((mb[2] + addB) >> 9, (mb[3] + addB) >> 9)); + bdiv0 = v_reinterpret_as_s16(v_pack(v_shr<9>(v_add(mb[0], addB)), v_shr<9>(v_add(mb[1], addB)))); + bdiv1 = v_reinterpret_as_s16(v_pack(v_shr<9>(v_add(mb[2], addB)), v_shr<9>(v_add(mb[3], addB)))); // 0 <= adiv <= 8356, 0 <= bdiv <= 20890 /* x = ifxz[0]; y = y; z = ifxz[1]; */ @@ -2570,7 +2571,7 @@ struct Lab2RGBinteger { bool srgb = issRGB; ushort* tab = sRGBInvGammaTab_b; - const int vsize = v_uint8::nlanes; + const int vsize = VTraits::vlanes(); v_uint8 valpha = vx_setall_u8(alpha); v_int32 vc[9]; for(int k = 0; k < 9; k++) @@ -2592,9 +2593,9 @@ struct Lab2RGBinteger v_int32 rq[4], gq[4], bq[4]; for(int k = 0; k < 4; k++) { - rq[k] = (vc[0] * xq[k] + vc[1] * yq[k] + vc[2] * zq[k] + vdescale) >> shift; - gq[k] = (vc[3] * xq[k] + vc[4] * yq[k] + vc[5] * zq[k] + vdescale) >> shift; - bq[k] = (vc[6] * xq[k] + vc[7] * yq[k] + vc[8] * zq[k] + vdescale) >> shift; + rq[k] = v_shr(v_add(v_add(v_add(v_mul(vc[0], xq[k]), v_mul(vc[1], yq[k])), v_mul(vc[2], zq[k])), vdescale)); + gq[k] = v_shr(v_add(v_add(v_add(v_mul(vc[3], xq[k]), v_mul(vc[4], yq[k])), v_mul(vc[5], zq[k])), vdescale)); + bq[k] = v_shr(v_add(v_add(v_add(v_mul(vc[6], xq[k]), v_mul(vc[7], yq[k])), v_mul(vc[8], zq[k])), vdescale)); } //limit indices in table and then substitute @@ -2611,7 +2612,7 @@ struct Lab2RGBinteger if(srgb) { // [RRR... , GGG... , BBB...] - int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vidx[vsize*3]; + int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vidx[VTraits::max_nlanes*3]; for (int k = 0; k < 4; k++) v_store_aligned(vidx + 0*vsize + k*vsize/4, rq[k]); for (int k = 0; k < 4; k++) @@ -2631,9 +2632,9 @@ struct Lab2RGBinteger // rgb = (rgb*255) >> inv_gamma_shift for(int k = 0; k < 4; k++) { - rq[k] = ((rq[k] << 8) - rq[k]) >> inv_gamma_shift; - gq[k] = ((gq[k] << 8) - gq[k]) >> inv_gamma_shift; - bq[k] = ((bq[k] << 8) - bq[k]) >> inv_gamma_shift; + rq[k] = v_shr((v_sub(v_shl(rq[k], 8), rq[k])), inv_gamma_shift); + gq[k] = v_shr((v_sub(v_shl(gq[k], 8), gq[k])), inv_gamma_shift); + bq[k] = v_shr((v_sub(v_shl(bq[k], 8), bq[k])), inv_gamma_shift); } rgb[0] = v_reinterpret_as_u16(v_pack(rq[0], rq[1])); rgb[1] = v_reinterpret_as_u16(v_pack(rq[2], rq[3])); @@ -2730,13 +2731,13 @@ struct Lab2RGB_b static const softfloat fl = softfloat(100)/f255; #if CV_SIMD - const int fsize = v_float32::nlanes; + const int fsize = VTraits::vlanes(); v_float32 vl = vx_setall_f32((float)fl); v_float32 va = vx_setall_f32(1.f); v_float32 vb = vx_setall_f32(1.f); v_float32 vaLow = vx_setall_f32(-128.f), vbLow = vx_setall_f32(-128.f); //TODO: fix that when v_interleave is available - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3]; + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits::max_nlanes*3], interTmpA[VTraits::max_nlanes*3]; v_store_interleave(interTmpM, vl, va, vb); v_store_interleave(interTmpA, vx_setzero_f32(), vaLow, vbLow); v_float32 mluv[3], aluv[3]; @@ -2754,7 +2755,7 @@ struct Lab2RGB_b j = 0; #if CV_SIMD - const int vsize = v_uint8::nlanes; + const int vsize = VTraits::vlanes(); for( ; j <= (dn - vsize)*3; j += 3*vsize ) { v_uint8 s0, s1, s2; @@ -2808,7 +2809,7 @@ struct Lab2RGB_b v_int32 vi[4*3]; for(int k = 0; k < 4*3; k++) { - vi[k] = v_round(vf[k]*v255); + vi[k] = v_round(v_mul(vf[k], v255)); } v_uint8 rgb[3]; @@ -2830,7 +2831,7 @@ struct Lab2RGB_b for(int k = 0; k < 4; k++) { vf[k] = vx_load_aligned(buf + j + k*fsize); - vi[k] = v_round(vf[k]*v255); + vi[k] = v_round(v_mul(vf[k], v255)); } v_store(dst, v_pack_u(v_pack(vi[0], vi[1]), v_pack(vi[2], vi[3]))); } @@ -2910,8 +2911,8 @@ struct RGB2Luvfloat C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; #if CV_SIMD - const int vsize = v_float32::nlanes; - const int nrepeats = vsize == 4 ? 2 : 1; + const int vsize = VTraits::vlanes(); + const int nrepeats = VTraits::nlanes == 4 ? 2 : 1; for( ; i <= n-vsize*nrepeats; i+= vsize*nrepeats, src += scn*vsize*nrepeats, dst += 3*vsize*nrepeats) { @@ -2944,9 +2945,9 @@ struct RGB2Luvfloat v_float32 vgscale = vx_setall_f32(gscale); for (int k = 0; k < nrepeats; k++) { - R[k] *= vgscale; - G[k] *= vgscale; - B[k] *= vgscale; + R[k] = v_mul(R[k], vgscale); + G[k] = v_mul(G[k], vgscale); + B[k] = v_mul(B[k], vgscale); } for (int k = 0; k < nrepeats; k++) @@ -2963,27 +2964,27 @@ struct RGB2Luvfloat v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8); for (int k = 0; k < nrepeats; k++) { - X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, B[k]*vc2)); - Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, B[k]*vc5)); - Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, B[k]*vc8)); + X[k] = v_fma(R[k], vc0, v_fma(G[k], vc1, v_mul(B[k], vc2))); + Y[k] = v_fma(R[k], vc3, v_fma(G[k], vc4, v_mul(B[k], vc5))); + Z[k] = v_fma(R[k], vc6, v_fma(G[k], vc7, v_mul(B[k], vc8))); } v_float32 L[nrepeats], u[nrepeats], v[nrepeats]; v_float32 vmun = vx_setall_f32(-un), vmvn = vx_setall_f32(-vn); for (int k = 0; k < nrepeats; k++) { - L[k] = splineInterpolate(Y[k]*vx_setall_f32(LabCbrtTabScale), LabCbrtTab, LAB_CBRT_TAB_SIZE); + L[k] = splineInterpolate(v_mul(Y[k], vx_setall_f32(LabCbrtTabScale)), LabCbrtTab, LAB_CBRT_TAB_SIZE); // L = 116.f*L - 16.f; L[k] = v_fma(L[k], vx_setall_f32(116.f), vx_setall_f32(-16.f)); v_float32 d; // d = (4*13) / max(X + 15 * Y + 3 * Z, FLT_EPSILON) d = v_fma(Y[k], vx_setall_f32(15.f), v_fma(Z[k], vx_setall_f32(3.f), X[k])); - d = vx_setall_f32(4.f*13.f) / v_max(d, vx_setall_f32(FLT_EPSILON)); + d = v_div(vx_setall_f32(4.F * 13.F), v_max(d, vx_setall_f32(FLT_EPSILON))); // u = L*(X*d - un) - u[k] = L[k]*v_fma(X[k], d, vmun); + u[k] = v_mul(L[k], v_fma(X[k], d, vmun)); // v = L*((9*0.25f)*Y*d - vn); - v[k] = L[k]*v_fma(vx_setall_f32(9.f*0.25f)*Y[k], d, vmvn); + v[k] = v_mul(L[k], v_fma(v_mul(vx_setall_f32(9.F * 0.25F), Y[k]), d, vmvn)); } for (int k = 0; k < nrepeats; k++) @@ -3099,8 +3100,8 @@ struct Luv2RGBfloat float _un = un, _vn = vn; #if CV_SIMD - const int vsize = v_float32::nlanes; - const int nrepeats = vsize == 4 ? 2 : 1; + const int vsize = VTraits::vlanes(); + const int nrepeats = VTraits::nlanes == 4 ? 2 : 1; for( ; i <= n - vsize*nrepeats; i += vsize*nrepeats, src += vsize*3*nrepeats, dst += dcn*vsize*nrepeats) { @@ -3120,13 +3121,13 @@ struct Luv2RGBfloat v_float32 Ylo, Yhi; // ((L + 16)/116)^3 - Ylo = (L[k] + v16) * v116inv; - Ylo = Ylo*Ylo*Ylo; + Ylo = v_mul(v_add(L[k], v16), v116inv); + Ylo = v_mul(v_mul(Ylo, Ylo), Ylo); // L*(3./29.)^3 - Yhi = L[k] * v903inv; + Yhi = v_mul(L[k], v903inv); // Y = (L <= 8) ? Y0 : Y1; - Y[k] = v_select(L[k] >= vx_setall_f32(8.f), Ylo, Yhi); + Y[k] = v_select(v_ge(L[k], vx_setall_f32(8.f)), Ylo, Yhi); } v_float32 v4inv = vx_setall_f32(0.25f), v3 = vx_setall_f32(3.f); @@ -3135,18 +3136,18 @@ struct Luv2RGBfloat v_float32 up, vp; // up = 3*(u + L*_un); - up = v3*(v_fma(L[k], vx_setall_f32(_un), u[k])); + up = v_mul(v3, v_fma(L[k], vx_setall_f32(_un), u[k])); // vp = 0.25/(v + L*_vn); - vp = v4inv/(v_fma(L[k], vx_setall_f32(_vn), v[k])); + vp = v_div(v4inv, v_fma(L[k], vx_setall_f32(_vn), v[k])); // vp = max(-0.25, min(0.25, vp)); vp = v_max(vx_setall_f32(-0.25f), v_min(v4inv, vp)); //X = 3*up*vp; // (*Y) is done later - X[k] = v3*up*vp; + X[k] = v_mul(v_mul(v3, up), vp); //Z = ((12*13*L - up)*vp - 5); // (*Y) is done later // xor flips the sign, works like unary minus - Z[k] = v_fma(v_fma(L[k], vx_setall_f32(12.f*13.f), (vx_setall_f32(-0.f) ^ up)), vp, vx_setall_f32(-5.f)); + Z[k] = v_fma(v_fma(L[k], vx_setall_f32(12.f*13.f), (v_xor(vx_setall_f32(-0.F), up))), vp, vx_setall_f32(-5.f)); } v_float32 R[nrepeats], G[nrepeats], B[nrepeats]; @@ -3156,9 +3157,9 @@ struct Luv2RGBfloat for(int k = 0; k < nrepeats; k++) { // R = (X*C0 + C1 + Z*C2)*Y; // here (*Y) is done - R[k] = v_fma(Z[k], vc2, v_fma(X[k], vc0, vc1))*Y[k]; - G[k] = v_fma(Z[k], vc5, v_fma(X[k], vc3, vc4))*Y[k]; - B[k] = v_fma(Z[k], vc8, v_fma(X[k], vc6, vc7))*Y[k]; + R[k] = v_mul(v_fma(Z[k], vc2, v_fma(X[k], vc0, vc1)), Y[k]); + G[k] = v_mul(v_fma(Z[k], vc5, v_fma(X[k], vc3, vc4)), Y[k]); + B[k] = v_mul(v_fma(Z[k], vc8, v_fma(X[k], vc6, vc7)), Y[k]); } v_float32 vzero = vx_setzero_f32(), v1 = vx_setall_f32(1.f); @@ -3174,9 +3175,9 @@ struct Luv2RGBfloat v_float32 vgscale = vx_setall_f32(gscale); for(int k = 0; k < nrepeats; k++) { - R[k] *= vgscale; - G[k] *= vgscale; - B[k] *= vgscale; + R[k] = v_mul(R[k], vgscale); + G[k] = v_mul(G[k], vgscale); + B[k] = v_mul(B[k], vgscale); } for(int k = 0; k < nrepeats; k++) { @@ -3285,7 +3286,7 @@ struct RGB2Luvinterpolate #if CV_SIMD if(enablePackedRGB2Luv) { - const int vsize = v_uint16::nlanes; + const int vsize = VTraits::vlanes(); static const int nPixels = vsize*2; for(; i < n - 3*nPixels; i += 3*nPixels, src += scn*nPixels) { @@ -3315,9 +3316,9 @@ struct RGB2Luvinterpolate v_expand(r, r0, r1); v_expand(g, g0, g1); v_expand(b, b0, b1); - r0 = r0 << (lab_base_shift - 8); r1 = r1 << (lab_base_shift - 8); - g0 = g0 << (lab_base_shift - 8); g1 = g1 << (lab_base_shift - 8); - b0 = b0 << (lab_base_shift - 8); b1 = b1 << (lab_base_shift - 8); + r0 = v_shl(r0); r1 = v_shl(r1); + g0 = v_shl(g0); g1 = v_shl(g1); + b0 = v_shl(b0); b1 = v_shl(b1); /* int L, u, v; @@ -3332,9 +3333,9 @@ struct RGB2Luvinterpolate dst[i+1] = saturate_cast(u/baseDiv); dst[i+2] = saturate_cast(v/baseDiv); */ - l0 = l0 >> (lab_base_shift - 8); l1 = l1 >> (lab_base_shift - 8); - u0 = u0 >> (lab_base_shift - 8); u1 = u1 >> (lab_base_shift - 8); - v0 = v0 >> (lab_base_shift - 8); v1 = v1 >> (lab_base_shift - 8); + l0 = v_shr(l0); l1 = v_shr(l1); + u0 = v_shr(u0); u1 = v_shr(u1); + v0 = v_shr(v0); v1 = v_shr(v1); v_uint8 l = v_pack(l0, l1); v_uint8 u = v_pack(u0, u1); v_uint8 v = v_pack(v0, v1); @@ -3405,12 +3406,12 @@ struct RGB2Luv_b static const softfloat su = -uLow*f255/uRange; static const softfloat sv = -vLow*f255/vRange; #if CV_SIMD - const int fsize = v_float32::nlanes; + const int fsize = VTraits::vlanes(); v_float32 ml = vx_setall_f32((float)fL), al = vx_setzero_f32(); v_float32 mu = vx_setall_f32((float)fu), au = vx_setall_f32((float)su); v_float32 mv = vx_setall_f32((float)fv), av = vx_setall_f32((float)sv); //TODO: fix that when v_interleave is available - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3]; + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits::max_nlanes*3], interTmpA[VTraits::max_nlanes*3]; v_store_interleave(interTmpM, ml, mu, mv); v_store_interleave(interTmpA, al, au, av); v_float32 mluv[3], aluv[3]; @@ -3452,7 +3453,7 @@ struct RGB2Luv_b v_float32 f[3*4]; for(int k = 0; k < 3*4; k++) { - f[k] = v_cvt_f32(q[k])*v255inv; + f[k] = v_mul(v_cvt_f32(q[k]), v255inv); } for(int k = 0; k < 4; k++) @@ -3478,8 +3479,8 @@ struct RGB2Luv_b v_int32 q0, q1; v_expand(v_reinterpret_as_s16(d), q0, q1); - v_store_aligned(buf + j + 0*fsize, v_cvt_f32(q0)*v255inv); - v_store_aligned(buf + j + 1*fsize, v_cvt_f32(q1)*v255inv); + v_store_aligned(buf + j + 0*fsize, v_mul(v_cvt_f32(q0), v255inv)); + v_store_aligned(buf + j + 1*fsize, v_mul(v_cvt_f32(q1), v255inv)); } for( ; j < dn*bufChannels; j++, src++ ) { @@ -3633,7 +3634,8 @@ struct Luv2RGBinteger inline void processLuvToXYZ(const v_uint8& lv, const v_uint8& uv, const v_uint8& vv, v_int32 (&x)[4], v_int32 (&y)[4], v_int32 (&z)[4]) const { - const int vsize = v_uint8::nlanes; + const int vsize = VTraits::vlanes(); + const int vsize_max = VTraits::max_nlanes; v_uint16 lv0, lv1; v_expand(lv, lv0, lv1); @@ -3646,7 +3648,7 @@ struct Luv2RGBinteger v_int32 mask16 = vx_setall_s32(0xFFFF); for(int k = 0; k < 4; k++) { - y[k] = v_lut((const int*)LabToYF_b, v_reinterpret_as_s32(lq[k])) & mask16; + y[k] = v_and(v_lut((const int *)LabToYF_b, v_reinterpret_as_s32(lq[k])), mask16); } v_int32 up[4], vp[4]; @@ -3657,10 +3659,10 @@ struct Luv2RGBinteger v_expand(vv, vv0, vv1); // LL*256 v_uint16 ll0, ll1; - ll0 = lv0 << 8; ll1 = lv1 << 8; + ll0 = v_shl<8>(lv0); ll1 = v_shl<8>(lv1); v_uint16 upidx0, upidx1, vpidx0, vpidx1; - upidx0 = ll0 + uv0; upidx1 = ll1 + uv1; - vpidx0 = ll0 + vv0; vpidx1 = ll1 + vv1; + upidx0 = v_add(ll0, uv0); upidx1 = v_add(ll1, uv1); + vpidx0 = v_add(ll0, vv0); vpidx1 = v_add(ll1, vv1); v_uint32 upidx[4], vpidx[4]; v_expand(upidx0, upidx[0], upidx[1]); v_expand(upidx1, upidx[2], upidx[3]); v_expand(vpidx0, vpidx[0], vpidx[1]); v_expand(vpidx1, vpidx[2], vpidx[3]); @@ -3672,7 +3674,7 @@ struct Luv2RGBinteger // long long int vpl = LUVLUT.LvToVpl_b[LL*256+v]; v_int64 vpl[8]; - int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vpidxstore[vsize]; + int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vpidxstore[vsize_max]; for(int k = 0; k < 4; k++) { v_store_aligned(vpidxstore + k*vsize/4, v_reinterpret_as_s32(vpidx[k])); @@ -3684,12 +3686,13 @@ struct Luv2RGBinteger // not all 64-bit arithmetic is available in univ. intrinsics // need to handle it with scalar code - int64_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vvpl[vsize]; + int64_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vvpl[vsize_max]; for(int k = 0; k < 8; k++) { v_store_aligned(vvpl + k*vsize/8, vpl[k]); } - int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vup[vsize], vvp[vsize], vx[vsize], vy[vsize], vzm[vsize]; + int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vup[vsize_max], vvp[vsize_max], + vx[vsize_max], vy[vsize_max], vzm[vsize_max]; for(int k = 0; k < 4; k++) { v_store_aligned(vup + k*vsize/4, up[k]); @@ -3724,7 +3727,7 @@ struct Luv2RGBinteger // z = zm/256 + zm/65536; for (int k = 0; k < 4; k++) { - z[k] = (zm[k] >> 8) + (zm[k] >> 16); + z[k] = v_add(v_shr<8>(zm[k]), v_shr<16>(zm[k])); } // (x, z) = clip((x, z), min=0, max=2*BASE) @@ -3751,7 +3754,7 @@ struct Luv2RGBinteger { ushort* tab = sRGBInvGammaTab_b; bool srgb = issRGB; - static const int vsize = v_uint8::nlanes; + static const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_int16 vdescale = vx_setall_s16(descaleShift); v_int16 vc[9]; @@ -3771,12 +3774,12 @@ struct Luv2RGBinteger // fixing 16bit signed multiplication // by subtracting 2^(base_shift-1) and then adding result back v_int32 dummy32, fm[3]; - v_expand(vc[0]+vc[1]+vc[2], fm[0], dummy32); - v_expand(vc[3]+vc[4]+vc[5], fm[1], dummy32); - v_expand(vc[6]+vc[7]+vc[8], fm[2], dummy32); - fm[0] = fm[0] << (base_shift-1); - fm[1] = fm[1] << (base_shift-1); - fm[2] = fm[2] << (base_shift-1); + v_expand(v_add(vc[0],vc[1],vc[2]), fm[0], dummy32); + v_expand(v_add(vc[3],vc[4],vc[5]), fm[1], dummy32); + v_expand(v_add(vc[6],vc[7],vc[8]), fm[2], dummy32); + fm[0] = v_shl(fm[0], (base_shift-1)); + fm[1] = v_shl(fm[1], (base_shift-1)); + fm[2] = v_shl(fm[2], (base_shift-1)); for (; i <= n-vsize; i += vsize, src += 3*vsize, dst += dcn*vsize) { @@ -3816,15 +3819,15 @@ struct Luv2RGBinteger // a bit faster than one loop for all for(int k = 0; k < 4; k++) { - i_rgb[k+4*0] = (v_dotprod(xy[k], crxy) + v_dotprod(zd[k], crz1) + fm[0]) >> shift; + i_rgb[k+4*0] = v_shr(v_add(v_add(v_dotprod(xy[k], crxy), v_dotprod(zd[k], crz1)), fm[0])); } for(int k = 0; k < 4; k++) { - i_rgb[k+4*1] = (v_dotprod(xy[k], cgxy) + v_dotprod(zd[k], cgz1) + fm[1]) >> shift; + i_rgb[k+4*1] = v_shr(v_add(v_add(v_dotprod(xy[k], cgxy), v_dotprod(zd[k], cgz1)), fm[1])); } for(int k = 0; k < 4; k++) { - i_rgb[k+4*2] = (v_dotprod(xy[k], cbxy) + v_dotprod(zd[k], cbz1) + fm[2]) >> shift; + i_rgb[k+4*2] = v_shr(v_add(v_add(v_dotprod(xy[k], cbxy), v_dotprod(zd[k], cbz1)), fm[2])); } // [rrggbb] @@ -3842,7 +3845,7 @@ struct Luv2RGBinteger if(srgb) { // [rr.., gg.., bb..] - int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) rgbshifts[3*vsize]; + int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) rgbshifts[3*VTraits::max_nlanes]; for(int k = 0; k < 12; k++) { v_store_aligned(rgbshifts + k*vsize/4, i_rgb[k]); @@ -3857,7 +3860,7 @@ struct Luv2RGBinteger // rgb = (rgb*255) >> inv_gamma_shift for(int k = 0; k < 12; k++) { - i_rgb[k] = ((i_rgb[k] << 8) - i_rgb[k]) >> inv_gamma_shift; + i_rgb[k] = v_shr((v_sub((v_shl(i_rgb[k], 8)), i_rgb[k])), inv_gamma_shift); } for(int k = 0; k < 6; k++) @@ -3940,13 +3943,13 @@ struct Luv2RGB_b static const softfloat fv = vRange/f255; #if CV_SIMD - const int fsize = v_float32::nlanes; + const int fsize = VTraits::vlanes(); v_float32 vl = vx_setall_f32((float)fl); v_float32 vu = vx_setall_f32((float)fu); v_float32 vv = vx_setall_f32((float)fv); v_float32 vuLow = vx_setall_f32((float)uLow), vvLow = vx_setall_f32((float)vLow); //TODO: fix that when v_interleave is available - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[fsize*3], interTmpA[fsize*3]; + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) interTmpM[VTraits::max_nlanes*3], interTmpA[VTraits::max_nlanes*3]; v_store_interleave(interTmpM, vl, vu, vv); v_store_interleave(interTmpA, vx_setzero_f32(), vuLow, vvLow); v_float32 mluv[3], aluv[3]; @@ -3964,7 +3967,7 @@ struct Luv2RGB_b j = 0; #if CV_SIMD - const int vsize = v_uint8::nlanes; + const int vsize = VTraits::vlanes(); for( ; j <= (dn - vsize)*3; j += 3*vsize ) { v_uint8 s0, s1, s2; @@ -4017,7 +4020,7 @@ struct Luv2RGB_b v_int32 vi[4*3]; for(int k = 0; k < 4*3; k++) { - vi[k] = v_round(vf[k]*v255); + vi[k] = v_round(v_mul(vf[k], v255)); } v_uint8 rgb[3]; @@ -4039,7 +4042,7 @@ struct Luv2RGB_b for(int k = 0; k < 4; k++) { vf[k] = vx_load_aligned(buf + j + k*fsize); - vi[k] = v_round(vf[k]*v255); + vi[k] = v_round(v_mul(vf[k], v255)); } v_store(dst, v_pack_u(v_pack(vi[0], vi[1]), v_pack(vi[2], vi[3]))); } diff --git a/modules/imgproc/src/color_rgb.simd.hpp b/modules/imgproc/src/color_rgb.simd.hpp index 67e2febd5b..ca39d8a908 100644 --- a/modules/imgproc/src/color_rgb.simd.hpp +++ b/modules/imgproc/src/color_rgb.simd.hpp @@ -882,7 +882,7 @@ struct RGBA2mRGBA int i = 0; #if CV_SIMD - const int vsize = v_uint8::nlanes; + const int vsize = VTraits::vlanes(); v_uint8 amask = v_reinterpret_as_u8(vx_setall_u32(0xFF000000)); v_uint16 vh = vx_setall_u16(half_val+1); @@ -901,27 +901,27 @@ struct RGBA2mRGBA v_uint16 a16[4]; for(int j = 0; j < 4; j++) - a16[j] = v_reinterpret_as_u16(v[j] & amask); + a16[j] = v_reinterpret_as_u16(v_and(v[j], amask)); v_uint32 a32[4]; for(int j = 0; j < 4; j++) - a32[j] = v_reinterpret_as_u32(a16[j] | (a16[j] >> 8)); + a32[j] = v_reinterpret_as_u32(v_or(a16[j], (v_shr(a16[j], 8)))); v_uint8 a[4]; for(int j = 0; j < 4; j++) - a[j] = v_reinterpret_as_u8(a32[j] | (a32[j] >> 16)); + a[j] = v_reinterpret_as_u8(v_or(a32[j], (v_shr(a32[j], 16)))); v_uint16 m[8]; for(int j = 0; j < 4; j++) v_mul_expand(v[j], a[j], m[j], m[j+4]); for(int j = 0; j < 8; j++) - m[j] += vh; + m[j] = v_add(m[j], vh); // div 255: (v+1+(v>>8))>8 // +1 is in vh, has no effect on (v>>8) for(int j = 0; j < 8; j++) - m[j] = (m[j] + (m[j] >> 8)) >> 8; + m[j] = v_shr((v_add(m[j], (v_shr(m[j], 8)))), 8); v_uint8 d[4]; for(int j = 0; j < 4; j++) diff --git a/modules/imgproc/src/demosaicing.cpp b/modules/imgproc/src/demosaicing.cpp index 627c052aea..148df552e4 100644 --- a/modules/imgproc/src/demosaicing.cpp +++ b/modules/imgproc/src/demosaicing.cpp @@ -188,21 +188,21 @@ public: v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step)); v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2)); - v_uint16x8 b1 = ((r0 << 8) >> 7) + ((r2 << 8) >> 7); - v_uint16x8 b0 = v_rotate_right<1>(b1) + b1; - b1 = v_rotate_right<1>(b1) << 1; + v_uint16x8 b1 = v_add(v_shr<7>(v_shl<8>(r0)), v_shr<7>(v_shl<8>(r2))); + v_uint16x8 b0 = v_add(v_rotate_right<1>(b1), b1); + b1 = v_shl<1>(v_rotate_right<1>(b1)); - v_uint16x8 g0 = (r0 >> 7) + (r2 >> 7); - v_uint16x8 g1 = (r1 << 8) >> 7; - g0 += v_rotate_right<1>(g1) + g1; - g1 = v_rotate_right<1>(g1) << 2; + v_uint16x8 g0 = v_add(v_shr<7>(r0), v_shr<7>(r2)); + v_uint16x8 g1 = v_shr<7>(v_shl<8>(r1)); + g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1)); + g1 = v_shl<2>(v_rotate_right<1>(g1)); - r0 = r1 >> 8; - r1 = (v_rotate_right<1>(r0) + r0) << 2; - r0 = r0 << 3; + r0 = v_shr<8>(r1); + r1 = v_shl<2>(v_add(v_rotate_right<1>(r0), r0)); + r0 = v_shl<3>(r0); - g0 = (v_mul_hi(b0, _b2y) + v_mul_hi(g0, _g2y) + v_mul_hi(r0, _r2y)) >> 2; - g1 = (v_mul_hi(b1, _b2y) + v_mul_hi(g1, _g2y) + v_mul_hi(r1, _r2y)) >> 2; + g0 = v_shr<2>(v_add(v_add(v_mul_hi(b0, _b2y), v_mul_hi(g0, _g2y)), v_mul_hi(r0, _r2y))); + g1 = v_shr<2>(v_add(v_add(v_mul_hi(b1, _b2y), v_mul_hi(g1, _g2y)), v_mul_hi(r1, _r2y))); v_uint8x16 pack_lo, pack_hi; v_zip(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g0)), v_pack_u(v_reinterpret_as_s16(g1), v_reinterpret_as_s16(g1)), @@ -269,31 +269,31 @@ public: v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step)); v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2)); - v_uint16x8 b1 = (r0 & masklo) + (r2 & masklo); + v_uint16x8 b1 = v_add(v_and(r0, masklo), v_and(r2, masklo)); v_uint16x8 nextb1 = v_rotate_right<1>(b1); - v_uint16x8 b0 = b1 + nextb1; - b1 = (nextb1 + delta1) >> 1; - b0 = (b0 + delta2) >> 2; + v_uint16x8 b0 = v_add(b1, nextb1); + b1 = v_shr<1>(v_add(nextb1, delta1)); + b0 = v_shr<2>(v_add(b0, delta2)); // b0 b2 ... b14 b1 b3 ... b15 b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1))); - v_uint16x8 g0 = (r0 >> 8) + (r2 >> 8); - v_uint16x8 g1 = r1 & masklo; - g0 += v_rotate_right<1>(g1) + g1; + v_uint16x8 g0 = v_add(v_shr<8>(r0), v_shr<8>(r2)); + v_uint16x8 g1 = v_and(r1, masklo); + g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1)); g1 = v_rotate_right<1>(g1); - g0 = (g0 + delta2) >> 2; + g0 = v_shr<2>(v_add(g0, delta2)); // g0 g2 ... g14 g1 g3 ... g15 g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g1))); - r0 = r1 >> 8; - r1 = v_rotate_right<1>(r0) + r0; - r1 = (r1 + delta1) >> 1; + r0 = v_shr<8>(r1); + r1 = v_add(v_rotate_right<1>(r0), r0); + r1 = v_shr<1>(v_add(r1, delta1)); // r0 r2 ... r14 r1 r3 ... r15 r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1))); - b1 = (b0 ^ r0) & mask; - b0 = b0 ^ b1; - r0 = r0 ^ b1; + b1 = v_and(v_xor(b0, r0), mask); + b0 = v_xor(b0, b1); + r0 = v_xor(r0, b1); // b1 g1 b3 g3 b5 g5... v_uint8x16 pack_lo, pack_hi; @@ -402,31 +402,31 @@ public: v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step)); v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2)); - v_uint16x8 b1 = (r0 & masklo) + (r2 & masklo); + v_uint16x8 b1 = v_add(v_and(r0, masklo), v_and(r2, masklo)); v_uint16x8 nextb1 = v_rotate_right<1>(b1); - v_uint16x8 b0 = b1 + nextb1; - b1 = (nextb1 + delta1) >> 1; - b0 = (b0 + delta2) >> 2; + v_uint16x8 b0 = v_add(b1, nextb1); + b1 = v_shr<1>(v_add(nextb1, delta1)); + b0 = v_shr<2>(v_add(b0, delta2)); // b0 b2 ... b14 b1 b3 ... b15 b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1))); - v_uint16x8 g0 = (r0 >> 8) + (r2 >> 8); - v_uint16x8 g1 = r1 & masklo; - g0 += v_rotate_right<1>(g1) + g1; + v_uint16x8 g0 = v_add(v_shr<8>(r0), v_shr<8>(r2)); + v_uint16x8 g1 = v_and(r1, masklo); + g0 = v_add(g0, v_add(v_rotate_right<1>(g1), g1)); g1 = v_rotate_right<1>(g1); - g0 = (g0 + delta2) >> 2; + g0 = v_shr<2>(v_add(g0, delta2)); // g0 g2 ... g14 g1 g3 ... g15 g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(g1))); - r0 = r1 >> 8; - r1 = v_rotate_right<1>(r0) + r0; - r1 = (r1 + delta1) >> 1; + r0 = v_shr<8>(r1); + r1 = v_add(v_rotate_right<1>(r0), r0); + r1 = v_shr<1>(v_add(r1, delta1)); // r0 r2 ... r14 r1 r3 ... r15 r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1))); - b1 = (b0 ^ r0) & mask; - b0 = b0 ^ b1; - r0 = r0 ^ b1; + b1 = v_and(v_xor(b0, r0), mask); + b0 = v_xor(b0, b1); + r0 = v_xor(r0, b1); // b1 g1 b3 g3 b5 g5... v_uint8x16 pack_lo, pack_hi; @@ -498,40 +498,40 @@ public: v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step)); v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2)); - v_uint16x8 b1 = (r0 & masklow) + (r2 & masklow); + v_uint16x8 b1 = v_add(v_and(r0, masklow), v_and(r2, masklow)); v_uint16x8 nextb1 = v_rotate_right<1>(b1); - v_uint16x8 b0 = b1 + nextb1; - b1 = (nextb1 + delta1) >> 1; - b0 = (b0 + delta2) >> 2; + v_uint16x8 b0 = v_add(b1, nextb1); + b1 = v_shr<1>(v_add(nextb1, delta1)); + b0 = v_shr<2>(v_add(b0, delta2)); // b0 b2 ... b14 b1 b3 ... b15 b0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(b0), v_reinterpret_as_s16(b1))); // vertical sum - v_uint16x8 r0g = r0 >> 8; - v_uint16x8 r2g = r2 >> 8; - v_uint16x8 sumv = ((r0g + r2g) + delta1) >> 1; + v_uint16x8 r0g = v_shr<8>(r0); + v_uint16x8 r2g = v_shr<8>(r2); + v_uint16x8 sumv = v_shr<1>(v_add(v_add(r0g, r2g), delta1)); // horizontal sum - v_uint16x8 g1 = r1 & masklow; + v_uint16x8 g1 = v_and(r1, masklow); v_uint16x8 nextg1 = v_rotate_right<1>(g1); - v_uint16x8 sumg = (g1 + nextg1 + delta1) >> 1; + v_uint16x8 sumg = v_shr<1>(v_add(v_add(g1, nextg1), delta1)); // gradients - v_uint16x8 gradv = (r0g - r2g) + (r2g - r0g); - v_uint16x8 gradg = (nextg1 - g1) + (g1 - nextg1); - v_uint16x8 gmask = gradg > gradv; - v_uint16x8 g0 = (gmask & sumv) + (sumg & (gmask ^ full)); + v_uint16x8 gradv = v_add(v_sub(r0g, r2g), v_sub(r2g, r0g)); + v_uint16x8 gradg = v_add(v_sub(nextg1, g1), v_sub(g1, nextg1)); + v_uint16x8 gmask = v_gt(gradg, gradv); + v_uint16x8 g0 = v_add(v_and(gmask, sumv), v_and(sumg, v_xor(gmask, full))); // g0 g2 ... g14 g1 g3 ... g0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(g0), v_reinterpret_as_s16(nextg1))); - r0 = r1 >> 8; - r1 = v_rotate_right<1>(r0) + r0; - r1 = (r1 + delta1) >> 1; + r0 = v_shr<8>(r1); + r1 = v_add(v_rotate_right<1>(r0), r0); + r1 = v_shr<1>(v_add(r1, delta1)); // r0 r2 ... r14 r1 r3 ... r15 r0 = v_reinterpret_as_u16(v_pack_u(v_reinterpret_as_s16(r0), v_reinterpret_as_s16(r1))); - b1 = (b0 ^ r0) & mask; - b0 = b0 ^ b1; - r0 = r0 ^ b1; + b1 = v_and(v_xor(b0, r0), mask); + b0 = v_xor(b0, b1); + r0 = v_xor(r0, b1); // b1 g1 b3 g3 b5 g5... v_uint8x16 pack_lo, pack_hi; @@ -1060,19 +1060,19 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code ) v_uint16x8 b0, b1, b2, b3, b4, b5, b6; - b0 = (v_absdiff(s2, s8)<<1) + v_absdiff(s1, s7) + v_absdiff(s3, s9); - b1 = (v_absdiff(s4, s6)<<1) + v_absdiff(s1, s3) + v_absdiff(s7, s9); - b2 = v_absdiff(s3, s7)<<1; - b3 = v_absdiff(s1, s9)<<1; + b0 = v_add(v_add(v_shl<1>(v_absdiff(s2, s8)), v_absdiff(s1, s7)), v_absdiff(s3, s9)); + b1 = v_add(v_add(v_shl<1>(v_absdiff(s4, s6)), v_absdiff(s1, s3)), v_absdiff(s7, s9)); + b2 = v_shl<1>(v_absdiff(s3, s7)); + b3 = v_shl<1>(v_absdiff(s1, s9)); v_store(brow, b0); v_store(brow + N, b1); v_store(brow + N2, b2); v_store(brow + N3, b3); - b4 = b2 + v_absdiff(s2, s4) + v_absdiff(s6, s8); - b5 = b3 + v_absdiff(s2, s6) + v_absdiff(s4, s8); - b6 = (s2 + s4 + s6 + s8)>>1; + b4 = v_add(v_add(b2, v_absdiff(s2, s4)), v_absdiff(s6, s8)); + b5 = v_add(v_add(b3, v_absdiff(s2, s6)), v_absdiff(s4, s8)); + b6 = v_shr<1>(v_add(v_add(v_add(s2, s4), s6), s8)); v_store(brow + N4, b4); v_store(brow + N5, b5); @@ -1279,7 +1279,7 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code ) v_uint16x8 one = v_setall_u16(1), z = v_setzero_u16(); v_float32x4 _0_5 = v_setall_f32(0.5f); - #define v_merge_u16(a, b) (((a) & v_reinterpret_as_u16(emask)) | ((b) & v_reinterpret_as_u16(omask))) //(aA_aA_aA_aA) * (bB_bB_bB_bB) => (bA_bA_bA_bA) + #define v_merge_u16(a, b) (v_or((v_and((a), v_reinterpret_as_u16(emask))), (v_and((b), v_reinterpret_as_u16(omask))))) //(aA_aA_aA_aA) * (bB_bB_bB_bB) => (bA_bA_bA_bA) #define v_cvt_s16f32_lo(a) v_cvt_f32(v_expand_low(v_reinterpret_as_s16(a))) //(1,2,3,4,5,6,7,8) => (1f,2f,3f,4f) #define v_cvt_s16f32_hi(a) v_cvt_f32(v_expand_high(v_reinterpret_as_s16(a))) //(1,2,3,4,5,6,7,8) => (5f,6f,7f,8f) @@ -1287,16 +1287,16 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code ) for( ; i <= N - 10; i += 8, srow += 8, brow0 += 8, brow1 += 8, brow2 += 8 ) { //int gradN = brow0[0] + brow1[0]; - v_uint16x8 gradN = v_load(brow0) + v_load(brow1); + v_uint16x8 gradN = v_add(v_load(brow0), v_load(brow1)); //int gradS = brow1[0] + brow2[0]; - v_uint16x8 gradS = v_load(brow1) + v_load(brow2); + v_uint16x8 gradS = v_add(v_load(brow1), v_load(brow2)); //int gradW = brow1[N-1] + brow1[N]; - v_uint16x8 gradW = v_load(brow1+N-1) + v_load(brow1+N); + v_uint16x8 gradW = v_add(v_load(brow1 + N - 1), v_load(brow1 + N)); //int gradE = brow1[N+1] + brow1[N]; - v_uint16x8 gradE = v_load(brow1+N+1) + v_load(brow1+N); + v_uint16x8 gradE = v_add(v_load(brow1 + N + 1), v_load(brow1 + N)); //int minGrad = std::min(std::min(std::min(gradN, gradS), gradW), gradE); //int maxGrad = std::max(std::max(std::max(gradN, gradS), gradW), gradE); @@ -1307,14 +1307,14 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code ) //int gradNE = brow0[N4+1] + brow1[N4]; //int gradNE = brow0[N2] + brow0[N2+1] + brow1[N2] + brow1[N2+1]; - grad0 = v_load(brow0+N4+1) + v_load(brow1+N4); - grad1 = v_load(brow0+N2) + v_load(brow0+N2+1) + v_load(brow1+N2) + v_load(brow1+N2+1); + grad0 = v_add(v_load(brow0 + N4 + 1), v_load(brow1 + N4)); + grad1 = v_add(v_add(v_add(v_load(brow0 + N2), v_load(brow0 + N2 + 1)), v_load(brow1 + N2)), v_load(brow1 + N2 + 1)); v_uint16x8 gradNE = v_merge_u16(grad0, grad1); //int gradSW = brow1[N4] + brow2[N4-1]; //int gradSW = brow1[N2] + brow1[N2-1] + brow2[N2] + brow2[N2-1]; - grad0 = v_load(brow2+N4-1) + v_load(brow1+N4); - grad1 = v_load(brow2+N2) + v_load(brow2+N2-1) + v_load(brow1+N2) + v_load(brow1+N2-1); + grad0 = v_add(v_load(brow2 + N4 - 1), v_load(brow1 + N4)); + grad1 = v_add(v_add(v_add(v_load(brow2 + N2), v_load(brow2 + N2 - 1)), v_load(brow1 + N2)), v_load(brow1 + N2 - 1)); v_uint16x8 gradSW = v_merge_u16(grad0, grad1); minGrad = v_min(v_min(minGrad, gradNE), gradSW); @@ -1322,21 +1322,21 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code ) //int gradNW = brow0[N5-1] + brow1[N5]; //int gradNW = brow0[N3] + brow0[N3-1] + brow1[N3] + brow1[N3-1]; - grad0 = v_load(brow0+N5-1) + v_load(brow1+N5); - grad1 = v_load(brow0+N3) + v_load(brow0+N3-1) + v_load(brow1+N3) + v_load(brow1+N3-1); + grad0 = v_add(v_load(brow0 + N5 - 1), v_load(brow1 + N5)); + grad1 = v_add(v_add(v_add(v_load(brow0 + N3), v_load(brow0 + N3 - 1)), v_load(brow1 + N3)), v_load(brow1 + N3 - 1)); v_uint16x8 gradNW = v_merge_u16(grad0, grad1); //int gradSE = brow1[N5] + brow2[N5+1]; //int gradSE = brow1[N3] + brow1[N3+1] + brow2[N3] + brow2[N3+1]; - grad0 = v_load(brow2+N5+1) + v_load(brow1+N5); - grad1 = v_load(brow2+N3) + v_load(brow2+N3+1) + v_load(brow1+N3) + v_load(brow1+N3+1); + grad0 = v_add(v_load(brow2 + N5 + 1), v_load(brow1 + N5)); + grad1 = v_add(v_add(v_add(v_load(brow2 + N3), v_load(brow2 + N3 + 1)), v_load(brow1 + N3)), v_load(brow1 + N3 + 1)); v_uint16x8 gradSE = v_merge_u16(grad0, grad1); minGrad = v_min(v_min(minGrad, gradNW), gradSE); maxGrad = v_max(v_max(maxGrad, gradNW), gradSE); //int T = minGrad + maxGrad/2; - v_uint16x8 T = v_max((maxGrad >> 1), one) + minGrad; + v_uint16x8 T = v_add(v_max((v_shr<1>(maxGrad)), one), minGrad); v_uint16x8 RGs = z, GRs = z, Bs = z, ng = z; @@ -1361,133 +1361,135 @@ static void Bayer2RGB_VNG_8u( const Mat& srcmat, Mat& dstmat, int code ) v_uint16x8 t0, t1, mask; // gradN *********************************************** - mask = (T > gradN); // mask = T>gradN - ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradN) + mask = (v_gt(T, gradN)); // mask = T>gradN + ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradN) - t0 = (x3 << 1); // srow[-bstep]*2 - t1 = v_load_expand(srow - bstep*2) + x0; // srow[-bstep*2] + srow[0] + t0 = (v_shl<1>(x3)); // srow[-bstep]*2 + t1 = v_add(v_load_expand(srow - bstep * 2), x0); // srow[-bstep*2] + srow[0] // RGs += (srow[-bstep*2] + srow[0]) * (T>gradN) - RGs += (t1 & mask); + RGs = v_add(RGs, v_and(t1, mask)); // GRs += {srow[-bstep]*2; (srow[-bstep*2-1] + srow[-bstep*2+1])} * (T>gradN) - GRs += (v_merge_u16(t0, x2 + x4) & mask); + GRs = v_add(GRs, (v_and(v_merge_u16(t0, v_add(x2, x4)), mask))); // Bs += {(srow[-bstep-1]+srow[-bstep+1]); srow[-bstep]*2 } * (T>gradN) - Bs += (v_merge_u16(x1 + x5, t0) & mask); + Bs = v_add(Bs, v_and(v_merge_u16(v_add(x1, x5), t0), mask)); // gradNE ********************************************** - mask = (T > gradNE); // mask = T>gradNE - ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradNE) + mask = (v_gt(T, gradNE)); // mask = T>gradNE + ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradNE) - t0 = (x5 << 1); // srow[-bstep+1]*2 - t1 = v_load_expand(srow - bstep*2+2) + x0; // srow[-bstep*2+2] + srow[0] + t0 = (v_shl<1>(x5)); // srow[-bstep+1]*2 + t1 = v_add(v_load_expand(srow - bstep * 2 + 2), x0); // srow[-bstep*2+2] + srow[0] // RGs += {(srow[-bstep*2+2] + srow[0]); srow[-bstep+1]*2} * (T>gradNE) - RGs += (v_merge_u16(t1, t0) & mask); + RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask)); // GRs += {brow0[N6+1]; (srow[-bstep*2+1] + srow[1])} * (T>gradNE) - GRs += (v_merge_u16(v_load(brow0+N6+1), x4 + x7) & mask); + GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow0+N6+1), v_add(x4, x7)), mask)); // Bs += {srow[-bstep+1]*2; (srow[-bstep] + srow[-bstep+2])} * (T>gradNE) - Bs += (v_merge_u16(t0, x3 + x6) & mask); + Bs = v_add(Bs, v_and(v_merge_u16(t0, v_add(x3, x6)), mask)); // gradE *********************************************** - mask = (T > gradE); // mask = T>gradE - ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradE) + mask = (v_gt(T, gradE)); // mask = T>gradE + ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradE) - t0 = (x7 << 1); // srow[1]*2 - t1 = v_load_expand(srow +2) + x0; // srow[2] + srow[0] + t0 = (v_shl<1>(x7)); // srow[1]*2 + t1 = v_add(v_load_expand(srow + 2), x0); // srow[2] + srow[0] // RGs += (srow[2] + srow[0]) * (T>gradE) - RGs += (t1 & mask); + RGs = v_add(RGs, v_and(t1, mask)); // GRs += (srow[1]*2) * (T>gradE) - GRs += (t0 & mask); + GRs = v_add(GRs, v_and(t0, mask)); // Bs += {(srow[-bstep+1]+srow[bstep+1]); (srow[-bstep+2]+srow[bstep+2])} * (T>gradE) - Bs += (v_merge_u16(x5 + x9, x6 + x8) & mask); + Bs = v_add(Bs, v_and(v_merge_u16(v_add(x5, x9), v_add(x6, x8)), mask)); // gradSE ********************************************** - mask = (T > gradSE); // mask = T>gradSE - ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradSE) + mask = (v_gt(T, gradSE)); // mask = T>gradSE + ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradSE) - t0 = (x9 << 1); // srow[bstep+1]*2 - t1 = v_load_expand(srow + bstep*2+2) + x0; // srow[bstep*2+2] + srow[0] + t0 = (v_shl<1>(x9)); // srow[bstep+1]*2 + t1 = v_add(v_load_expand(srow + bstep * 2 + 2), x0); // srow[bstep*2+2] + srow[0] // RGs += {(srow[bstep*2+2] + srow[0]); srow[bstep+1]*2} * (T>gradSE) - RGs += (v_merge_u16(t1, t0) & mask); + RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask)); // GRs += {brow2[N6+1]; (srow[1]+srow[bstep*2+1])} * (T>gradSE) - GRs += (v_merge_u16(v_load(brow2+N6+1), x7 + x10) & mask); + GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow2+N6+1), v_add(x7, x10)), mask)); // Bs += {srow[bstep+1]*2; (srow[bstep+2]+srow[bstep])} * (T>gradSE) - Bs += (v_merge_u16((x9 << 1), x8 + x11) & mask); + Bs = v_add(Bs, v_and(v_merge_u16((v_shl<1>(x9)), v_add(x8, x11)), mask)); // gradS *********************************************** - mask = (T > gradS); // mask = T>gradS - ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradS) + mask = (v_gt(T, gradS)); // mask = T>gradS + ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradS) - t0 = (x11 << 1); // srow[bstep]*2 - t1 = v_load_expand(srow + bstep*2) + x0; // srow[bstep*2]+srow[0] + t0 = (v_shl<1>(x11)); // srow[bstep]*2 + t1 = v_add(v_load_expand(srow + bstep * 2), x0); // srow[bstep*2]+srow[0] // RGs += (srow[bstep*2]+srow[0]) * (T>gradS) - RGs += (t1 & mask); + RGs = v_add(RGs, v_and(t1, mask)); // GRs += {srow[bstep]*2; (srow[bstep*2+1]+srow[bstep*2-1])} * (T>gradS) - GRs += (v_merge_u16(t0, x10 + x12) & mask); + GRs = v_add(GRs, v_and(v_merge_u16(t0, v_add(x10, x12)), mask)); // Bs += {(srow[bstep+1]+srow[bstep-1]); srow[bstep]*2} * (T>gradS) - Bs += (v_merge_u16(x9 + x13, t0) & mask); + Bs = v_add(Bs, v_and(v_merge_u16(v_add(x9, x13), t0), mask)); // gradSW ********************************************** - mask = (T > gradSW); // mask = T>gradSW - ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradSW) + mask = (v_gt(T, gradSW)); // mask = T>gradSW + ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradSW) - t0 = (x13 << 1); // srow[bstep-1]*2 - t1 = v_load_expand(srow + bstep*2-2) + x0; // srow[bstep*2-2]+srow[0] + t0 = (v_shl<1>(x13)); // srow[bstep-1]*2 + t1 = v_add(v_load_expand(srow + bstep * 2 - 2), x0); // srow[bstep*2-2]+srow[0] // RGs += {(srow[bstep*2-2]+srow[0]); srow[bstep-1]*2} * (T>gradSW) - RGs += (v_merge_u16(t1, t0) & mask); + RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask)); // GRs += {brow2[N6-1]; (srow[bstep*2-1]+srow[-1])} * (T>gradSW) - GRs += (v_merge_u16(v_load(brow2+N6-1), x12 + x15) & mask); + GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow2+N6-1), v_add(x12, x15)), mask)); // Bs += {srow[bstep-1]*2; (srow[bstep]+srow[bstep-2])} * (T>gradSW) - Bs += (v_merge_u16(t0, x11 + x14) & mask); + Bs = v_add(Bs, v_and(v_merge_u16(t0, v_add(x11, x14)), mask)); // gradW *********************************************** - mask = (T > gradW); // mask = T>gradW - ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradW) + mask = (v_gt(T, gradW)); // mask = T>gradW + ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradW) - t0 = (x15 << 1); // srow[-1]*2 - t1 = v_load_expand(srow -2) + x0; // srow[-2]+srow[0] + t0 = (v_shl<1>(x15)); // srow[-1]*2 + t1 = v_add(v_load_expand(srow - 2), x0); // srow[-2]+srow[0] // RGs += (srow[-2]+srow[0]) * (T>gradW) - RGs += (t1 & mask); + RGs = v_add(RGs, v_and(t1, mask)); // GRs += (srow[-1]*2) * (T>gradW) - GRs += (t0 & mask); + GRs = v_add(GRs, v_and(t0, mask)); // Bs += {(srow[-bstep-1]+srow[bstep-1]); (srow[bstep-2]+srow[-bstep-2])} * (T>gradW) - Bs += (v_merge_u16(x1 + x13, x14 + x16) & mask); + Bs = v_add(Bs, v_and(v_merge_u16(v_add(x1, x13), v_add(x14, x16)), mask)); // gradNW ********************************************** - mask = (T > gradNW); // mask = T>gradNW - ng = v_reinterpret_as_u16(v_reinterpret_as_s16(ng) - v_reinterpret_as_s16(mask)); // ng += (T>gradNW) + mask = (v_gt(T, gradNW)); // mask = T>gradNW + ng = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(ng), v_reinterpret_as_s16(mask))); // ng += (T>gradNW) - t0 = (x1 << 1); // srow[-bstep-1]*2 - t1 = v_load_expand(srow -bstep*2-2) + x0; // srow[-bstep*2-2]+srow[0] + t0 = (v_shl<1>(x1)); // srow[-bstep-1]*2 + t1 = v_add(v_load_expand(srow - bstep * 2 - 2), x0); // srow[-bstep*2-2]+srow[0] // RGs += {(srow[-bstep*2-2]+srow[0]); srow[-bstep-1]*2} * (T>gradNW) - RGs += (v_merge_u16(t1, t0) & mask); + RGs = v_add(RGs, v_and(v_merge_u16(t1, t0), mask)); // GRs += {brow0[N6-1]; (srow[-bstep*2-1]+srow[-1])} * (T>gradNW) - GRs += (v_merge_u16(v_load(brow0+N6-1), x2 + x15) & mask); + GRs = v_add(GRs, v_and(v_merge_u16(v_load(brow0+N6-1), v_add(x2, x15)), mask)); // Bs += {srow[-bstep-1]*2; (srow[-bstep]+srow[-bstep-2])} * (T>gradNW) - Bs += (v_merge_u16((x1 << 1), x3 + x16) & mask); + Bs = v_add(Bs, v_and(v_merge_u16(v_shl<1>(x1), v_add(x3, x16)), mask)); - v_float32x4 ngf0 = _0_5 / v_cvt_s16f32_lo(ng); - v_float32x4 ngf1 = _0_5 / v_cvt_s16f32_hi(ng); + v_float32x4 ngf0 = v_div(_0_5, v_cvt_s16f32_lo(ng)); + v_float32x4 ngf1 = v_div(_0_5, v_cvt_s16f32_hi(ng)); // now interpolate r, g & b - t0 = v_reinterpret_as_u16(v_reinterpret_as_s16(GRs) - v_reinterpret_as_s16(RGs)); - t1 = v_reinterpret_as_u16(v_reinterpret_as_s16(Bs) - v_reinterpret_as_s16(RGs)); + t0 = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(GRs), v_reinterpret_as_s16(RGs))); + t1 = v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(Bs), v_reinterpret_as_s16(RGs))); - t0 = v_reinterpret_as_u16(v_reinterpret_as_s16(x0) + + t0 = v_reinterpret_as_u16( + v_add(v_reinterpret_as_s16(x0), v_pack( - v_round(v_cvt_s16f32_lo(t0) * ngf0), - v_round(v_cvt_s16f32_hi(t0) * ngf1))); + v_round(v_mul(v_cvt_s16f32_lo(t0), ngf0)), + v_round(v_mul(v_cvt_s16f32_hi(t0), ngf1))))); - t1 = v_reinterpret_as_u16(v_reinterpret_as_s16(x0) + + t1 = v_reinterpret_as_u16( + v_add(v_reinterpret_as_s16(x0), v_pack( - v_round(v_cvt_s16f32_lo(t1) * ngf0), - v_round(v_cvt_s16f32_hi(t1) * ngf1))); + v_round(v_mul(v_cvt_s16f32_lo(t1), ngf0)), + v_round(v_mul(v_cvt_s16f32_hi(t1), ngf1))))); x1 = v_merge_u16(x0, t0); x2 = v_merge_u16(t0, x0); diff --git a/modules/imgproc/src/filter.simd.hpp b/modules/imgproc/src/filter.simd.hpp index 06053e63fe..9306c78a30 100644 --- a/modules/imgproc/src/filter.simd.hpp +++ b/modules/imgproc/src/filter.simd.hpp @@ -1084,9 +1084,9 @@ struct SymmColumnVec_32s8u i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - 4 /*v_int32x4::nlanes*/ ) + while( i <= width - 4 /*VTraits::vlanes()*/ ) #else - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) #endif { v_float32 s0 = v_muladd(v_cvt_f32(vx_load(src[0] + i)), vx_setall_f32(ky[0]), vx_setall_f32(delta)); @@ -1140,9 +1140,9 @@ struct SymmColumnVec_32s8u i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - 4 /*v_int32x4::nlanes*/ ) + while( i <= width - 4 /*VTraits::vlanes()*/ ) #else - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) #endif { v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i))), vx_setall_f32(ky[1]), vx_setall_f32(delta)); @@ -1321,23 +1321,23 @@ struct SymmColumnSmallVec_32s16s { v_int32 k0 = vx_setall_s32((int)ky[0]), k1 = vx_setall_s32((int)ky[1]); v_int32 d4 = vx_setall_s32(d); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4)), - v_muladd(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + v_int32::nlanes), k0, d4)))); - v_store(dst + i + v_int16::nlanes, v_pack(v_muladd(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + 2*v_int32::nlanes), k0, d4)), - v_muladd(vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + 3*v_int32::nlanes), k0, d4)))); + v_store(dst + i, v_pack(v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)), + v_muladd(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), k1, v_muladd(vx_load(S1 + i + VTraits::vlanes()), k0, d4)))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_muladd(v_add(vx_load(S0 + i + 2 * VTraits::vlanes()), vx_load(S2 + i + 2 * VTraits::vlanes())), k1, v_muladd(vx_load(S1 + i + 2*VTraits::vlanes()), k0, d4)), + v_muladd(v_add(vx_load(S0 + i + 3 * VTraits::vlanes()), vx_load(S2 + i + 3 * VTraits::vlanes())), k1, v_muladd(vx_load(S1 + i + 3*VTraits::vlanes()), k0, d4)))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4)), - v_muladd(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes), k1, v_muladd(vx_load(S1 + i + v_int32::nlanes), k0, d4)))); - i += v_int16::nlanes; + v_store(dst + i, v_pack(v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4)), + v_muladd(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), k1, v_muladd(vx_load(S1 + i + VTraits::vlanes()), k0, d4)))); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_pack_store(dst + i, v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4))); - i += v_int32::nlanes; + v_pack_store(dst + i, v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4))); + i += VTraits::vlanes(); } } #endif @@ -2237,9 +2237,9 @@ struct FilterVec_8u i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - 4 /*v_int32x4::nlanes*/ ) + while( i <= width - 4 /*VTraits::vlanes()*/ ) #else - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) #endif { v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[0] + i))), vx_setall_f32(kf[0]), vx_setall_f32(delta)); @@ -2248,7 +2248,7 @@ struct FilterVec_8u v_int32 s32 = v_round(s0); v_int16 s16 = v_pack(s32, s32); *(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16))); - i += 4 /*v_int32x4::nlanes*/ ; + i += 4 /*VTraits::vlanes()*/ ; } return i; } diff --git a/modules/imgproc/src/histogram.cpp b/modules/imgproc/src/histogram.cpp index 23fe2bb42a..449909b945 100644 --- a/modules/imgproc/src/histogram.cpp +++ b/modules/imgproc/src/histogram.cpp @@ -2106,7 +2106,7 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) v_float32 v_s11 = vx_setzero_f32(); v_float32 v_s12 = vx_setzero_f32(); v_float32 v_s22 = vx_setzero_f32(); - for (; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_a = vx_load(h1 + j); v_float32 v_b = vx_load(h2 + j); @@ -2147,10 +2147,10 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) result += v_reduce_sum(v_result); #elif CV_SIMD v_float32 v_result = vx_setzero_f32(); - for (; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_src = v_min(vx_load(h1 + j), vx_load(h2 + j)); - v_result += v_src; + v_result = v_add(v_result, v_src); } result += v_reduce_sum(v_result); #endif @@ -2187,7 +2187,7 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) v_float32 v_s1 = vx_setzero_f32(); v_float32 v_s2 = vx_setzero_f32(); v_float32 v_result = vx_setzero_f32(); - for (; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_a = vx_load(h1 + j); v_float32 v_b = vx_load(h2 + j); diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index a1441fd44f..5c1d2d4dd3 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -455,7 +455,7 @@ struct RemapVec_8u v_int32x4 delta = v_setall_s32(INTER_REMAP_COEF_SCALE / 2); v_int16x8 xy2ofs = v_reinterpret_as_s16(v_setall_s32(cn + (sstep << 16))); int CV_DECL_ALIGNED(16) iofs0[4], iofs1[4]; - const uchar* src_limit_8bytes = _src.datalimit - v_int16x8::nlanes; + const uchar* src_limit_8bytes = _src.datalimit - VTraits::vlanes(); #define CV_PICK_AND_PACK_RGB(ptr, offset, result) \ { \ const uchar* const p = ((const uchar*)ptr) + (offset); \ @@ -483,7 +483,7 @@ struct RemapVec_8u v_uint8x16 rrggbbaa, dummy; \ v_uint16x8 rrggbbaa8, dummy8; \ v_uint8x16 rgba0 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p), 0, 0, 0)); \ - v_uint8x16 rgba1 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p + v_int32x4::nlanes), 0, 0, 0)); \ + v_uint8x16 rgba1 = v_reinterpret_as_u8(v_int32x4(*(unaligned_int*)(p + VTraits::vlanes()), 0, 0, 0)); \ v_zip(rgba0, rgba1, rrggbbaa, dummy); \ v_expand(rrggbbaa, rrggbbaa8, dummy8); \ result = v_reinterpret_as_s16(rrggbbaa8); \ @@ -534,8 +534,8 @@ struct RemapVec_8u v3 = v_dotprod(v_reinterpret_as_s16(v3), v_reinterpret_as_s16(d2), delta); v2 = v_dotprod(v_reinterpret_as_s16(v2), v_reinterpret_as_s16(c2), v3); - v0 = v0 >> INTER_REMAP_COEF_BITS; - v2 = v2 >> INTER_REMAP_COEF_BITS; + v0 = v_shr(v0); + v2 = v_shr(v2); v_pack_u_store(D + x, v_pack(v0, v2)); } } @@ -563,8 +563,8 @@ struct RemapVec_8u CV_PICK_AND_PACK_RGB(S0, iofs0[1], u1); CV_PICK_AND_PACK_RGB(S1, iofs0[1], v1); - v_int32x4 result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS; - v_int32x4 result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS; + v_int32x4 result0 = v_shr(v_dotprod(u0, w00, v_dotprod(v0, w01, delta))); + v_int32x4 result1 = v_shr(v_dotprod(u1, w10, v_dotprod(v1, w11, delta))); result0 = v_rotate_left<1>(result0); v_int16x8 result8 = v_pack(result0, result1); @@ -581,8 +581,8 @@ struct RemapVec_8u CV_PICK_AND_PACK_RGB(S0, iofs0[3], u1); CV_PICK_AND_PACK_RGB(S1, iofs0[3], v1); - result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS; - result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS; + result0 = v_shr(v_dotprod(u0, w00, v_dotprod(v0, w01, delta))); + result1 = v_shr(v_dotprod(u1, w10, v_dotprod(v1, w11, delta))); result0 = v_rotate_left<1>(result0); result8 = v_pack(result0, result1); @@ -613,8 +613,8 @@ struct RemapVec_8u CV_PICK_AND_PACK_RGBA(S0, iofs0[1], u1); CV_PICK_AND_PACK_RGBA(S1, iofs0[1], v1); - v_int32x4 result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS; - v_int32x4 result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS; + v_int32x4 result0 = v_shr(v_dotprod(u0, w00, v_dotprod(v0, w01, delta))); + v_int32x4 result1 = v_shr(v_dotprod(u1, w10, v_dotprod(v1, w11, delta))); v_int16x8 result8 = v_pack(result0, result1); v_pack_u_store(D, result8); @@ -627,8 +627,8 @@ struct RemapVec_8u CV_PICK_AND_PACK_RGBA(S0, iofs0[3], u1); CV_PICK_AND_PACK_RGBA(S1, iofs0[3], v1); - result0 = v_dotprod(u0, w00, v_dotprod(v0, w01, delta)) >> INTER_REMAP_COEF_BITS; - result1 = v_dotprod(u1, w10, v_dotprod(v1, w11, delta)) >> INTER_REMAP_COEF_BITS; + result0 = v_shr(v_dotprod(u0, w00, v_dotprod(v0, w01, delta))); + result1 = v_shr(v_dotprod(u1, w10, v_dotprod(v1, w11, delta))); result8 = v_pack(result0, result1); v_pack_u_store(D + 8, result8); } @@ -1164,7 +1164,7 @@ public: #if CV_SIMD128 { - int span = v_float32x4::nlanes; + int span = VTraits::vlanes(); for( ; x1 <= bcols - span * 2; x1 += span * 2 ) { v_int32x4 ix0 = v_round(v_load(sX + x1)); @@ -1206,9 +1206,9 @@ public: #if CV_SIMD128 { v_uint16x8 v_scale = v_setall_u16(INTER_TAB_SIZE2 - 1); - int span = v_uint16x8::nlanes; + int span = VTraits::vlanes(); for( ; x1 <= bcols - span; x1 += span ) - v_store((unsigned short*)(A + x1), v_load(sA + x1) & v_scale); + v_store((unsigned short*)(A + x1), v_and(v_load(sA + x1), v_scale)); } #endif for( ; x1 < bcols; x1++ ) @@ -1224,16 +1224,16 @@ public: { v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE); v_int32x4 v_scale2 = v_setall_s32(INTER_TAB_SIZE - 1); - int span = v_float32x4::nlanes; + int span = VTraits::vlanes(); for( ; x1 <= bcols - span * 2; x1 += span * 2 ) { - v_int32x4 v_sx0 = v_round(v_scale * v_load(sX + x1)); - v_int32x4 v_sy0 = v_round(v_scale * v_load(sY + x1)); - v_int32x4 v_sx1 = v_round(v_scale * v_load(sX + x1 + span)); - v_int32x4 v_sy1 = v_round(v_scale * v_load(sY + x1 + span)); - v_uint16x8 v_sx8 = v_reinterpret_as_u16(v_pack(v_sx0 & v_scale2, v_sx1 & v_scale2)); - v_uint16x8 v_sy8 = v_reinterpret_as_u16(v_pack(v_sy0 & v_scale2, v_sy1 & v_scale2)); - v_uint16x8 v_v = v_shl(v_sy8) | (v_sx8); + v_int32x4 v_sx0 = v_round(v_mul(v_scale, v_load(sX + x1))); + v_int32x4 v_sy0 = v_round(v_mul(v_scale, v_load(sY + x1))); + v_int32x4 v_sx1 = v_round(v_mul(v_scale, v_load(sX + x1 + span))); + v_int32x4 v_sy1 = v_round(v_mul(v_scale, v_load(sY + x1 + span))); + v_uint16x8 v_sx8 = v_reinterpret_as_u16(v_pack(v_and(v_sx0, v_scale2), v_and(v_sx1, v_scale2))); + v_uint16x8 v_sy8 = v_reinterpret_as_u16(v_pack(v_and(v_sy0, v_scale2), v_and(v_sy1, v_scale2))); + v_uint16x8 v_v = v_or(v_shl(v_sy8), v_sx8); v_store(A + x1, v_v); v_int16x8 v_d0 = v_pack(v_shr(v_sx0), v_shr(v_sx1)); @@ -1261,18 +1261,18 @@ public: { v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE); v_int32x4 v_scale2 = v_setall_s32(INTER_TAB_SIZE - 1), v_scale3 = v_setall_s32(INTER_TAB_SIZE); - int span = v_float32x4::nlanes; + int span = VTraits::vlanes(); for( ; x1 <= bcols - span * 2; x1 += span * 2 ) { v_float32x4 v_fx, v_fy; v_load_deinterleave(sXY + (x1 << 1), v_fx, v_fy); - v_int32x4 v_sx0 = v_round(v_fx * v_scale); - v_int32x4 v_sy0 = v_round(v_fy * v_scale); + v_int32x4 v_sx0 = v_round(v_mul(v_fx, v_scale)); + v_int32x4 v_sy0 = v_round(v_mul(v_fy, v_scale)); v_load_deinterleave(sXY + ((x1 + span) << 1), v_fx, v_fy); - v_int32x4 v_sx1 = v_round(v_fx * v_scale); - v_int32x4 v_sy1 = v_round(v_fy * v_scale); - v_int32x4 v_v0 = v_muladd(v_scale3, (v_sy0 & v_scale2), (v_sx0 & v_scale2)); - v_int32x4 v_v1 = v_muladd(v_scale3, (v_sy1 & v_scale2), (v_sx1 & v_scale2)); + v_int32x4 v_sx1 = v_round(v_mul(v_fx, v_scale)); + v_int32x4 v_sy1 = v_round(v_mul(v_fy, v_scale)); + v_int32x4 v_v0 = v_muladd(v_scale3, (v_and(v_sy0, v_scale2)), (v_and(v_sx0, v_scale2))); + v_int32x4 v_v1 = v_muladd(v_scale3, (v_and(v_sy1, v_scale2)), (v_and(v_sx1, v_scale2))); v_uint16x8 v_v8 = v_reinterpret_as_u16(v_pack(v_v0, v_v1)); v_store(A + x1, v_v8); v_int16x8 v_dx = v_pack(v_shr(v_sx0), v_shr(v_sx1)); @@ -1941,7 +1941,7 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, { #if CV_SIMD128 { - int span = v_int16x8::nlanes; + int span = VTraits::vlanes(); for( ; x <= size.width - span; x += span ) { v_int16x8 v_dst[2]; @@ -1973,21 +1973,21 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE); v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1); v_int32x4 v_scale3 = v_setall_s32(INTER_TAB_SIZE); - int span = v_float32x4::nlanes; + int span = VTraits::vlanes(); for( ; x <= size.width - span * 2; x += span * 2 ) { - v_int32x4 v_ix0 = v_round(v_scale * (v_load(src1f + x))); - v_int32x4 v_ix1 = v_round(v_scale * (v_load(src1f + x + span))); - v_int32x4 v_iy0 = v_round(v_scale * (v_load(src2f + x))); - v_int32x4 v_iy1 = v_round(v_scale * (v_load(src2f + x + span))); + v_int32x4 v_ix0 = v_round(v_mul(v_scale, v_load(src1f + x))); + v_int32x4 v_ix1 = v_round(v_mul(v_scale, v_load(src1f + x + span))); + v_int32x4 v_iy0 = v_round(v_mul(v_scale, v_load(src2f + x))); + v_int32x4 v_iy1 = v_round(v_mul(v_scale, v_load(src2f + x + span))); v_int16x8 v_dst[2]; v_dst[0] = v_pack(v_shr(v_ix0), v_shr(v_ix1)); v_dst[1] = v_pack(v_shr(v_iy0), v_shr(v_iy1)); v_store_interleave(dst1 + (x << 1), v_dst[0], v_dst[1]); - v_int32x4 v_dst0 = v_muladd(v_scale3, (v_iy0 & v_mask), (v_ix0 & v_mask)); - v_int32x4 v_dst1 = v_muladd(v_scale3, (v_iy1 & v_mask), (v_ix1 & v_mask)); + v_int32x4 v_dst0 = v_muladd(v_scale3, (v_and(v_iy0, v_mask)), (v_and(v_ix0, v_mask))); + v_int32x4 v_dst1 = v_muladd(v_scale3, (v_and(v_iy1, v_mask)), (v_and(v_ix1, v_mask))); v_store(dst2 + x, v_pack_u(v_dst0, v_dst1)); } } @@ -2008,7 +2008,7 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, if( nninterpolate ) { #if CV_SIMD128 - int span = v_float32x4::nlanes; + int span = VTraits::vlanes(); { for( ; x <= (size.width << 1) - span * 2; x += span * 2 ) v_store(dst1 + x, v_pack(v_round(v_load(src1f + x)), @@ -2034,16 +2034,16 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, v_float32x4 v_scale = v_setall_f32((float)INTER_TAB_SIZE); v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1); v_int32x4 v_scale3 = v_setall_s32(INTER_TAB_SIZE); - int span = v_uint16x8::nlanes; + int span = VTraits::vlanes(); for (; x <= size.width - span; x += span ) { v_float32x4 v_src0[2], v_src1[2]; v_load_deinterleave(src1f + (x << 1), v_src0[0], v_src0[1]); v_load_deinterleave(src1f + (x << 1) + span, v_src1[0], v_src1[1]); - v_int32x4 v_ix0 = v_round(v_src0[0] * v_scale); - v_int32x4 v_ix1 = v_round(v_src1[0] * v_scale); - v_int32x4 v_iy0 = v_round(v_src0[1] * v_scale); - v_int32x4 v_iy1 = v_round(v_src1[1] * v_scale); + v_int32x4 v_ix0 = v_round(v_mul(v_src0[0], v_scale)); + v_int32x4 v_ix1 = v_round(v_mul(v_src1[0], v_scale)); + v_int32x4 v_iy0 = v_round(v_mul(v_src0[1], v_scale)); + v_int32x4 v_iy1 = v_round(v_mul(v_src1[1], v_scale)); v_int16x8 v_dst[2]; v_dst[0] = v_pack(v_shr(v_ix0), v_shr(v_ix1)); @@ -2051,8 +2051,8 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, v_store_interleave(dst1 + (x << 1), v_dst[0], v_dst[1]); v_store(dst2 + x, v_pack_u( - v_muladd(v_scale3, (v_iy0 & v_mask), (v_ix0 & v_mask)), - v_muladd(v_scale3, (v_iy1 & v_mask), (v_ix1 & v_mask)))); + v_muladd(v_scale3, (v_and(v_iy0, v_mask)), (v_and(v_ix0, v_mask))), + v_muladd(v_scale3, (v_and(v_iy1, v_mask)), (v_and(v_ix1, v_mask))))); } } #endif @@ -2074,13 +2074,13 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, v_uint16x8 v_mask2 = v_setall_u16(INTER_TAB_SIZE2-1); v_uint32x4 v_zero = v_setzero_u32(), v_mask = v_setall_u32(INTER_TAB_SIZE-1); v_float32x4 v_scale = v_setall_f32(scale); - int span = v_float32x4::nlanes; + int span = VTraits::vlanes(); for( ; x <= size.width - span * 2; x += span * 2 ) { v_uint32x4 v_fxy1, v_fxy2; if ( src2 ) { - v_uint16x8 v_src2 = v_load(src2 + x) & v_mask2; + v_uint16x8 v_src2 = v_and(v_load(src2 + x), v_mask2); v_expand(v_src2, v_fxy1, v_fxy2); } else @@ -2091,9 +2091,9 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, v_load_deinterleave(src1 + (x << 1), v_src[0], v_src[1]); v_expand(v_src[0], v_src0[0], v_src0[1]); v_expand(v_src[1], v_src1[0], v_src1[1]); - #define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32((FXY) & v_mask)),\ + #define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32(v_and((FXY), v_mask))),\ v_cvt_f32(v_reinterpret_as_s32(X))) - #define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32((FXY) >> INTER_BITS)),\ + #define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_reinterpret_as_s32(v_shr((FXY)))),\ v_cvt_f32(v_reinterpret_as_s32(Y))) v_float32x4 v_dst1 = CV_COMPUTE_MAP_X(v_src0[0], v_fxy1); v_float32x4 v_dst2 = CV_COMPUTE_MAP_Y(v_src1[0], v_fxy1); @@ -2123,13 +2123,13 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, v_int16x8 v_mask2 = v_setall_s16(INTER_TAB_SIZE2-1); v_int32x4 v_zero = v_setzero_s32(), v_mask = v_setall_s32(INTER_TAB_SIZE-1); v_float32x4 v_scale = v_setall_f32(scale); - int span = v_int16x8::nlanes; + int span = VTraits::vlanes(); for( ; x <= size.width - span; x += span ) { v_int32x4 v_fxy1, v_fxy2; if (src2) { - v_int16x8 v_src2 = v_load((short *)src2 + x) & v_mask2; + v_int16x8 v_src2 = v_and(v_load((short *)src2 + x), v_mask2); v_expand(v_src2, v_fxy1, v_fxy2); } else @@ -2142,8 +2142,8 @@ void cv::convertMaps( InputArray _map1, InputArray _map2, v_expand(v_src[0], v_src0[0], v_src0[1]); v_expand(v_src[1], v_src1[0], v_src1[1]); - #define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32((FXY) & v_mask), v_cvt_f32(X)) - #define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32((FXY) >> INTER_BITS), v_cvt_f32(Y)) + #define CV_COMPUTE_MAP_X(X, FXY) v_muladd(v_scale, v_cvt_f32(v_and((FXY), v_mask)), v_cvt_f32(X)) + #define CV_COMPUTE_MAP_Y(Y, FXY) v_muladd(v_scale, v_cvt_f32(v_shr((FXY))), v_cvt_f32(Y)) v_dst[0] = CV_COMPUTE_MAP_X(v_src0[0], v_fxy1); v_dst[1] = CV_COMPUTE_MAP_Y(v_src1[0], v_fxy1); v_store_interleave(dst1f + (x << 1), v_dst[0], v_dst[1]); @@ -2234,12 +2234,12 @@ public: #if CV_SIMD128 { v_int32x4 v_X0 = v_setall_s32(X0), v_Y0 = v_setall_s32(Y0); - int span = v_uint16x8::nlanes; + int span = VTraits::vlanes(); for( ; x1 <= bw - span; x1 += span ) { v_int16x8 v_dst[2]; - #define CV_CONVERT_MAP(ptr,offset,shift) v_pack(v_shr(shift+v_load(ptr + offset)),\ - v_shr(shift+v_load(ptr + offset + 4))) + #define CV_CONVERT_MAP(ptr,offset,shift) v_pack(v_shr(v_add(shift,v_load(ptr + offset))),\ + v_shr(v_add(shift,v_load(ptr + offset + 4)))) v_dst[0] = CV_CONVERT_MAP(adelta, x+x1, v_X0); v_dst[1] = CV_CONVERT_MAP(bdelta, x+x1, v_Y0); #undef CV_CONVERT_MAP @@ -2272,21 +2272,21 @@ public: { v_int32x4 v__X0 = v_setall_s32(X0), v__Y0 = v_setall_s32(Y0); v_int32x4 v_mask = v_setall_s32(INTER_TAB_SIZE - 1); - int span = v_float32x4::nlanes; + int span = VTraits::vlanes(); for( ; x1 <= bw - span * 2; x1 += span * 2 ) { - v_int32x4 v_X0 = v_shr(v__X0 + v_load(adelta + x + x1)); - v_int32x4 v_Y0 = v_shr(v__Y0 + v_load(bdelta + x + x1)); - v_int32x4 v_X1 = v_shr(v__X0 + v_load(adelta + x + x1 + span)); - v_int32x4 v_Y1 = v_shr(v__Y0 + v_load(bdelta + x + x1 + span)); + v_int32x4 v_X0 = v_shr(v_add(v__X0, v_load(this->adelta + x + x1))); + v_int32x4 v_Y0 = v_shr(v_add(v__Y0, v_load(this->bdelta + x + x1))); + v_int32x4 v_X1 = v_shr(v_add(v__X0, v_load(this->adelta + x + x1 + span))); + v_int32x4 v_Y1 = v_shr(v_add(v__Y0, v_load(this->bdelta + x + x1 + span))); v_int16x8 v_xy[2]; v_xy[0] = v_pack(v_shr(v_X0), v_shr(v_X1)); v_xy[1] = v_pack(v_shr(v_Y0), v_shr(v_Y1)); v_store_interleave(xy + (x1 << 1), v_xy[0], v_xy[1]); - v_int32x4 v_alpha0 = v_shl(v_Y0 & v_mask) | (v_X0 & v_mask); - v_int32x4 v_alpha1 = v_shl(v_Y1 & v_mask) | (v_X1 & v_mask); + v_int32x4 v_alpha0 = v_or(v_shl(v_and(v_Y0, v_mask)), v_and(v_X0, v_mask)); + v_int32x4 v_alpha1 = v_or(v_shl(v_and(v_Y1, v_mask)), v_and(v_X1, v_mask)); v_store(alpha + x1, v_pack(v_alpha0, v_alpha1)); } } @@ -2866,16 +2866,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0 v_int32x4 v_X0, v_Y0; { v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero); - v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero); + v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero); - v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero); + v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_X0 = v_round(v_fX0, v_fX1); v_Y0 = v_round(v_fY0, v_fY1); @@ -2885,16 +2885,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0 v_int32x4 v_X1, v_Y1; { v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero); - v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero); + v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero); - v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero); + v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_X1 = v_round(v_fX0, v_fX1); v_Y1 = v_round(v_fY0, v_fY1); @@ -2904,16 +2904,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0 v_int32x4 v_X2, v_Y2; { v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero); - v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero); + v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero); - v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero); + v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_X2 = v_round(v_fX0, v_fX1); v_Y2 = v_round(v_fY0, v_fY1); @@ -2923,16 +2923,16 @@ void WarpPerspectiveLine_ProcessNN_CV_SIMD(const double *M, short* xy, double X0 v_int32x4 v_X3, v_Y3; { v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero); - v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero); + v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_1 / v_W, v_zero); - v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_1, v_W), v_zero); + v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_X3 = v_round(v_fX0, v_fX1); v_Y3 = v_round(v_fY0, v_fY1); @@ -2987,16 +2987,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph v_int32x4 v_X0, v_Y0; { v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_its / v_W, v_zero); - v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero); + v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_its / v_W, v_zero); - v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero); + v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_X0 = v_round(v_fX0, v_fX1); v_Y0 = v_round(v_fY0, v_fY1); @@ -3006,16 +3006,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph v_int32x4 v_X1, v_Y1; { v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_its / v_W, v_zero); - v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero); + v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_its / v_W, v_zero); - v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero); + v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_X1 = v_round(v_fX0, v_fX1); v_Y1 = v_round(v_fY0, v_fY1); @@ -3025,16 +3025,16 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph v_int32x4 v_X2, v_Y2; { v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_its / v_W, v_zero); - v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero); + v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_its / v_W, v_zero); - v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero); + v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_X2 = v_round(v_fX0, v_fX1); v_Y2 = v_round(v_fY0, v_fY1); @@ -3044,35 +3044,35 @@ void WarpPerspectiveLine_Process_CV_SIMD(const double *M, short* xy, short* alph v_int32x4 v_X3, v_Y3; { v_float64x2 v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_its / v_W, v_zero); - v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero); + v_float64x2 v_fX0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY0 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_W = v_muladd(v_M6, v_x1, v_W0); - v_W = v_select(v_W != v_zero, v_its / v_W, v_zero); - v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M0, v_x1, v_X0d) * v_W)); - v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_muladd(v_M3, v_x1, v_Y0d) * v_W)); - v_x1 += v_2; + v_W = v_select(v_ne(v_W, v_zero), v_div(v_its, v_W), v_zero); + v_float64x2 v_fX1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M0, v_x1, v_X0d), v_W))); + v_float64x2 v_fY1 = v_max(v_intmin, v_min(v_intmax, v_mul(v_muladd(v_M3, v_x1, v_Y0d), v_W))); + v_x1 = v_add(v_x1, v_2); v_X3 = v_round(v_fX0, v_fX1); v_Y3 = v_round(v_fY0, v_fY1); } // store alpha - v_int32x4 v_alpha0 = ((v_Y0 & v_itsi1) << INTER_BITS) + (v_X0 & v_itsi1); - v_int32x4 v_alpha1 = ((v_Y1 & v_itsi1) << INTER_BITS) + (v_X1 & v_itsi1); + v_int32x4 v_alpha0 = v_add(v_shl(v_and(v_Y0, v_itsi1)), v_and(v_X0, v_itsi1)); + v_int32x4 v_alpha1 = v_add(v_shl(v_and(v_Y1, v_itsi1)), v_and(v_X1, v_itsi1)); v_store((alpha + x1), v_pack(v_alpha0, v_alpha1)); - v_alpha0 = ((v_Y2 & v_itsi1) << INTER_BITS) + (v_X2 & v_itsi1); - v_alpha1 = ((v_Y3 & v_itsi1) << INTER_BITS) + (v_X3 & v_itsi1); + v_alpha0 = v_add(v_shl(v_and(v_Y2, v_itsi1)), v_and(v_X2, v_itsi1)); + v_alpha1 = v_add(v_shl(v_and(v_Y3, v_itsi1)), v_and(v_X3, v_itsi1)); v_store((alpha + x1 + 8), v_pack(v_alpha0, v_alpha1)); // convert to 16s - v_X0 = v_reinterpret_as_s32(v_pack(v_X0 >> INTER_BITS, v_X1 >> INTER_BITS)); - v_X1 = v_reinterpret_as_s32(v_pack(v_X2 >> INTER_BITS, v_X3 >> INTER_BITS)); - v_Y0 = v_reinterpret_as_s32(v_pack(v_Y0 >> INTER_BITS, v_Y1 >> INTER_BITS)); - v_Y1 = v_reinterpret_as_s32(v_pack(v_Y2 >> INTER_BITS, v_Y3 >> INTER_BITS)); + v_X0 = v_reinterpret_as_s32(v_pack(v_shr(v_X0), v_shr(v_X1))); + v_X1 = v_reinterpret_as_s32(v_pack(v_shr(v_X2), v_shr(v_X3))); + v_Y0 = v_reinterpret_as_s32(v_pack(v_shr(v_Y0), v_shr(v_Y1))); + v_Y1 = v_reinterpret_as_s32(v_pack(v_shr(v_Y2), v_shr(v_Y3))); v_store_interleave(xy + x1 * 2, (v_reinterpret_as_s16)(v_X0), (v_reinterpret_as_s16)(v_Y0)); v_store_interleave(xy + x1 * 2 + 16, (v_reinterpret_as_s16)(v_X1), (v_reinterpret_as_s16)(v_Y1)); diff --git a/modules/imgproc/src/median_blur.simd.hpp b/modules/imgproc/src/median_blur.simd.hpp index 7d8423d322..1fe2e4060c 100644 --- a/modules/imgproc/src/median_blur.simd.hpp +++ b/modules/imgproc/src/median_blur.simd.hpp @@ -179,10 +179,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize ) for (k = 0; k < 16; ++k) { #if CV_SIMD256 - v_store(H.fine[k], v_mul_wrap(v256_load(h_fine + 16 * n*(16 * c + k)), v256_setall_u16(2 * r + 1)) + v256_load(H.fine[k])); + v_store(H.fine[k], v_mul_wrap(v256_load(h_fine + 16 * n*(16 * c + k)), v_add(v256_setall_u16(2 * r + 1), v256_load(H.fine[k])))); #elif CV_SIMD128 - v_store(H.fine[k], v_mul_wrap(v_load(h_fine + 16 * n*(16 * c + k)), v_setall_u16((ushort)(2 * r + 1))) + v_load(H.fine[k])); - v_store(H.fine[k] + 8, v_mul_wrap(v_load(h_fine + 16 * n*(16 * c + k) + 8), v_setall_u16((ushort)(2 * r + 1))) + v_load(H.fine[k] + 8)); + v_store(H.fine[k], v_add(v_mul_wrap(v_load(h_fine + 16 * n * (16 * c + k)), v_setall_u16((ushort)(2 * r + 1))), v_load(H.fine[k]))); + v_store(H.fine[k] + 8, v_add(v_mul_wrap(v_load(h_fine + 16 * n * (16 * c + k) + 8), v_setall_u16((ushort)(2 * r + 1))), v_load(H.fine[k] + 8))); #else for (int ind = 0; ind < 16; ++ind) H.fine[k][ind] = (HT)(H.fine[k][ind] + (2 * r + 1) * h_fine[16 * n*(16 * c + k) + ind]); @@ -199,10 +199,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize ) for( j = 0; j < 2*r; ++j, px += 16 ) { #if CV_SIMD256 - v_coarse += v256_load(px); + v_coarse = v_add(v_coarse, v256_load(px)); #elif CV_SIMD128 - v_coarsel += v_load(px); - v_coarseh += v_load(px + 8); + v_coarsel = v_add(v_coarsel, v_load(px)); + v_coarseh = v_add(v_coarseh, v_load(px + 8)); #else for (int ind = 0; ind < 16; ++ind) H.coarse[ind] += px[ind]; @@ -216,11 +216,11 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize ) px = h_coarse + 16 * (n*c + std::min(j + r, n - 1)); #if CV_SIMD256 - v_coarse += v256_load(px); + v_coarse = v_add(v_coarse, v256_load(px)); v_store(H.coarse, v_coarse); #elif CV_SIMD128 - v_coarsel += v_load(px); - v_coarseh += v_load(px + 8); + v_coarsel = v_add(v_coarsel, v_load(px)); + v_coarseh = v_add(v_coarseh, v_load(px + 8)); v_store(H.coarse, v_coarsel); v_store(H.coarse + 8, v_coarseh); #else @@ -261,10 +261,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize ) for (luc[k] = HT(j - r); luc[k] < MIN(j + r + 1, n); ++luc[k], px += 16) { #if CV_SIMD256 - v_fine += v256_load(px); + v_fine = v_add(v_fine, v256_load(px)); #elif CV_SIMD128 - v_finel += v_load(px); - v_fineh += v_load(px + 8); + v_finel = v_add(v_finel, v_load(px)); + v_fineh = v_add(v_fineh, v_load(px + 8)); #else for (int ind = 0; ind < 16; ++ind) H.fine[k][ind] += px[ind]; @@ -275,10 +275,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize ) { px = h_fine + 16 * (n*(16 * c + k) + (n - 1)); #if CV_SIMD256 - v_fine += v_mul_wrap(v256_load(px), v256_setall_u16(j + r + 1 - n)); + v_fine = v_add(v_fine, v_mul_wrap(v256_load(px), v256_setall_u16(j + r + 1 - n))); #elif CV_SIMD128 - v_finel += v_mul_wrap(v_load(px), v_setall_u16((ushort)(j + r + 1 - n))); - v_fineh += v_mul_wrap(v_load(px + 8), v_setall_u16((ushort)(j + r + 1 - n))); + v_finel = v_add(v_finel, v_mul_wrap(v_load(px), v_setall_u16((ushort)(j + r + 1 - n)))); + v_fineh = v_add(v_fineh, v_mul_wrap(v_load(px + 8), v_setall_u16((ushort)(j + r + 1 - n)))); #else for (int ind = 0; ind < 16; ++ind) H.fine[k][ind] = (HT)(H.fine[k][ind] + (j + r + 1 - n) * px[ind]); @@ -298,10 +298,10 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize ) for ( ; luc[k] < j+r+1; ++luc[k] ) { #if CV_SIMD256 - v_fine = v_fine + v256_load(px + 16 * MIN(luc[k], n - 1)) - v256_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0)); + v_fine = v_sub(v_add(v_fine, v256_load(px + 16 * MIN(luc[k], n - 1))), v256_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0))); #elif CV_SIMD128 - v_finel = v_finel + v_load(px + 16 * MIN(luc[k], n - 1) ) - v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0)); - v_fineh = v_fineh + v_load(px + 16 * MIN(luc[k], n - 1) + 8) - v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0) + 8); + v_finel = v_sub(v_add(v_finel, v_load(px + 16 * MIN(luc[k], n - 1) )), v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0))); + v_fineh = v_sub(v_add(v_fineh, v_load(px + 16 * MIN(luc[k], n - 1) + 8)), v_load(px + 16 * MAX(luc[k] - 2 * r - 1, 0) + 8)); #else for (int ind = 0; ind < 16; ++ind) H.fine[k][ind] += px[16 * MIN(luc[k], n - 1) + ind] - px[16 * MAX(luc[k] - 2 * r - 1, 0) + ind]; @@ -312,12 +312,12 @@ medianBlur_8u_O1( const Mat& _src, Mat& _dst, int ksize ) px = h_coarse + 16 * (n*c + MAX(j - r, 0)); #if CV_SIMD256 v_store(H.fine[k], v_fine); - v_coarse -= v256_load(px); + v_coarse = v_sub(v_coarse, v256_load(px)); #elif CV_SIMD128 v_store(H.fine[k], v_finel); v_store(H.fine[k] + 8, v_fineh); - v_coarsel -= v_load(px); - v_coarseh -= v_load(px + 8); + v_coarsel = v_sub(v_coarsel, v_load(px)); + v_coarseh = v_sub(v_coarseh, v_load(px + 8)); #else for (int ind = 0; ind < 16; ++ind) H.coarse[ind] -= px[ind]; diff --git a/modules/imgproc/src/moments.cpp b/modules/imgproc/src/moments.cpp index 964e78cfb1..2bca8701f8 100644 --- a/modules/imgproc/src/moments.cpp +++ b/modules/imgproc/src/moments.cpp @@ -236,12 +236,12 @@ struct MomentsInTile_SIMD v_int16x8 p = v_reinterpret_as_s16(v_load_expand(ptr + x)); v_int16x8 sx = v_mul_wrap(qx, qx); - qx0 += v_reinterpret_as_u32(p); + qx0 = v_add(qx0, v_reinterpret_as_u32(p)); qx1 = v_reinterpret_as_u32(v_dotprod(p, qx, v_reinterpret_as_s32(qx1))); qx2 = v_reinterpret_as_u32(v_dotprod(p, sx, v_reinterpret_as_s32(qx2))); qx3 = v_reinterpret_as_u32(v_dotprod(v_mul_wrap(p, qx), sx, v_reinterpret_as_s32(qx3))); - qx += dx; + qx = v_add(qx, dx); } x0 = v_reduce_sum(qx0); @@ -276,19 +276,19 @@ struct MomentsInTile_SIMD { v_int32x4 v_src = v_reinterpret_as_s32(v_load_expand(ptr + x)); - v_x0 += v_reinterpret_as_u32(v_src); - v_x1 += v_reinterpret_as_u32(v_src * v_ix0); + v_x0 = v_add(v_x0, v_reinterpret_as_u32(v_src)); + v_x1 = v_add(v_x1, v_reinterpret_as_u32(v_mul(v_src, v_ix0))); - v_int32x4 v_ix1 = v_ix0 * v_ix0; - v_x2 += v_reinterpret_as_u32(v_src * v_ix1); + v_int32x4 v_ix1 = v_mul(v_ix0, v_ix0); + v_x2 = v_add(v_x2, v_reinterpret_as_u32(v_mul(v_src, v_ix1))); - v_ix1 = v_ix0 * v_ix1; - v_src = v_src * v_ix1; + v_ix1 = v_mul(v_ix0, v_ix1); + v_src = v_mul(v_src, v_ix1); v_uint64x2 v_lo, v_hi; v_expand(v_reinterpret_as_u32(v_src), v_lo, v_hi); - v_x3 += v_lo + v_hi; + v_x3 = v_add(v_x3, v_add(v_lo, v_hi)); - v_ix0 += v_delta; + v_ix0 = v_add(v_ix0, v_delta); } x0 = v_reduce_sum(v_x0); diff --git a/modules/imgproc/src/pyramids.cpp b/modules/imgproc/src/pyramids.cpp index f1544bd5e7..9d6d76c812 100644 --- a/modules/imgproc/src/pyramids.cpp +++ b/modules/imgproc/src/pyramids.cpp @@ -463,7 +463,7 @@ template<> int PyrDownVecV(int** src, uchar* dst, int width) } #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; - for ( ; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) + for ( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32x4 r0, r1, r2, r3, r4, t0; r0 = v_load(row0 + x); @@ -473,7 +473,7 @@ template<> int PyrDownVecV(int** src, uchar* dst, int width) r4 = v_load(row4 + x); t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); - *((unaligned_int*) (dst + x)) = v_reinterpret_as_s32(v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16())).get0(); + *((unaligned_int*) (dst + x)) = v_get0(v_reinterpret_as_s32(v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16()))); } #else for (; x <= width - 1; x += 1) @@ -615,15 +615,15 @@ template <> int PyrUpVecV(int** src, uchar** dst, int width) } #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; - for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); v_int32 v_2r10 = v_add(v_r10, v_r10); v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20))); - *(unaligned_int*)(dst0 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0(); - *(unaligned_int*)(dst1 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16())).get0(); + *(unaligned_int*)(dst0 + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16()))); + *(unaligned_int*)(dst1 + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16()))); } #else for (; x <= width - 1; x += 1) @@ -754,14 +754,14 @@ template <> int PyrUpVecVOneRow(int** src, uchar* dst, int width) } #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; - for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); v_int32 v_2r10 = v_add(v_r10, v_r10); v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20))); - *(unaligned_int*)(dst + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0(); + *(unaligned_int*)(dst + x) = v_get0(v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16()))); } #else for (; x <= width - 1; x += 1) diff --git a/modules/imgproc/src/resize.cpp b/modules/imgproc/src/resize.cpp index bba8cda4f7..a1d4433f68 100644 --- a/modules/imgproc/src/resize.cpp +++ b/modules/imgproc/src/resize.cpp @@ -2473,7 +2473,7 @@ public: v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_uint16 bl, gl, rl; #if CV_SIMD_WIDTH == 16 - bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; + bl = v_add(t0, t3); gl = v_add(t1, t4); rl = v_add(t2, t5); #elif CV_SIMD_WIDTH == 32 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bl = v_add(s0, s3); gl = v_add(s1, s4); rl = v_add(s2, s5); @@ -2493,7 +2493,7 @@ public: v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_uint16 bh, gh, rh; #if CV_SIMD_WIDTH == 16 - bh = t0 + t3; gh = t1 + t4; rh = t2 + t5; + bh = v_add(t0, t3); gh = v_add(t1, t4); rh = v_add(t2, t5); #elif CV_SIMD_WIDTH == 32 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bh = v_add(s0, s3); gh = v_add(s1, s4); rh = v_add(s2, s5); @@ -2566,7 +2566,7 @@ public: v_rshr_pack_store<2>(D, r0 + v_rotate_left<1>(r1, r0)); } #else - v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3)); + v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_load_expand(S0), v_load_expand(S0 + 3)), v_load_expand(S1)), v_load_expand(S1 + 3))); #endif #elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64 for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) @@ -2609,7 +2609,7 @@ public: } #elif CV_SIMD_WIDTH >= 64 v_uint32 masklow = vx_setall_u32(0x0000ffff); - for ( ; dx <= w - 3*v_uint16::nlanes; dx += 3*v_uint16::nlanes, S0 += 6*v_uint16::nlanes, S1 += 6*v_uint16::nlanes, D += 3*v_uint16::nlanes) + for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { v_uint16 b0, g0, r0, b1, g1, r1; v_load_deinterleave(S0, b0, g0, r0); @@ -2617,8 +2617,8 @@ public: v_uint32 bl = (v_reinterpret_as_u32(b0) >> 16) + (v_reinterpret_as_u32(b0) & masklow) + (v_reinterpret_as_u32(b1) >> 16) + (v_reinterpret_as_u32(b1) & masklow); v_uint32 gl = (v_reinterpret_as_u32(g0) >> 16) + (v_reinterpret_as_u32(g0) & masklow) + (v_reinterpret_as_u32(g1) >> 16) + (v_reinterpret_as_u32(g1) & masklow); v_uint32 rl = (v_reinterpret_as_u32(r0) >> 16) + (v_reinterpret_as_u32(r0) & masklow) + (v_reinterpret_as_u32(r1) >> 16) + (v_reinterpret_as_u32(r1) & masklow); - v_load_deinterleave(S0 + 3*v_uint16::nlanes, b0, g0, r0); - v_load_deinterleave(S1 + 3*v_uint16::nlanes, b1, g1, r1); + v_load_deinterleave(S0 + 3*VTraits::vlanes(), b0, g0, r0); + v_load_deinterleave(S1 + 3*VTraits::vlanes(), b1, g1, r1); v_uint32 bh = (v_reinterpret_as_u32(b0) >> 16) + (v_reinterpret_as_u32(b0) & masklow) + (v_reinterpret_as_u32(b1) >> 16) + (v_reinterpret_as_u32(b1) & masklow); v_uint32 gh = (v_reinterpret_as_u32(g0) >> 16) + (v_reinterpret_as_u32(g0) & masklow) + (v_reinterpret_as_u32(g1) >> 16) + (v_reinterpret_as_u32(g1) & masklow); v_uint32 rh = (v_reinterpret_as_u32(r0) >> 16) + (v_reinterpret_as_u32(r0) & masklow) + (v_reinterpret_as_u32(r1) >> 16) + (v_reinterpret_as_u32(r1) & masklow); @@ -2630,7 +2630,7 @@ public: { CV_Assert(cn == 4); #if CV_SIMD_WIDTH >= 64 - for ( ; dx <= w - v_uint16::nlanes; dx += v_uint16::nlanes, S0 += 2*v_uint16::nlanes, S1 += 2*v_uint16::nlanes, D += v_uint16::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2*VTraits::vlanes(), S1 += 2*VTraits::vlanes(), D += VTraits::vlanes()) { v_uint64 r00, r01, r10, r11; v_load_deinterleave((uint64_t*)S0, r00, r01); @@ -2652,7 +2652,7 @@ public: r0 = v_add(r0, r2); r1 = v_add(r1, r3); v_uint32 v_d; #if CV_SIMD_WIDTH == 16 - v_d = r0 + r1; + v_d = v_add(r0, r1); #elif CV_SIMD_WIDTH == 32 v_uint32 t0, t1; v_recombine(r0, r1, t0, t1); @@ -2697,7 +2697,7 @@ public: { #if CV_SIMD_WIDTH == 16 for ( ; dx <= w - 4; dx += 3, S0 += 6, S1 += 6, D += 3) - v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3)); + v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_load_expand(S0), v_load_expand(S0 + 3)), v_load_expand(S1)), v_load_expand(S1 + 3))); #elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64 for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { @@ -2738,7 +2738,7 @@ public: v_store_interleave(D, v_rshr_pack<2>(bl, bh), v_rshr_pack<2>(gl, gh), v_rshr_pack<2>(rl, rh)); } #elif CV_SIMD_WIDTH >= 64 - for ( ; dx <= w - 3*v_int16::nlanes; dx += 3*v_int16::nlanes, S0 += 6*v_int16::nlanes, S1 += 6*v_int16::nlanes, D += 3*v_int16::nlanes) + for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { v_int16 b0, g0, r0, b1, g1, r1; v_load_deinterleave(S0, b0, g0, r0); @@ -2746,8 +2746,8 @@ public: v_int32 bl = (v_reinterpret_as_s32(b0) >> 16) + ((v_reinterpret_as_s32(b0) << 16) >> 16) + (v_reinterpret_as_s32(b1) >> 16) + ((v_reinterpret_as_s32(b1) << 16) >> 16); v_int32 gl = (v_reinterpret_as_s32(g0) >> 16) + ((v_reinterpret_as_s32(g0) << 16) >> 16) + (v_reinterpret_as_s32(g1) >> 16) + ((v_reinterpret_as_s32(g1) << 16) >> 16); v_int32 rl = (v_reinterpret_as_s32(r0) >> 16) + ((v_reinterpret_as_s32(r0) << 16) >> 16) + (v_reinterpret_as_s32(r1) >> 16) + ((v_reinterpret_as_s32(r1) << 16) >> 16); - v_load_deinterleave(S0 + 3*v_int16::nlanes, b0, g0, r0); - v_load_deinterleave(S1 + 3*v_int16::nlanes, b1, g1, r1); + v_load_deinterleave(S0 + 3*VTraits::vlanes(), b0, g0, r0); + v_load_deinterleave(S1 + 3*VTraits::vlanes(), b1, g1, r1); v_int32 bh = (v_reinterpret_as_s32(b0) >> 16) + ((v_reinterpret_as_s32(b0) << 16) >> 16) + (v_reinterpret_as_s32(b1) >> 16) + ((v_reinterpret_as_s32(b1) << 16) >> 16); v_int32 gh = (v_reinterpret_as_s32(g0) >> 16) + ((v_reinterpret_as_s32(g0) << 16) >> 16) + (v_reinterpret_as_s32(g1) >> 16) + ((v_reinterpret_as_s32(g1) << 16) >> 16); v_int32 rh = (v_reinterpret_as_s32(r0) >> 16) + ((v_reinterpret_as_s32(r0) << 16) >> 16) + (v_reinterpret_as_s32(r1) >> 16) + ((v_reinterpret_as_s32(r1) << 16) >> 16); @@ -2779,7 +2779,7 @@ public: r3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); v_int32 dl, dh; #if CV_SIMD_WIDTH == 16 - dl = r0 + r1; dh = r2 + r3; + dl = v_add(r0, r1); dh = v_add(r2, r3); #elif CV_SIMD_WIDTH == 32 v_int32 t0, t1, t2, t3; v_recombine(r0, r1, t0, t1); v_recombine(r2, r3, t2, t3); @@ -2829,14 +2829,14 @@ struct ResizeAreaFastVec_SIMD_32f { #if CV_SIMD_WIDTH == 16 v_float32 v_025 = vx_setall_f32(0.25f); - for (; dx <= w - v_float32::nlanes; dx += v_float32::nlanes, S0 += 2*v_float32::nlanes, S1 += 2*v_float32::nlanes, D += v_float32::nlanes) - v_store(D, ((vx_load(S0) + vx_load(S0 + v_float32::nlanes)) + (vx_load(S1) + vx_load(S1 + v_float32::nlanes))) * v_025); + for (; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2*VTraits::vlanes(), S1 += 2*VTraits::vlanes(), D += VTraits::vlanes()) + v_store(D, v_mul(v_add(v_add(vx_load(S0), vx_load(S0 + VTraits::vlanes())), v_add(vx_load(S1), vx_load(S1 + VTraits::vlanes()))), v_025)); #elif CV_SIMD256 v_float32x8 v_025 = v256_setall_f32(0.25f); - for (; dx <= w - v_float32x8::nlanes; dx += v_float32x8::nlanes, S0 += 2*v_float32x8::nlanes, S1 += 2*v_float32x8::nlanes, D += v_float32x8::nlanes) + for (; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2*VTraits::vlanes(), S1 += 2*VTraits::vlanes(), D += VTraits::vlanes()) { v_float32x8 dst0, dst1; - v_recombine(v_add(v256_load(S0), v256_load(S1)), v_add(v256_load(S0 + v_float32x8::nlanes), v256_load(S1 + v_float32x8::nlanes)), dst0, dst1); + v_recombine(v_add(v256_load(S0), v256_load(S1)), v_add(v256_load(S0 + VTraits::vlanes()), v256_load(S1 + VTraits::vlanes())), dst0, dst1); v_store(D, v_mul(v_add(dst0, dst1), v_025)); } #endif @@ -3019,6 +3019,111 @@ struct DecimateAlpha }; +namespace inter_area { +#if (CV_SIMD || CV_SIMD_SCALABLE) +inline void saturate_store(const float* src, uchar* dst) { + const v_int32 tmp0 = v_round(vx_load(src + 0 * VTraits::vlanes())); + const v_int32 tmp1 = v_round(vx_load(src + 1 * VTraits::vlanes())); + const v_int32 tmp2 = v_round(vx_load(src + 2 * VTraits::vlanes())); + const v_int32 tmp3 = v_round(vx_load(src + 3 * VTraits::vlanes())); + v_store(dst, v_pack(v_pack_u(tmp0, tmp1), v_pack_u(tmp2, tmp3))); +} + +inline void saturate_store(const float* src, ushort* dst) { + const v_int32 tmp0 = v_round(vx_load(src + 0 * VTraits::vlanes())); + const v_int32 tmp1 = v_round(vx_load(src + 1 * VTraits::vlanes())); + v_store(dst, v_pack_u(tmp0, tmp1)); +} + +inline void saturate_store(const float* src, short* dst) { + const v_int32 tmp0 = v_round(vx_load(src + 0 * VTraits::vlanes())); + const v_int32 tmp1 = v_round(vx_load(src + 1 * VTraits::vlanes())); + v_store(dst, v_pack(tmp0, tmp1)); +} + +static inline v_float32 vx_setall(float coeff) { return vx_setall_f32(coeff); } + +template +struct VArea {}; + +template <> +struct VArea { + typedef v_float32 vWT; +}; +#endif + +#if (CV_SIMD128_64F || CV_SIMD_SCALABLE_64F) +static inline v_float64 vx_setall(double coeff) { return vx_setall_f64(coeff); } + +template <> +struct VArea { + typedef v_float64 vWT; +}; + +#else +inline void mul(const double* buf, int width, double beta, double* sum) { + for (int dx = 0; dx < width; ++dx) { + sum[dx] = beta * buf[dx]; + } +} + +inline void muladd(const double* buf, int width, double beta, double* sum) { + for (int dx = 0; dx < width; ++dx) { + sum[dx] += beta * buf[dx]; + } +} +#endif + +template +inline void saturate_store(const WT* sum, int width, T* D) { + int dx = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int step = VTraits::vWT>::vlanes() * sizeof(WT) / sizeof(T); + for (; dx + step < width; dx += step) { + saturate_store(sum + dx, D + dx); + } +#endif + for (; dx < width; ++dx) { + D[dx] = saturate_cast(sum[dx]); + } +} + +// Optimization when T == WT. +template +inline void saturate_store(const WT* sum, int width, WT* D) { + std::copy(sum, sum + width, D); +} + +template +inline void mul(const WT* buf, int width, WT beta, WT* sum) { + int dx = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int step = VTraits::vWT>::vlanes(); + for (; dx + step < width; dx += step) { + vx_store(sum + dx, v_mul(vx_setall(beta), vx_load(buf + dx))); + } +#endif + for (; dx < width; ++dx) { + sum[dx] = beta * buf[dx]; + } +} + +template +inline void muladd(const WT* buf, int width, WT beta, WT* sum) { + int dx = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int step = VTraits::vWT>::vlanes(); + for (; dx + step < width; dx += step) { + vx_store(sum + dx, v_add(vx_load(sum + dx), v_mul(vx_setall(beta), vx_load(buf + dx)))); + } +#endif + for (; dx < width; ++dx) { + sum[dx] += beta * buf[dx]; + } +} + +} // namespace inter_area + template class ResizeArea_Invoker : public ParallelLoopBody { @@ -3120,27 +3225,17 @@ public: if( dy != prev_dy ) { - T* D = dst->template ptr(prev_dy); - - for( dx = 0; dx < dsize.width; dx++ ) - { - D[dx] = saturate_cast(sum[dx]); - sum[dx] = beta*buf[dx]; - } + inter_area::saturate_store(sum, dsize.width, dst->template ptr(prev_dy)); + inter_area::mul(buf, dsize.width, beta, sum); prev_dy = dy; } else { - for( dx = 0; dx < dsize.width; dx++ ) - sum[dx] += beta*buf[dx]; + inter_area::muladd(buf, dsize.width, beta, sum); } } - { - T* D = dst->template ptr(prev_dy); - for( dx = 0; dx < dsize.width; dx++ ) - D[dx] = saturate_cast(sum[dx]); - } + inter_area::saturate_store(sum, dsize.width, dst->template ptr(prev_dy)); } private: diff --git a/modules/imgproc/src/sumpixels.simd.hpp b/modules/imgproc/src/sumpixels.simd.hpp index f5f3a92d85..208ffc1231 100644 --- a/modules/imgproc/src/sumpixels.simd.hpp +++ b/modules/imgproc/src/sumpixels.simd.hpp @@ -114,7 +114,7 @@ struct Integral_SIMD v_int32 prev = vx_setzero_s32(); int j = 0; - for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes) + for ( ; j + VTraits::vlanes() <= width; j += VTraits::vlanes()) { v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j)); v_int32 el4l, el4h; @@ -127,8 +127,8 @@ struct Integral_SIMD el4h.val = _mm256_add_epi32(_mm256_cvtepi16_epi32(_v256_extract_high(vsum)), _mm256_permutevar8x32_epi32(el4l.val, shmask)); prev.val = _mm256_permutevar8x32_epi32(el4h.val, shmask); #else - el8 += v_rotate_left<1>(el8); - el8 += v_rotate_left<2>(el8); + el8 = v_add(el8, v_rotate_left<1>(el8)); + el8 = v_add(el8, v_rotate_left<2>(el8)); #if CV_SIMD_WIDTH >= 32 el8 += v_rotate_left<4>(el8); #if CV_SIMD_WIDTH == 64 @@ -136,12 +136,12 @@ struct Integral_SIMD #endif #endif v_expand(el8, el4l, el4h); - el4l += prev; - el4h += el4l; - prev = v_broadcast_element(el4h); + el4l = v_add(el4l, prev); + el4h = v_add(el4h, el4l); + prev = v_broadcast_highest(el4h); #endif - v_store(sum_row + j , el4l + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_int32::nlanes, el4h + vx_load(prev_sum_row + j + v_int32::nlanes)); + v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits::vlanes()))); } for (int v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j) @@ -162,11 +162,11 @@ struct Integral_SIMD v_int32 prev_1 = vx_setzero_s32(), prev_2 = vx_setzero_s32(); int j = 0; - for ( ; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn) + for ( ; j + VTraits::vlanes() * cn <= width; j += VTraits::vlanes() * cn) { v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j)); - v_int16 el8_1 = v_src_row & mask; - v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8); + v_int16 el8_1 = v_and(v_src_row, mask); + v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row))); v_int32 el4l_1, el4h_1, el4l_2, el4h_2; #if CV_AVX2 && CV_SIMD_WIDTH == 32 __m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2)); @@ -183,10 +183,10 @@ struct Integral_SIMD prev_1.val = _mm256_permutevar8x32_epi32(el4h_1.val, shmask); prev_2.val = _mm256_permutevar8x32_epi32(el4h_2.val, shmask); #else - el8_1 += v_rotate_left<1>(el8_1); - el8_2 += v_rotate_left<1>(el8_2); - el8_1 += v_rotate_left<2>(el8_1); - el8_2 += v_rotate_left<2>(el8_2); + el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2)); + el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2)); #if CV_SIMD_WIDTH >= 32 el8_1 += v_rotate_left<4>(el8_1); el8_2 += v_rotate_left<4>(el8_2); @@ -197,20 +197,20 @@ struct Integral_SIMD #endif v_expand(el8_1, el4l_1, el4h_1); v_expand(el8_2, el4l_2, el4h_2); - el4l_1 += prev_1; - el4l_2 += prev_2; - el4h_1 += el4l_1; - el4h_2 += el4l_2; - prev_1 = v_broadcast_element(el4h_1); - prev_2 = v_broadcast_element(el4h_2); + el4l_1 = v_add(el4l_1, prev_1); + el4l_2 = v_add(el4l_2, prev_2); + el4h_1 = v_add(el4h_1, el4l_1); + el4h_2 = v_add(el4h_2, el4l_2); + prev_1 = v_broadcast_highest(el4h_1); + prev_2 = v_broadcast_highest(el4h_2); #endif v_int32 el4_1, el4_2, el4_3, el4_4; v_zip(el4l_1, el4l_2, el4_1, el4_2); v_zip(el4h_1, el4h_2, el4_3, el4_4); - v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_int32::nlanes , el4_2 + vx_load(prev_sum_row + j + v_int32::nlanes )); - v_store(sum_row + j + v_int32::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 2)); - v_store(sum_row + j + v_int32::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_int32::nlanes * 3)); + v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits::vlanes()))); + v_store(sum_row + j + VTraits::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 2))); + v_store(sum_row + j + VTraits::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits::vlanes() * 3))); } for (int v2 = sum_row[j - 1] - prev_sum_row[j - 1], @@ -230,7 +230,7 @@ struct Integral_SIMD const uchar * src_row = src + _srcstep * i; int * prev_sum_row = (int *)((uchar *)sum + _sumstep * i) + cn; int * sum_row = (int *)((uchar *)sum + _sumstep * (i + 1)) + cn; - int row_cache[v_int32::nlanes * 6]; + int row_cache[VTraits::max_nlanes * 6]; sum_row[-1] = sum_row[-2] = sum_row[-3] = 0; @@ -238,10 +238,10 @@ struct Integral_SIMD prev_3 = vx_setzero_s32(); int j = 0; const int j_max = - ((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height) - ? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave() - : width - v_uint16::nlanes * cn; // v_expand_low - for ( ; j <= j_max; j += v_uint16::nlanes * cn) + ((_srcstep * i + (width - VTraits::vlanes() * cn + VTraits::vlanes() * cn)) >= _srcstep * height) + ? width - VTraits::vlanes() * cn // uint8 in v_load_deinterleave() + : width - VTraits::vlanes() * cn; // v_expand_low + for ( ; j <= j_max; j += VTraits::vlanes() * cn) { v_uint8 v_src_row_1, v_src_row_2, v_src_row_3; v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3); @@ -270,49 +270,49 @@ struct Integral_SIMD prev_2.val = _mm256_permutevar8x32_epi32(el4h_2.val, shmask); prev_3.val = _mm256_permutevar8x32_epi32(el4h_3.val, shmask); #else - el8_1 += v_rotate_left<1>(el8_1); - el8_2 += v_rotate_left<1>(el8_2); - el8_3 += v_rotate_left<1>(el8_3); - el8_1 += v_rotate_left<2>(el8_1); - el8_2 += v_rotate_left<2>(el8_2); - el8_3 += v_rotate_left<2>(el8_3); + el8_1 = v_add(el8_1,v_rotate_left<1>(el8_1)); + el8_2 = v_add(el8_2,v_rotate_left<1>(el8_2)); + el8_3 = v_add(el8_3,v_rotate_left<1>(el8_3)); + el8_1 = v_add(el8_1,v_rotate_left<2>(el8_1)); + el8_2 = v_add(el8_2,v_rotate_left<2>(el8_2)); + el8_3 = v_add(el8_3,v_rotate_left<2>(el8_3)); #if CV_SIMD_WIDTH >= 32 - el8_1 += v_rotate_left<4>(el8_1); - el8_2 += v_rotate_left<4>(el8_2); - el8_3 += v_rotate_left<4>(el8_3); + el8_1 = v_add(el8_1, v_rotate_left<4>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<4>(el8_2)); + el8_3 = v_add(el8_3, v_rotate_left<4>(el8_3)); #if CV_SIMD_WIDTH == 64 - el8_1 += v_rotate_left<8>(el8_1); - el8_2 += v_rotate_left<8>(el8_2); - el8_3 += v_rotate_left<8>(el8_3); + el8_1 = v_add(el8_1, v_rotate_left<8>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<8>(el8_2)); + el8_3 = v_add(el8_3, v_rotate_left<8>(el8_3)); #endif #endif v_expand(el8_1, el4l_1, el4h_1); v_expand(el8_2, el4l_2, el4h_2); v_expand(el8_3, el4l_3, el4h_3); - el4l_1 += prev_1; - el4l_2 += prev_2; - el4l_3 += prev_3; - el4h_1 += el4l_1; - el4h_2 += el4l_2; - el4h_3 += el4l_3; - prev_1 = v_broadcast_element(el4h_1); - prev_2 = v_broadcast_element(el4h_2); - prev_3 = v_broadcast_element(el4h_3); + el4l_1 = v_add(el4l_1, prev_1); + el4l_2 = v_add(el4l_2, prev_2); + el4l_3 = v_add(el4l_3, prev_3); + el4h_1 = v_add(el4h_1, el4l_1); + el4h_2 = v_add(el4h_2, el4l_2); + el4h_3 = v_add(el4h_3, el4l_3); + prev_1 = v_broadcast_highest(el4h_1); + prev_2 = v_broadcast_highest(el4h_2); + prev_3 = v_broadcast_highest(el4h_3); #endif v_store_interleave(row_cache , el4l_1, el4l_2, el4l_3); - v_store_interleave(row_cache + v_int32::nlanes * 3, el4h_1, el4h_2, el4h_3); + v_store_interleave(row_cache + VTraits::vlanes() * 3, el4h_1, el4h_2, el4h_3); el4l_1 = vx_load(row_cache ); - el4l_2 = vx_load(row_cache + v_int32::nlanes ); - el4l_3 = vx_load(row_cache + v_int32::nlanes * 2); - el4h_1 = vx_load(row_cache + v_int32::nlanes * 3); - el4h_2 = vx_load(row_cache + v_int32::nlanes * 4); - el4h_3 = vx_load(row_cache + v_int32::nlanes * 5); - v_store(sum_row + j , el4l_1 + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_int32::nlanes , el4l_2 + vx_load(prev_sum_row + j + v_int32::nlanes )); - v_store(sum_row + j + v_int32::nlanes * 2, el4l_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 2)); - v_store(sum_row + j + v_int32::nlanes * 3, el4h_1 + vx_load(prev_sum_row + j + v_int32::nlanes * 3)); - v_store(sum_row + j + v_int32::nlanes * 4, el4h_2 + vx_load(prev_sum_row + j + v_int32::nlanes * 4)); - v_store(sum_row + j + v_int32::nlanes * 5, el4h_3 + vx_load(prev_sum_row + j + v_int32::nlanes * 5)); + el4l_2 = vx_load(row_cache + VTraits::vlanes() ); + el4l_3 = vx_load(row_cache + VTraits::vlanes() * 2); + el4h_1 = vx_load(row_cache + VTraits::vlanes() * 3); + el4h_2 = vx_load(row_cache + VTraits::vlanes() * 4); + el4h_3 = vx_load(row_cache + VTraits::vlanes() * 5); + v_store(sum_row + j , v_add(el4l_1, vx_load(prev_sum_row + j ))); + v_store(sum_row + j + VTraits::vlanes() , v_add(el4l_2, vx_load(prev_sum_row + j + VTraits::vlanes() ))); + v_store(sum_row + j + VTraits::vlanes() * 2, v_add(el4l_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 2))); + v_store(sum_row + j + VTraits::vlanes() * 3, v_add(el4h_1, vx_load(prev_sum_row + j + VTraits::vlanes() * 3))); + v_store(sum_row + j + VTraits::vlanes() * 4, v_add(el4h_2, vx_load(prev_sum_row + j + VTraits::vlanes() * 4))); + v_store(sum_row + j + VTraits::vlanes() * 5, v_add(el4h_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 5))); } for (int v3 = sum_row[j - 1] - prev_sum_row[j - 1], @@ -339,7 +339,7 @@ struct Integral_SIMD v_int32 prev = vx_setzero_s32(); int j = 0; - for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes) + for ( ; j + VTraits::vlanes() <= width; j += VTraits::vlanes()) { v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j)); v_int32 el4l, el4h; @@ -356,8 +356,8 @@ struct Integral_SIMD #endif #endif v_expand(el8, el4l, el4h); - el4l += prev; - el4h += el4l; + el4l = v_add(el4l, prev); + el4h = v_add(el4h, el4l); #if CV_SIMD_WIDTH == 16 prev = el4h; #elif CV_SIMD_WIDTH == 32 @@ -368,8 +368,8 @@ struct Integral_SIMD prev = v_combine_low(t, t); #endif #endif - v_store(sum_row + j , el4l + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_int32::nlanes, el4h + vx_load(prev_sum_row + j + v_int32::nlanes)); + v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits::vlanes()))); } for (int v4 = sum_row[j - 1] - prev_sum_row[j - 1], @@ -426,7 +426,7 @@ struct Integral_SIMD v_float32 prev = vx_setzero_f32(); int j = 0; - for (; j + v_uint16::nlanes <= width; j += v_uint16::nlanes) + for (; j + VTraits::vlanes() <= width; j += VTraits::vlanes()) { v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j)); v_float32 el4l, el4h; @@ -439,8 +439,8 @@ struct Integral_SIMD el4h.val = _mm256_add_ps(_mm256_cvtepi32_ps(_mm256_cvtepi16_epi32(_v256_extract_high(vsum))), _mm256_permutevar8x32_ps(el4l.val, shmask)); prev.val = _mm256_permutevar8x32_ps(el4h.val, shmask); #else - el8 += v_rotate_left<1>(el8); - el8 += v_rotate_left<2>(el8); + el8 = v_add(el8, v_rotate_left<1>(el8)); + el8 = v_add(el8, v_rotate_left<2>(el8)); #if CV_SIMD_WIDTH >= 32 el8 += v_rotate_left<4>(el8); #if CV_SIMD_WIDTH == 64 @@ -449,12 +449,12 @@ struct Integral_SIMD #endif v_int32 el4li, el4hi; v_expand(el8, el4li, el4hi); - el4l = v_cvt_f32(el4li) + prev; - el4h = v_cvt_f32(el4hi) + el4l; - prev = v_broadcast_element(el4h); + el4l = v_add(v_cvt_f32(el4li), prev); + el4h = v_add(v_cvt_f32(el4hi), el4l); + prev = v_broadcast_highest(el4h); #endif - v_store(sum_row + j , el4l + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_float32::nlanes, el4h + vx_load(prev_sum_row + j + v_float32::nlanes)); + v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits::vlanes()))); } for (float v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j) @@ -475,11 +475,11 @@ struct Integral_SIMD v_float32 prev_1 = vx_setzero_f32(), prev_2 = vx_setzero_f32(); int j = 0; - for (; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn) + for (; j + VTraits::vlanes() * cn <= width; j += VTraits::vlanes() * cn) { v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j)); - v_int16 el8_1 = v_src_row & mask; - v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8); + v_int16 el8_1 = v_and(v_src_row, mask); + v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row))); v_float32 el4l_1, el4h_1, el4l_2, el4h_2; #if CV_AVX2 && CV_SIMD_WIDTH == 32 __m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2)); @@ -496,10 +496,10 @@ struct Integral_SIMD prev_1.val = _mm256_permutevar8x32_ps(el4h_1.val, shmask); prev_2.val = _mm256_permutevar8x32_ps(el4h_2.val, shmask); #else - el8_1 += v_rotate_left<1>(el8_1); - el8_2 += v_rotate_left<1>(el8_2); - el8_1 += v_rotate_left<2>(el8_1); - el8_2 += v_rotate_left<2>(el8_2); + el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2)); + el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2)); #if CV_SIMD_WIDTH >= 32 el8_1 += v_rotate_left<4>(el8_1); el8_2 += v_rotate_left<4>(el8_2); @@ -511,20 +511,20 @@ struct Integral_SIMD v_int32 el4li_1, el4hi_1, el4li_2, el4hi_2; v_expand(el8_1, el4li_1, el4hi_1); v_expand(el8_2, el4li_2, el4hi_2); - el4l_1 = v_cvt_f32(el4li_1) + prev_1; - el4l_2 = v_cvt_f32(el4li_2) + prev_2; - el4h_1 = v_cvt_f32(el4hi_1) + el4l_1; - el4h_2 = v_cvt_f32(el4hi_2) + el4l_2; - prev_1 = v_broadcast_element(el4h_1); - prev_2 = v_broadcast_element(el4h_2); + el4l_1 = v_add(v_cvt_f32(el4li_1), prev_1); + el4l_2 = v_add(v_cvt_f32(el4li_2), prev_2); + el4h_1 = v_add(v_cvt_f32(el4hi_1), el4l_1); + el4h_2 = v_add(v_cvt_f32(el4hi_2), el4l_2); + prev_1 = v_broadcast_highest(el4h_1); + prev_2 = v_broadcast_highest(el4h_2); #endif v_float32 el4_1, el4_2, el4_3, el4_4; v_zip(el4l_1, el4l_2, el4_1, el4_2); v_zip(el4h_1, el4h_2, el4_3, el4_4); - v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_float32::nlanes , el4_2 + vx_load(prev_sum_row + j + v_float32::nlanes )); - v_store(sum_row + j + v_float32::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 2)); - v_store(sum_row + j + v_float32::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_float32::nlanes * 3)); + v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits::vlanes()))); + v_store(sum_row + j + VTraits::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 2))); + v_store(sum_row + j + VTraits::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits::vlanes() * 3))); } for (float v2 = sum_row[j - 1] - prev_sum_row[j - 1], @@ -543,7 +543,7 @@ struct Integral_SIMD const uchar * src_row = src + _srcstep * i; float * prev_sum_row = (float *)((uchar *)sum + _sumstep * i) + cn; float * sum_row = (float *)((uchar *)sum + _sumstep * (i + 1)) + cn; - float row_cache[v_float32::nlanes * 6]; + float row_cache[VTraits::max_nlanes * 6]; sum_row[-1] = sum_row[-2] = sum_row[-3] = 0; @@ -551,10 +551,10 @@ struct Integral_SIMD prev_3 = vx_setzero_f32(); int j = 0; const int j_max = - ((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height) - ? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave() - : width - v_uint16::nlanes * cn; // v_expand_low - for ( ; j <= j_max; j += v_uint16::nlanes * cn) + ((_srcstep * i + (width - VTraits::vlanes() * cn + VTraits::vlanes() * cn)) >= _srcstep * height) + ? width - VTraits::vlanes() * cn // uint8 in v_load_deinterleave() + : width - VTraits::vlanes() * cn; // v_expand_low + for ( ; j <= j_max; j += VTraits::vlanes() * cn) { v_uint8 v_src_row_1, v_src_row_2, v_src_row_3; v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3); @@ -583,12 +583,12 @@ struct Integral_SIMD prev_2.val = _mm256_permutevar8x32_ps(el4h_2.val, shmask); prev_3.val = _mm256_permutevar8x32_ps(el4h_3.val, shmask); #else - el8_1 += v_rotate_left<1>(el8_1); - el8_2 += v_rotate_left<1>(el8_2); - el8_3 += v_rotate_left<1>(el8_3); - el8_1 += v_rotate_left<2>(el8_1); - el8_2 += v_rotate_left<2>(el8_2); - el8_3 += v_rotate_left<2>(el8_3); + el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2)); + el8_3 = v_add(el8_3, v_rotate_left<1>(el8_3)); + el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2)); + el8_3 = v_add(el8_3, v_rotate_left<2>(el8_3)); #if CV_SIMD_WIDTH >= 32 el8_1 += v_rotate_left<4>(el8_1); el8_2 += v_rotate_left<4>(el8_2); @@ -603,30 +603,30 @@ struct Integral_SIMD v_expand(el8_1, el4li_1, el4hi_1); v_expand(el8_2, el4li_2, el4hi_2); v_expand(el8_3, el4li_3, el4hi_3); - el4l_1 = v_cvt_f32(el4li_1) + prev_1; - el4l_2 = v_cvt_f32(el4li_2) + prev_2; - el4l_3 = v_cvt_f32(el4li_3) + prev_3; - el4h_1 = v_cvt_f32(el4hi_1) + el4l_1; - el4h_2 = v_cvt_f32(el4hi_2) + el4l_2; - el4h_3 = v_cvt_f32(el4hi_3) + el4l_3; - prev_1 = v_broadcast_element(el4h_1); - prev_2 = v_broadcast_element(el4h_2); - prev_3 = v_broadcast_element(el4h_3); + el4l_1 = v_add(v_cvt_f32(el4li_1), prev_1); + el4l_2 = v_add(v_cvt_f32(el4li_2), prev_2); + el4l_3 = v_add(v_cvt_f32(el4li_3), prev_3); + el4h_1 = v_add(v_cvt_f32(el4hi_1), el4l_1); + el4h_2 = v_add(v_cvt_f32(el4hi_2), el4l_2); + el4h_3 = v_add(v_cvt_f32(el4hi_3), el4l_3); + prev_1 = v_broadcast_highest(el4h_1); + prev_2 = v_broadcast_highest(el4h_2); + prev_3 = v_broadcast_highest(el4h_3); #endif v_store_interleave(row_cache , el4l_1, el4l_2, el4l_3); - v_store_interleave(row_cache + v_float32::nlanes * 3, el4h_1, el4h_2, el4h_3); + v_store_interleave(row_cache + VTraits::vlanes() * 3, el4h_1, el4h_2, el4h_3); el4l_1 = vx_load(row_cache ); - el4l_2 = vx_load(row_cache + v_float32::nlanes ); - el4l_3 = vx_load(row_cache + v_float32::nlanes * 2); - el4h_1 = vx_load(row_cache + v_float32::nlanes * 3); - el4h_2 = vx_load(row_cache + v_float32::nlanes * 4); - el4h_3 = vx_load(row_cache + v_float32::nlanes * 5); - v_store(sum_row + j , el4l_1 + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_float32::nlanes , el4l_2 + vx_load(prev_sum_row + j + v_float32::nlanes )); - v_store(sum_row + j + v_float32::nlanes * 2, el4l_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 2)); - v_store(sum_row + j + v_float32::nlanes * 3, el4h_1 + vx_load(prev_sum_row + j + v_float32::nlanes * 3)); - v_store(sum_row + j + v_float32::nlanes * 4, el4h_2 + vx_load(prev_sum_row + j + v_float32::nlanes * 4)); - v_store(sum_row + j + v_float32::nlanes * 5, el4h_3 + vx_load(prev_sum_row + j + v_float32::nlanes * 5)); + el4l_2 = vx_load(row_cache + VTraits::vlanes() ); + el4l_3 = vx_load(row_cache + VTraits::vlanes() * 2); + el4h_1 = vx_load(row_cache + VTraits::vlanes() * 3); + el4h_2 = vx_load(row_cache + VTraits::vlanes() * 4); + el4h_3 = vx_load(row_cache + VTraits::vlanes() * 5); + v_store(sum_row + j , v_add(el4l_1, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes() , v_add(el4l_2, vx_load(prev_sum_row + j + VTraits::vlanes()))); + v_store(sum_row + j + VTraits::vlanes() * 2, v_add(el4l_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 2))); + v_store(sum_row + j + VTraits::vlanes() * 3, v_add(el4h_1, vx_load(prev_sum_row + j + VTraits::vlanes() * 3))); + v_store(sum_row + j + VTraits::vlanes() * 4, v_add(el4h_2, vx_load(prev_sum_row + j + VTraits::vlanes() * 4))); + v_store(sum_row + j + VTraits::vlanes() * 5, v_add(el4h_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 5))); } for (float v3 = sum_row[j - 1] - prev_sum_row[j - 1], @@ -652,7 +652,7 @@ struct Integral_SIMD v_float32 prev = vx_setzero_f32(); int j = 0; - for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes) + for ( ; j + VTraits::vlanes() <= width; j += VTraits::vlanes()) { v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j)); v_float32 el4l, el4h; @@ -670,8 +670,8 @@ struct Integral_SIMD #endif v_int32 el4li, el4hi; v_expand(el8, el4li, el4hi); - el4l = v_cvt_f32(el4li) + prev; - el4h = v_cvt_f32(el4hi) + el4l; + el4l = v_add(v_cvt_f32(el4li), prev); + el4h = v_add(v_cvt_f32(el4hi), el4l); #if CV_SIMD_WIDTH == 16 prev = el4h; #elif CV_SIMD_WIDTH == 32 @@ -682,8 +682,8 @@ struct Integral_SIMD prev = v_combine_low(t, t); #endif #endif - v_store(sum_row + j , el4l + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_float32::nlanes, el4h + vx_load(prev_sum_row + j + v_float32::nlanes)); + v_store(sum_row + j , v_add(el4l, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes(), v_add(el4h, vx_load(prev_sum_row + j + VTraits::vlanes()))); } for (float v4 = sum_row[j - 1] - prev_sum_row[j - 1], @@ -750,7 +750,7 @@ struct Integral_SIMD v_float64 prev = vx_setzero_f64(); int j = 0; - for (; j + v_uint16::nlanes <= width; j += v_uint16::nlanes) + for (; j + VTraits::vlanes() <= width; j += VTraits::vlanes()) { v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j)); v_float64 el4ll, el4lh, el4hl, el4hh; @@ -767,8 +767,8 @@ struct Integral_SIMD el4hh.val = _mm256_add_pd(_mm256_cvtepi32_pd(_v256_extract_high(el4h_32)), el4d); prev.val = _mm256_permute4x64_pd(el4hh.val, 0xff); #else - el8 += v_rotate_left<1>(el8); - el8 += v_rotate_left<2>(el8); + el8 = v_add(el8, v_rotate_left<1>(el8)); + el8 = v_add(el8, v_rotate_left<2>(el8)); #if CV_SIMD_WIDTH >= 32 el8 += v_rotate_left<4>(el8); #if CV_SIMD_WIDTH == 64 @@ -777,17 +777,17 @@ struct Integral_SIMD #endif v_int32 el4li, el4hi; v_expand(el8, el4li, el4hi); - el4ll = v_cvt_f64(el4li) + prev; - el4lh = v_cvt_f64_high(el4li) + prev; - el4hl = v_cvt_f64(el4hi) + el4ll; - el4hh = v_cvt_f64_high(el4hi) + el4lh; - prev = vx_setall_f64(v_extract_n(el4hh)); -// prev = v_broadcast_element(el4hh); + el4ll = v_add(v_cvt_f64(el4li), prev); + el4lh = v_add(v_cvt_f64_high(el4li), prev); + el4hl = v_add(v_cvt_f64(el4hi), el4ll); + el4hh = v_add(v_cvt_f64_high(el4hi), el4lh); + prev = vx_setall_f64(v_extract_highest(el4hh)); +// prev = v_broadcast_highest(el4hh); #endif - v_store(sum_row + j , el4ll + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_float64::nlanes , el4lh + vx_load(prev_sum_row + j + v_float64::nlanes )); - v_store(sum_row + j + v_float64::nlanes * 2, el4hl + vx_load(prev_sum_row + j + v_float64::nlanes * 2)); - v_store(sum_row + j + v_float64::nlanes * 3, el4hh + vx_load(prev_sum_row + j + v_float64::nlanes * 3)); + v_store(sum_row + j , v_add(el4ll, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes() , v_add(el4lh, vx_load(prev_sum_row + j + VTraits::vlanes()))); + v_store(sum_row + j + VTraits::vlanes() * 2, v_add(el4hl, vx_load(prev_sum_row + j + VTraits::vlanes() * 2))); + v_store(sum_row + j + VTraits::vlanes() * 3, v_add(el4hh, vx_load(prev_sum_row + j + VTraits::vlanes() * 3))); } for (double v = sum_row[j - 1] - prev_sum_row[j - 1]; j < width; ++j) @@ -808,11 +808,11 @@ struct Integral_SIMD v_float64 prev_1 = vx_setzero_f64(), prev_2 = vx_setzero_f64(); int j = 0; - for (; j + v_uint16::nlanes * cn <= width; j += v_uint16::nlanes * cn) + for (; j + VTraits::vlanes() * cn <= width; j += VTraits::vlanes() * cn) { v_int16 v_src_row = v_reinterpret_as_s16(vx_load(src_row + j)); - v_int16 el8_1 = v_src_row & mask; - v_int16 el8_2 = v_reinterpret_as_s16(v_reinterpret_as_u16(v_src_row) >> 8); + v_int16 el8_1 = v_and(v_src_row, mask); + v_int16 el8_2 = v_reinterpret_as_s16(v_shr<8>(v_reinterpret_as_u16(v_src_row))); v_float64 el4ll_1, el4lh_1, el4hl_1, el4hh_1, el4ll_2, el4lh_2, el4hl_2, el4hh_2; #if CV_AVX2 && CV_SIMD_WIDTH == 32 __m256i vsum_1 = _mm256_add_epi16(el8_1.val, _mm256_slli_si256(el8_1.val, 2)); @@ -838,10 +838,10 @@ struct Integral_SIMD prev_1.val = _mm256_permute4x64_pd(el4hh_1.val, 0xff); prev_2.val = _mm256_permute4x64_pd(el4hh_2.val, 0xff); #else - el8_1 += v_rotate_left<1>(el8_1); - el8_2 += v_rotate_left<1>(el8_2); - el8_1 += v_rotate_left<2>(el8_1); - el8_2 += v_rotate_left<2>(el8_2); + el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2)); + el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2)); #if CV_SIMD_WIDTH >= 32 el8_1 += v_rotate_left<4>(el8_1); el8_2 += v_rotate_left<4>(el8_2); @@ -853,32 +853,32 @@ struct Integral_SIMD v_int32 el4li_1, el4hi_1, el4li_2, el4hi_2; v_expand(el8_1, el4li_1, el4hi_1); v_expand(el8_2, el4li_2, el4hi_2); - el4ll_1 = v_cvt_f64(el4li_1) + prev_1; - el4ll_2 = v_cvt_f64(el4li_2) + prev_2; - el4lh_1 = v_cvt_f64_high(el4li_1) + prev_1; - el4lh_2 = v_cvt_f64_high(el4li_2) + prev_2; - el4hl_1 = v_cvt_f64(el4hi_1) + el4ll_1; - el4hl_2 = v_cvt_f64(el4hi_2) + el4ll_2; - el4hh_1 = v_cvt_f64_high(el4hi_1) + el4lh_1; - el4hh_2 = v_cvt_f64_high(el4hi_2) + el4lh_2; - prev_1 = vx_setall_f64(v_extract_n(el4hh_1)); - prev_2 = vx_setall_f64(v_extract_n(el4hh_2)); -// prev_1 = v_broadcast_element(el4hh_1); -// prev_2 = v_broadcast_element(el4hh_2); + el4ll_1 = v_add(v_cvt_f64(el4li_1), prev_1); + el4ll_2 = v_add(v_cvt_f64(el4li_2), prev_2); + el4lh_1 = v_add(v_cvt_f64_high(el4li_1), prev_1); + el4lh_2 = v_add(v_cvt_f64_high(el4li_2), prev_2); + el4hl_1 = v_add(v_cvt_f64(el4hi_1), el4ll_1); + el4hl_2 = v_add(v_cvt_f64(el4hi_2), el4ll_2); + el4hh_1 = v_add(v_cvt_f64_high(el4hi_1), el4lh_1); + el4hh_2 = v_add(v_cvt_f64_high(el4hi_2), el4lh_2); + prev_1 = vx_setall_f64(v_extract_highest(el4hh_1)); + prev_2 = vx_setall_f64(v_extract_highest(el4hh_2)); +// prev_1 = v_broadcast_highest(el4hh_1); +// prev_2 = v_broadcast_highest(el4hh_2); #endif v_float64 el4_1, el4_2, el4_3, el4_4, el4_5, el4_6, el4_7, el4_8; v_zip(el4ll_1, el4ll_2, el4_1, el4_2); v_zip(el4lh_1, el4lh_2, el4_3, el4_4); v_zip(el4hl_1, el4hl_2, el4_5, el4_6); v_zip(el4hh_1, el4hh_2, el4_7, el4_8); - v_store(sum_row + j , el4_1 + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_float64::nlanes , el4_2 + vx_load(prev_sum_row + j + v_float64::nlanes )); - v_store(sum_row + j + v_float64::nlanes * 2, el4_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 2)); - v_store(sum_row + j + v_float64::nlanes * 3, el4_4 + vx_load(prev_sum_row + j + v_float64::nlanes * 3)); - v_store(sum_row + j + v_float64::nlanes * 4, el4_5 + vx_load(prev_sum_row + j + v_float64::nlanes * 4)); - v_store(sum_row + j + v_float64::nlanes * 5, el4_6 + vx_load(prev_sum_row + j + v_float64::nlanes * 5)); - v_store(sum_row + j + v_float64::nlanes * 6, el4_7 + vx_load(prev_sum_row + j + v_float64::nlanes * 6)); - v_store(sum_row + j + v_float64::nlanes * 7, el4_8 + vx_load(prev_sum_row + j + v_float64::nlanes * 7)); + v_store(sum_row + j , v_add(el4_1, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes() , v_add(el4_2, vx_load(prev_sum_row + j + VTraits::vlanes()))); + v_store(sum_row + j + VTraits::vlanes() * 2, v_add(el4_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 2))); + v_store(sum_row + j + VTraits::vlanes() * 3, v_add(el4_4, vx_load(prev_sum_row + j + VTraits::vlanes() * 3))); + v_store(sum_row + j + VTraits::vlanes() * 4, v_add(el4_5, vx_load(prev_sum_row + j + VTraits::vlanes() * 4))); + v_store(sum_row + j + VTraits::vlanes() * 5, v_add(el4_6, vx_load(prev_sum_row + j + VTraits::vlanes() * 5))); + v_store(sum_row + j + VTraits::vlanes() * 6, v_add(el4_7, vx_load(prev_sum_row + j + VTraits::vlanes() * 6))); + v_store(sum_row + j + VTraits::vlanes() * 7, v_add(el4_8, vx_load(prev_sum_row + j + VTraits::vlanes() * 7))); } for (double v2 = sum_row[j - 1] - prev_sum_row[j - 1], @@ -897,7 +897,7 @@ struct Integral_SIMD const uchar * src_row = src + _srcstep * i; double * prev_sum_row = (double *)((uchar *)sum + _sumstep * i) + cn; double * sum_row = (double *)((uchar *)sum + _sumstep * (i + 1)) + cn; - double row_cache[v_float64::nlanes * 12]; + double row_cache[VTraits::max_nlanes * 12]; sum_row[-1] = sum_row[-2] = sum_row[-3] = 0; @@ -905,10 +905,10 @@ struct Integral_SIMD prev_3 = vx_setzero_f64(); int j = 0; const int j_max = - ((_srcstep * i + (width - v_uint16::nlanes * cn + v_uint8::nlanes * cn)) >= _srcstep * height) - ? width - v_uint8::nlanes * cn // uint8 in v_load_deinterleave() - : width - v_uint16::nlanes * cn; // v_expand_low - for ( ; j <= j_max; j += v_uint16::nlanes * cn) + ((_srcstep * i + (width - VTraits::vlanes() * cn + VTraits::vlanes() * cn)) >= _srcstep * height) + ? width - VTraits::vlanes() * cn // uint8 in v_load_deinterleave() + : width - VTraits::vlanes() * cn; // v_expand_low + for ( ; j <= j_max; j += VTraits::vlanes() * cn) { v_uint8 v_src_row_1, v_src_row_2, v_src_row_3; v_load_deinterleave(src_row + j, v_src_row_1, v_src_row_2, v_src_row_3); @@ -951,12 +951,12 @@ struct Integral_SIMD prev_2.val = _mm256_permute4x64_pd(el4hh_2.val, 0xff); prev_3.val = _mm256_permute4x64_pd(el4hh_3.val, 0xff); #else - el8_1 += v_rotate_left<1>(el8_1); - el8_2 += v_rotate_left<1>(el8_2); - el8_3 += v_rotate_left<1>(el8_3); - el8_1 += v_rotate_left<2>(el8_1); - el8_2 += v_rotate_left<2>(el8_2); - el8_3 += v_rotate_left<2>(el8_3); + el8_1 = v_add(el8_1, v_rotate_left<1>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<1>(el8_2)); + el8_3 = v_add(el8_3, v_rotate_left<1>(el8_3)); + el8_1 = v_add(el8_1, v_rotate_left<2>(el8_1)); + el8_2 = v_add(el8_2, v_rotate_left<2>(el8_2)); + el8_3 = v_add(el8_3, v_rotate_left<2>(el8_3)); #if CV_SIMD_WIDTH >= 32 el8_1 += v_rotate_left<4>(el8_1); el8_2 += v_rotate_left<4>(el8_2); @@ -971,53 +971,53 @@ struct Integral_SIMD v_expand(el8_1, el4li_1, el4hi_1); v_expand(el8_2, el4li_2, el4hi_2); v_expand(el8_3, el4li_3, el4hi_3); - el4ll_1 = v_cvt_f64(el4li_1) + prev_1; - el4ll_2 = v_cvt_f64(el4li_2) + prev_2; - el4ll_3 = v_cvt_f64(el4li_3) + prev_3; - el4lh_1 = v_cvt_f64_high(el4li_1) + prev_1; - el4lh_2 = v_cvt_f64_high(el4li_2) + prev_2; - el4lh_3 = v_cvt_f64_high(el4li_3) + prev_3; - el4hl_1 = v_cvt_f64(el4hi_1) + el4ll_1; - el4hl_2 = v_cvt_f64(el4hi_2) + el4ll_2; - el4hl_3 = v_cvt_f64(el4hi_3) + el4ll_3; - el4hh_1 = v_cvt_f64_high(el4hi_1) + el4lh_1; - el4hh_2 = v_cvt_f64_high(el4hi_2) + el4lh_2; - el4hh_3 = v_cvt_f64_high(el4hi_3) + el4lh_3; - prev_1 = vx_setall_f64(v_extract_n(el4hh_1)); - prev_2 = vx_setall_f64(v_extract_n(el4hh_2)); - prev_3 = vx_setall_f64(v_extract_n(el4hh_3)); -// prev_1 = v_broadcast_element(el4hh_1); -// prev_2 = v_broadcast_element(el4hh_2); -// prev_3 = v_broadcast_element(el4hh_3); + el4ll_1 = v_add(v_cvt_f64(el4li_1), prev_1); + el4ll_2 = v_add(v_cvt_f64(el4li_2), prev_2); + el4ll_3 = v_add(v_cvt_f64(el4li_3), prev_3); + el4lh_1 = v_add(v_cvt_f64_high(el4li_1), prev_1); + el4lh_2 = v_add(v_cvt_f64_high(el4li_2), prev_2); + el4lh_3 = v_add(v_cvt_f64_high(el4li_3), prev_3); + el4hl_1 = v_add(v_cvt_f64(el4hi_1), el4ll_1); + el4hl_2 = v_add(v_cvt_f64(el4hi_2), el4ll_2); + el4hl_3 = v_add(v_cvt_f64(el4hi_3), el4ll_3); + el4hh_1 = v_add(v_cvt_f64_high(el4hi_1), el4lh_1); + el4hh_2 = v_add(v_cvt_f64_high(el4hi_2), el4lh_2); + el4hh_3 = v_add(v_cvt_f64_high(el4hi_3), el4lh_3); + prev_1 = vx_setall_f64(v_extract_highest(el4hh_1)); + prev_2 = vx_setall_f64(v_extract_highest(el4hh_2)); + prev_3 = vx_setall_f64(v_extract_highest(el4hh_3)); +// prev_1 = v_broadcast_highest(el4hh_1); +// prev_2 = v_broadcast_highest(el4hh_2); +// prev_3 = v_broadcast_highest(el4hh_3); #endif v_store_interleave(row_cache , el4ll_1, el4ll_2, el4ll_3); - v_store_interleave(row_cache + v_float64::nlanes * 3, el4lh_1, el4lh_2, el4lh_3); - v_store_interleave(row_cache + v_float64::nlanes * 6, el4hl_1, el4hl_2, el4hl_3); - v_store_interleave(row_cache + v_float64::nlanes * 9, el4hh_1, el4hh_2, el4hh_3); + v_store_interleave(row_cache + VTraits::vlanes() * 3, el4lh_1, el4lh_2, el4lh_3); + v_store_interleave(row_cache + VTraits::vlanes() * 6, el4hl_1, el4hl_2, el4hl_3); + v_store_interleave(row_cache + VTraits::vlanes() * 9, el4hh_1, el4hh_2, el4hh_3); el4ll_1 = vx_load(row_cache ); - el4ll_2 = vx_load(row_cache + v_float64::nlanes ); - el4ll_3 = vx_load(row_cache + v_float64::nlanes * 2 ); - el4lh_1 = vx_load(row_cache + v_float64::nlanes * 3 ); - el4lh_2 = vx_load(row_cache + v_float64::nlanes * 4 ); - el4lh_3 = vx_load(row_cache + v_float64::nlanes * 5 ); - el4hl_1 = vx_load(row_cache + v_float64::nlanes * 6 ); - el4hl_2 = vx_load(row_cache + v_float64::nlanes * 7 ); - el4hl_3 = vx_load(row_cache + v_float64::nlanes * 8 ); - el4hh_1 = vx_load(row_cache + v_float64::nlanes * 9 ); - el4hh_2 = vx_load(row_cache + v_float64::nlanes * 10); - el4hh_3 = vx_load(row_cache + v_float64::nlanes * 11); - v_store(sum_row + j , el4ll_1 + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_float64::nlanes , el4ll_2 + vx_load(prev_sum_row + j + v_float64::nlanes )); - v_store(sum_row + j + v_float64::nlanes * 2 , el4ll_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 2 )); - v_store(sum_row + j + v_float64::nlanes * 3 , el4lh_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 3 )); - v_store(sum_row + j + v_float64::nlanes * 4 , el4lh_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 4 )); - v_store(sum_row + j + v_float64::nlanes * 5 , el4lh_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 5 )); - v_store(sum_row + j + v_float64::nlanes * 6 , el4hl_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 6 )); - v_store(sum_row + j + v_float64::nlanes * 7 , el4hl_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 7 )); - v_store(sum_row + j + v_float64::nlanes * 8 , el4hl_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 8 )); - v_store(sum_row + j + v_float64::nlanes * 9 , el4hh_1 + vx_load(prev_sum_row + j + v_float64::nlanes * 9 )); - v_store(sum_row + j + v_float64::nlanes * 10, el4hh_2 + vx_load(prev_sum_row + j + v_float64::nlanes * 10)); - v_store(sum_row + j + v_float64::nlanes * 11, el4hh_3 + vx_load(prev_sum_row + j + v_float64::nlanes * 11)); + el4ll_2 = vx_load(row_cache + VTraits::vlanes() ); + el4ll_3 = vx_load(row_cache + VTraits::vlanes() * 2 ); + el4lh_1 = vx_load(row_cache + VTraits::vlanes() * 3 ); + el4lh_2 = vx_load(row_cache + VTraits::vlanes() * 4 ); + el4lh_3 = vx_load(row_cache + VTraits::vlanes() * 5 ); + el4hl_1 = vx_load(row_cache + VTraits::vlanes() * 6 ); + el4hl_2 = vx_load(row_cache + VTraits::vlanes() * 7 ); + el4hl_3 = vx_load(row_cache + VTraits::vlanes() * 8 ); + el4hh_1 = vx_load(row_cache + VTraits::vlanes() * 9 ); + el4hh_2 = vx_load(row_cache + VTraits::vlanes() * 10); + el4hh_3 = vx_load(row_cache + VTraits::vlanes() * 11); + v_store(sum_row + j , v_add(el4ll_1, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes() , v_add(el4ll_2, vx_load(prev_sum_row + j + VTraits::vlanes()))); + v_store(sum_row + j + VTraits::vlanes() * 2 , v_add(el4ll_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 2))); + v_store(sum_row + j + VTraits::vlanes() * 3 , v_add(el4lh_1, vx_load(prev_sum_row + j + VTraits::vlanes() * 3))); + v_store(sum_row + j + VTraits::vlanes() * 4 , v_add(el4lh_2, vx_load(prev_sum_row + j + VTraits::vlanes() * 4))); + v_store(sum_row + j + VTraits::vlanes() * 5 , v_add(el4lh_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 5))); + v_store(sum_row + j + VTraits::vlanes() * 6 , v_add(el4hl_1, vx_load(prev_sum_row + j + VTraits::vlanes() * 6))); + v_store(sum_row + j + VTraits::vlanes() * 7 , v_add(el4hl_2, vx_load(prev_sum_row + j + VTraits::vlanes() * 7))); + v_store(sum_row + j + VTraits::vlanes() * 8 , v_add(el4hl_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 8))); + v_store(sum_row + j + VTraits::vlanes() * 9 , v_add(el4hh_1, vx_load(prev_sum_row + j + VTraits::vlanes() * 9))); + v_store(sum_row + j + VTraits::vlanes() * 10, v_add(el4hh_2, vx_load(prev_sum_row + j + VTraits::vlanes() * 10))); + v_store(sum_row + j + VTraits::vlanes() * 11, v_add(el4hh_3, vx_load(prev_sum_row + j + VTraits::vlanes() * 11))); } for (double v3 = sum_row[j - 1] - prev_sum_row[j - 1], @@ -1043,7 +1043,7 @@ struct Integral_SIMD v_float64 prev_1 = vx_setzero_f64(), prev_2 = vx_setzero_f64(); int j = 0; - for ( ; j + v_uint16::nlanes <= width; j += v_uint16::nlanes) + for ( ; j + VTraits::vlanes() <= width; j += VTraits::vlanes()) { v_int16 el8 = v_reinterpret_as_s16(vx_load_expand(src_row + j)); v_float64 el4ll, el4lh, el4hl, el4hh; @@ -1065,10 +1065,10 @@ struct Integral_SIMD #endif v_int32 el4li, el4hi; v_expand(el8, el4li, el4hi); - el4ll = v_cvt_f64(el4li) + prev_1; - el4lh = v_cvt_f64_high(el4li) + prev_2; - el4hl = v_cvt_f64(el4hi) + el4ll; - el4hh = v_cvt_f64_high(el4hi) + el4lh; + el4ll = v_add(v_cvt_f64(el4li), prev_1); + el4lh = v_add(v_cvt_f64_high(el4li), prev_2); + el4hl = v_add(v_cvt_f64(el4hi), el4ll); + el4hh = v_add(v_cvt_f64_high(el4hi), el4lh); #if CV_SIMD_WIDTH == 16 prev_1 = el4hl; prev_2 = el4hh; @@ -1078,10 +1078,10 @@ struct Integral_SIMD prev_1 = prev_2 = v_combine_high(el4hh, el4hh); #endif #endif - v_store(sum_row + j , el4ll + vx_load(prev_sum_row + j )); - v_store(sum_row + j + v_float64::nlanes , el4lh + vx_load(prev_sum_row + j + v_float64::nlanes )); - v_store(sum_row + j + v_float64::nlanes * 2, el4hl + vx_load(prev_sum_row + j + v_float64::nlanes * 2)); - v_store(sum_row + j + v_float64::nlanes * 3, el4hh + vx_load(prev_sum_row + j + v_float64::nlanes * 3)); + v_store(sum_row + j , v_add(el4ll, vx_load(prev_sum_row + j))); + v_store(sum_row + j + VTraits::vlanes() , v_add(el4lh, vx_load(prev_sum_row + j + VTraits::vlanes()))); + v_store(sum_row + j + VTraits::vlanes() * 2, v_add(el4hl, vx_load(prev_sum_row + j + VTraits::vlanes() * 2))); + v_store(sum_row + j + VTraits::vlanes() * 3, v_add(el4hh, vx_load(prev_sum_row + j + VTraits::vlanes() * 3))); } for (double v4 = sum_row[j - 1] - prev_sum_row[j - 1], diff --git a/modules/objdetect/src/barcode.cpp b/modules/objdetect/src/barcode.cpp index 549ea84a0a..e0c7d9cb57 100644 --- a/modules/objdetect/src/barcode.cpp +++ b/modules/objdetect/src/barcode.cpp @@ -343,11 +343,11 @@ BarcodeDetector::BarcodeDetector(const string &prototxt_path, const string &mode { Ptr p_ = new BarcodeImpl(); p = p_; + p_->sr = make_shared(); if (!prototxt_path.empty() && !model_path.empty()) { CV_Assert(utils::fs::exists(prototxt_path)); CV_Assert(utils::fs::exists(model_path)); - p_->sr = make_shared(); int res = p_->sr->init(prototxt_path, model_path); CV_Assert(res == 0); p_->use_nn_sr = true; diff --git a/modules/objdetect/src/face_detect.cpp b/modules/objdetect/src/face_detect.cpp index 17c982d92f..8da13eda45 100644 --- a/modules/objdetect/src/face_detect.cpp +++ b/modules/objdetect/src/face_detect.cpp @@ -102,12 +102,21 @@ public: return 0; } CV_CheckEQ(input_image.size(), Size(inputW, inputH), "Size does not match. Call setInputSize(size) if input size does not match the preset size"); - // Pad input_image with divisor 32 - Mat pad_image = padWithDivisor(input_image); - - // Build blob from input image - Mat input_blob = dnn::blobFromImage(pad_image); + Mat input_blob; + if(input_image.kind() == _InputArray::UMAT) { + // Pad input_image with divisor 32 + UMat pad_image; + padWithDivisor(input_image, pad_image); + // Build blob from input image + input_blob = dnn::blobFromImage(pad_image); + } else { + // Pad input_image with divisor 32 + Mat pad_image; + padWithDivisor(input_image, pad_image); + // Build blob from input image + input_blob = dnn::blobFromImage(pad_image); + } // Forward std::vector output_names = { "cls_8", "cls_16", "cls_32", "obj_8", "obj_16", "obj_32", "bbox_8", "bbox_16", "bbox_32", "kps_8", "kps_16", "kps_32" }; std::vector output_blobs; @@ -217,13 +226,11 @@ private: } } - Mat padWithDivisor(InputArray& input_image) + void padWithDivisor(InputArray input_image, OutputArray pad_image) { int bottom = padH - inputH; int right = padW - inputW; - Mat pad_image; copyMakeBorder(input_image, pad_image, 0, bottom, 0, right, BORDER_CONSTANT, 0); - return pad_image; } private: dnn::Net net; diff --git a/modules/objdetect/src/hog.cpp b/modules/objdetect/src/hog.cpp index b57e92ff9a..b83263304f 100644 --- a/modules/objdetect/src/hog.cpp +++ b/modules/objdetect/src/hog.cpp @@ -268,13 +268,13 @@ void HOGDescriptor::computeGradient(InputArray _img, InputOutputArray _grad, Inp for ( i = 0; i < 256; i += 4) { v_store(_data + i, v_sqrt(idx)); - idx += ifour; + idx = v_add(idx, ifour); } else for ( i = 0; i < 256; i += 4) { v_store(_data + i, idx); - idx += ifour; + idx = v_add(idx, ifour); } #else if( gammaCorrection ) @@ -320,7 +320,7 @@ void HOGDescriptor::computeGradient(InputArray _img, InputOutputArray _grad, Inp for ( ; x <= end - 4; x += 4) { v_int32x4 mul_res = v_load(xmap + x); - mul_res += mul_res + mul_res; + mul_res = v_add(mul_res, v_add(mul_res, mul_res)); v_store(xmap + x, mul_res); } #endif @@ -444,34 +444,34 @@ void HOGDescriptor::computeGradient(InputArray _img, InputOutputArray _grad, Inp { int x0 = xmap[x], x1 = xmap[x+1], x2 = xmap[x+2], x3 = xmap[x+3]; - v_float32x4 _dx0 = v_load(lutCurr+x+widthP2*0+2) - v_load(lutCurr+x+widthP2*0); - v_float32x4 _dx1 = v_load(lutCurr+x+widthP2*1+2) - v_load(lutCurr+x+widthP2*1); - v_float32x4 _dx2 = v_load(lutCurr+x+widthP2*2+2) - v_load(lutCurr+x+widthP2*2); + v_float32x4 _dx0 = v_sub(v_load(lutCurr + x + widthP2 * 0 + 2), v_load(lutCurr + x + widthP2 * 0)); + v_float32x4 _dx1 = v_sub(v_load(lutCurr + x + widthP2 * 1 + 2), v_load(lutCurr + x + widthP2 * 1)); + v_float32x4 _dx2 = v_sub(v_load(lutCurr + x + widthP2 * 2 + 2), v_load(lutCurr + x + widthP2 * 2)); v_float32x4 _dy00 = v_float32x4(lut[nextPtr[x0+0]], lut[nextPtr[x1+0]], lut[nextPtr[x2+0]], lut[nextPtr[x3+0]]); - v_float32x4 _dy0 = _dy00 - v_load(lutPrev+x+widthP2*0+1); + v_float32x4 _dy0 = v_sub(_dy00, v_load(lutPrev + x + widthP2 * 0 + 1)); v_store(lutNext+x+widthP2*0+1, _dy00); v_float32x4 _dy10 = v_float32x4(lut[nextPtr[x0+1]], lut[nextPtr[x1+1]], lut[nextPtr[x2+1]], lut[nextPtr[x3+1]]); - v_float32x4 _dy1 = _dy10 - v_load(lutPrev+x+widthP2*1+1); + v_float32x4 _dy1 = v_sub(_dy10, v_load(lutPrev + x + widthP2 * 1 + 1)); v_store(lutNext+x+widthP2*1+1, _dy10); v_float32x4 _dy20 = v_float32x4(lut[nextPtr[x0+2]], lut[nextPtr[x1+2]], lut[nextPtr[x2+2]], lut[nextPtr[x3+2]]); - v_float32x4 _dy2 = _dy20 - v_load(lutPrev+x+widthP2*2+1); + v_float32x4 _dy2 = v_sub(_dy20, v_load(lutPrev + x + widthP2 * 2 + 1)); v_store(lutNext+x+widthP2*2+1, _dy20); - v_float32x4 _mag0 = (_dx0 * _dx0) + (_dy0 * _dy0); - v_float32x4 _mag1 = (_dx1 * _dx1) + (_dy1 * _dy1); - v_float32x4 _mag2 = (_dx2 * _dx2) + (_dy2 * _dy2); + v_float32x4 _mag0 = v_add(v_mul(_dx0, _dx0), v_mul(_dy0, _dy0)); + v_float32x4 _mag1 = v_add(v_mul(_dx1, _dx1), v_mul(_dy1, _dy1)); + v_float32x4 _mag2 = v_add(v_mul(_dx2, _dx2), v_mul(_dy2, _dy2)); - v_float32x4 mask = v_reinterpret_as_f32(_mag2 > _mag1); + v_float32x4 mask = v_reinterpret_as_f32(v_gt(_mag2, _mag1)); _dx2 = v_select(mask, _dx2, _dx1); _dy2 = v_select(mask, _dy2, _dy1); - mask = v_reinterpret_as_f32(v_max(_mag2, _mag1) > _mag0); + mask = v_reinterpret_as_f32(v_gt(v_max(_mag2, _mag1), _mag0)); _dx2 = v_select(mask, _dx2, _dx0); _dy2 = v_select(mask, _dy2, _dy0); @@ -537,25 +537,25 @@ void HOGDescriptor::computeGradient(InputArray _img, InputOutputArray _grad, Inp int x2 = x << 1; v_float32x4 _mag = v_load(dbuf + x + (width << 1)); v_float32x4 _angle = v_load(dbuf + x + width * 3); - _angle = (_angleScale * _angle) - fhalf; + _angle = v_sub(v_mul(_angleScale, _angle), fhalf); v_int32x4 _hidx = v_floor(_angle); - _angle -= v_cvt_f32(_hidx); + _angle = v_sub(_angle, v_cvt_f32(_hidx)); - v_float32x4 ft0 = _mag * (fone - _angle); - v_float32x4 ft1 = _mag * _angle; + v_float32x4 ft0 = v_mul(_mag, v_sub(fone, _angle)); + v_float32x4 ft1 = v_mul(_mag, _angle); v_store_interleave(gradPtr + x2, ft0, ft1); - v_int32x4 mask0 = _hidx >> 31; - v_int32x4 it0 = mask0 & _nbins; - mask0 = (_hidx >= _nbins); - v_int32x4 it1 = mask0 & _nbins; - _hidx += (it0 - it1); + v_int32x4 mask0 = v_shr<31>(_hidx); + v_int32x4 it0 = v_and(mask0, _nbins); + mask0 = (v_ge(_hidx, _nbins)); + v_int32x4 it1 = v_and(mask0, _nbins); + _hidx = v_add(_hidx, v_sub(it0, it1)); it0 = v_reinterpret_as_s32(v_pack(v_pack(_hidx, izero), v_reinterpret_as_s16(izero))); - _hidx += ione; - _hidx &= (_hidx < _nbins); + _hidx = v_add(_hidx, ione); + _hidx = v_and(_hidx, v_lt(_hidx, _nbins)); it1 = v_reinterpret_as_s32(v_pack(v_pack(_hidx, izero), v_reinterpret_as_s16(izero))); v_uint8x16 it2, it3; v_zip(v_reinterpret_as_u8(it0), v_reinterpret_as_u8(it1), it2, it3); @@ -707,9 +707,9 @@ void HOGCache::init(const HOGDescriptor* _descriptor, for (; i <= blockSize.height - 4; i += 4) { - v_float32x4 t = idx - _bh; - t *= t; - idx += ifour; + v_float32x4 t = v_sub(idx, _bh); + t = v_mul(t, t); + idx = v_add(idx, ifour); v_store(_di + i, t); } #endif @@ -725,9 +725,9 @@ void HOGCache::init(const HOGDescriptor* _descriptor, for (; j <= blockSize.height - 4; j += 4) { - v_float32x4 t = idx - _bw; - t *= t; - idx += ifour; + v_float32x4 t = v_sub(idx, _bw); + t = v_mul(t, t); + idx = v_add(idx, ifour); v_store(_dj + j, t); } #endif @@ -936,8 +936,8 @@ const float* HOGCache::getBlock(Point pt, float* buf) int h0 = h[0], h1 = h[1]; v_float32x4 _a0 = v_setall_f32(a[0]), _a1 = v_setall_f32(a[1]); - v_float32x4 w = v_setall_f32(pk.gradWeight) * v_load(pk.histWeights); - v_float32x4 _t0 = _a0 * w, _t1 = _a1 * w; + v_float32x4 w = v_mul(v_setall_f32(pk.gradWeight), v_load(pk.histWeights)); + v_float32x4 _t0 = v_mul(_a0, w), _t1 = v_mul(_a1, w); v_store(hist0, _t0); v_store(hist1, _t1); @@ -984,8 +984,8 @@ const float* HOGCache::getBlock(Point pt, float* buf) int h0 = h[0], h1 = h[1]; v_float32x4 _a0 = v_setall_f32(a[0]), _a1 = v_setall_f32(a[1]); - v_float32x4 w = v_setall_f32(pk.gradWeight) * v_load(pk.histWeights); - v_float32x4 _t0 = _a0 * w, _t1 = _a1 * w; + v_float32x4 w = v_mul(v_setall_f32(pk.gradWeight), v_load(pk.histWeights)); + v_float32x4 _t0 = v_mul(_a0, w), _t1 = v_mul(_a1, w); v_store(hist0, _t0); v_store(hist1, _t1); @@ -1057,12 +1057,12 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const #if CV_SIMD128 v_float32x4 p0 = v_load(hist); - v_float32x4 s = p0 * p0; + v_float32x4 s = v_mul(p0, p0); for (i = 4; i <= sz - 4; i += 4) { p0 = v_load(hist + i); - s += p0 * p0; + s = v_add(s, v_mul(p0, p0)); } v_store(partSum, s); #else @@ -1091,17 +1091,17 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const v_float32x4 _scale = v_setall_f32(scale); static v_float32x4 _threshold = v_setall_f32(thresh); - v_float32x4 p = _scale * v_load(hist); + v_float32x4 p = v_mul(_scale, v_load(hist)); p = v_min(p, _threshold); - s = p * p; + s = v_mul(p, p); v_store(hist, p); for(i = 4 ; i <= sz - 4; i += 4) { p = v_load(hist + i); - p *= _scale; + p = v_mul(p, _scale); p = v_min(p, _threshold); - s += p * p; + s = v_add(s, v_mul(p, p)); v_store(hist + i, p); } @@ -1137,7 +1137,7 @@ void HOGCache::normalizeBlockHistogram(float* _hist) const v_float32x4 _scale2 = v_setall_f32(scale); for ( ; i <= sz - 4; i += 4) { - v_float32x4 t = _scale2 * v_load(hist + i); + v_float32x4 t = v_mul(_scale2, v_load(hist + i)); v_store(hist + i, t); } #endif @@ -1593,14 +1593,14 @@ void HOGDescriptor::detect(InputArray _img, #if CV_SIMD128 v_float32x4 _vec = v_load(vec); v_float32x4 _svmVec = v_load(svmVec); - v_float32x4 sum = _svmVec * _vec; + v_float32x4 sum = v_mul(_svmVec, _vec); for( k = 4; k <= blockHistogramSize - 4; k += 4 ) { _vec = v_load(vec + k); _svmVec = v_load(svmVec + k); - sum += _vec * _svmVec; + sum = v_add(sum, v_mul(_vec, _svmVec)); } v_store(partSum, sum); @@ -3392,14 +3392,14 @@ void HOGDescriptor::detectROI(InputArray _img, const std::vector &loc #if CV_SIMD128 v_float32x4 _vec = v_load(vec); v_float32x4 _svmVec = v_load(svmVec); - v_float32x4 sum = _svmVec * _vec; + v_float32x4 sum = v_mul(_svmVec, _vec); for( k = 4; k <= blockHistogramSize - 4; k += 4 ) { _vec = v_load(vec + k); _svmVec = v_load(svmVec + k); - sum += _vec * _svmVec; + sum = v_add(sum, v_mul(_vec, _svmVec)); } v_store(partSum, sum); diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index fa3a3fb9f1..f37e153779 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -1060,7 +1060,7 @@ class FuncInfo(object): else: py_name = classinfo.full_export_name + "." + self.variants[0].wname - if not self.is_static: + if not self.is_static and not self.isconstructor: cname = classinfo.cname + '::' + cname else: py_name = '.'.join([self.namespace, self.variants[0].wname]) diff --git a/modules/stitching/src/camera.cpp b/modules/stitching/src/camera.cpp index 149ba74760..2f7ee97e06 100644 --- a/modules/stitching/src/camera.cpp +++ b/modules/stitching/src/camera.cpp @@ -66,7 +66,7 @@ Mat CameraParams::K() const Mat_ k = Mat::eye(3, 3, CV_64F); k(0,0) = focal; k(0,2) = ppx; k(1,1) = focal * aspect; k(1,2) = ppy; - return std::move(k); + return Mat(k); } } // namespace detail diff --git a/modules/video/src/dis_flow.cpp b/modules/video/src/dis_flow.cpp index a260b8726b..40ac4517a4 100644 --- a/modules/video/src/dis_flow.cpp +++ b/modules/video/src/dis_flow.cpp @@ -520,16 +520,16 @@ DISOpticalFlowImpl::PatchInverseSearch_ParBody::PatchInverseSearch_ParBody(DISOp v_expand(I0_row_8, I0_row_4_left, I0_row_4_right); \ \ /* Compute diffs between I0 and bilinearly interpolated I1: */ \ - I_diff_left = w00v * v_cvt_f32(v_reinterpret_as_s32(I1_row_4_left)) + \ - w01v * v_cvt_f32(v_reinterpret_as_s32(I1_row_shifted_4_left)) + \ - w10v * v_cvt_f32(v_reinterpret_as_s32(I1_row_next_4_left)) + \ - w11v * v_cvt_f32(v_reinterpret_as_s32(I1_row_next_shifted_4_left)) - \ - v_cvt_f32(v_reinterpret_as_s32(I0_row_4_left)); \ - I_diff_right = w00v * v_cvt_f32(v_reinterpret_as_s32(I1_row_4_right)) + \ - w01v * v_cvt_f32(v_reinterpret_as_s32(I1_row_shifted_4_right)) + \ - w10v * v_cvt_f32(v_reinterpret_as_s32(I1_row_next_4_right)) + \ - w11v * v_cvt_f32(v_reinterpret_as_s32(I1_row_next_shifted_4_right)) - \ - v_cvt_f32(v_reinterpret_as_s32(I0_row_4_right)); + I_diff_left = v_sub(v_add(v_mul(w00v, v_cvt_f32(v_reinterpret_as_s32(I1_row_4_left))), \ + v_mul(w01v, v_cvt_f32(v_reinterpret_as_s32(I1_row_shifted_4_left))), \ + v_mul(w10v, v_cvt_f32(v_reinterpret_as_s32(I1_row_next_4_left))), \ + v_mul(w11v, v_cvt_f32(v_reinterpret_as_s32(I1_row_next_shifted_4_left)))), \ + v_cvt_f32(v_reinterpret_as_s32(I0_row_4_left))); \ + I_diff_right = v_sub(v_add(v_mul(w00v, v_cvt_f32(v_reinterpret_as_s32(I1_row_4_right))), \ + v_mul(w01v, v_cvt_f32(v_reinterpret_as_s32(I1_row_shifted_4_right))), \ + v_mul(w10v, v_cvt_f32(v_reinterpret_as_s32(I1_row_next_4_right))), \ + v_mul(w11v, v_cvt_f32(v_reinterpret_as_s32(I1_row_next_shifted_4_right)))), \ + v_cvt_f32(v_reinterpret_as_s32(I0_row_4_right))); #define HAL_BILINEAR_8x8_PATCH_EXTRACTION_NEXT_ROW \ I0_ptr += I0_stride; \ @@ -572,9 +572,9 @@ inline float processPatch(float &dst_dUx, float &dst_dUy, uchar *I0_ptr, uchar * v_expand(I0y_row, I0y_row_4_left, I0y_row_4_right); /* Update the sums: */ - Ux_vec += I_diff_left * v_cvt_f32(I0x_row_4_left) + I_diff_right * v_cvt_f32(I0x_row_4_right); - Uy_vec += I_diff_left * v_cvt_f32(I0y_row_4_left) + I_diff_right * v_cvt_f32(I0y_row_4_right); - SSD_vec += I_diff_left * I_diff_left + I_diff_right * I_diff_right; + Ux_vec = v_add(Ux_vec, v_add(v_mul(I_diff_left, v_cvt_f32(I0x_row_4_left)), v_mul(I_diff_right, v_cvt_f32(I0x_row_4_right)))); + Uy_vec = v_add(Uy_vec, v_add(v_mul(I_diff_left, v_cvt_f32(I0y_row_4_left)), v_mul(I_diff_right, v_cvt_f32(I0y_row_4_right)))); + SSD_vec = v_add(SSD_vec, v_add(v_mul(I_diff_left, I_diff_left), v_mul(I_diff_right, I_diff_right))); I0x_ptr += I0_stride; I0y_ptr += I0_stride; @@ -640,10 +640,10 @@ inline float processPatchMeanNorm(float &dst_dUx, float &dst_dUy, uchar *I0_ptr, v_expand(I0y_row, I0y_row_4_left, I0y_row_4_right); /* Update the sums: */ - sum_I0x_mul_vec += I_diff_left * v_cvt_f32(I0x_row_4_left) + I_diff_right * v_cvt_f32(I0x_row_4_right); - sum_I0y_mul_vec += I_diff_left * v_cvt_f32(I0y_row_4_left) + I_diff_right * v_cvt_f32(I0y_row_4_right); - sum_diff_sq_vec += I_diff_left * I_diff_left + I_diff_right * I_diff_right; - sum_diff_vec += I_diff_left + I_diff_right; + sum_I0x_mul_vec = v_add(sum_I0x_mul_vec, v_add(v_mul(I_diff_left, v_cvt_f32(I0x_row_4_left)), v_mul(I_diff_right, v_cvt_f32(I0x_row_4_right)))); + sum_I0y_mul_vec = v_add(sum_I0y_mul_vec, v_add(v_mul(I_diff_left, v_cvt_f32(I0y_row_4_left)), v_mul(I_diff_right, v_cvt_f32(I0y_row_4_right)))); + sum_diff_sq_vec = v_add(sum_diff_sq_vec, v_add(v_mul(I_diff_left, I_diff_left), v_mul(I_diff_right, I_diff_right))); + sum_diff_vec = v_add(sum_diff_vec, v_add(I_diff_left, I_diff_right)); I0x_ptr += I0_stride; I0y_ptr += I0_stride; @@ -692,7 +692,7 @@ inline float computeSSD(uchar *I0_ptr, uchar *I1_ptr, int I0_stride, int I1_stri for (int row = 0; row < 8; row++) { HAL_PROCESS_BILINEAR_8x8_PATCH_EXTRACTION; - SSD_vec += I_diff_left * I_diff_left + I_diff_right * I_diff_right; + SSD_vec = v_add(SSD_vec, v_add(v_mul(I_diff_left, I_diff_left), v_mul(I_diff_right, I_diff_right))); HAL_BILINEAR_8x8_PATCH_EXTRACTION_NEXT_ROW; } SSD = v_reduce_sum(SSD_vec); @@ -728,8 +728,8 @@ inline float computeSSDMeanNorm(uchar *I0_ptr, uchar *I1_ptr, int I0_stride, int for (int row = 0; row < 8; row++) { HAL_PROCESS_BILINEAR_8x8_PATCH_EXTRACTION; - sum_diff_sq_vec += I_diff_left * I_diff_left + I_diff_right * I_diff_right; - sum_diff_vec += I_diff_left + I_diff_right; + sum_diff_sq_vec = v_add(sum_diff_sq_vec, v_add(v_mul(I_diff_left, I_diff_left), v_mul(I_diff_right, I_diff_right))); + sum_diff_vec = v_add(sum_diff_vec, v_add(I_diff_left, I_diff_right)); HAL_BILINEAR_8x8_PATCH_EXTRACTION_NEXT_ROW; } sum_diff = v_reduce_sum(sum_diff_vec); diff --git a/modules/video/src/lkpyramid.cpp b/modules/video/src/lkpyramid.cpp index 824159b6ff..d27c4481ae 100644 --- a/modules/video/src/lkpyramid.cpp +++ b/modules/video/src/lkpyramid.cpp @@ -97,8 +97,8 @@ void cv::detail::ScharrDerivInvoker::operator()(const Range& range) const v_int16x8 s1 = v_reinterpret_as_s16(v_load_expand(srow1 + x)); v_int16x8 s2 = v_reinterpret_as_s16(v_load_expand(srow2 + x)); - v_int16x8 t1 = s2 - s0; - v_int16x8 t0 = v_mul_wrap(s0 + s2, c3) + v_mul_wrap(s1, c10); + v_int16x8 t1 = v_sub(s2, s0); + v_int16x8 t0 = v_add(v_mul_wrap(v_add(s0, s2), c3), v_mul_wrap(s1, c10)); v_store(trow0 + x, t0); v_store(trow1 + x, t1); @@ -134,8 +134,8 @@ void cv::detail::ScharrDerivInvoker::operator()(const Range& range) const v_int16x8 s3 = v_load(trow1 + x); v_int16x8 s4 = v_load(trow1 + x + cn); - v_int16x8 t0 = s1 - s0; - v_int16x8 t1 = v_mul_wrap(s2 + s4, c3) + v_mul_wrap(s3, c10); + v_int16x8 t0 = v_sub(s1, s0); + v_int16x8 t1 = v_add(v_mul_wrap(v_add(s2, s4), c3), v_mul_wrap(s3, c10)); v_store_interleave((drow + x*2), t0, t1); } @@ -293,10 +293,10 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const v_zip(v00, v01, t00, t01); v_zip(v10, v11, t10, t11); - t0 = v_dotprod(t00, qw0, qdelta) + v_dotprod(t10, qw1); - t1 = v_dotprod(t01, qw0, qdelta) + v_dotprod(t11, qw1); - t0 = t0 >> (W_BITS1-5); - t1 = t1 >> (W_BITS1-5); + t0 = v_add(v_dotprod(t00, qw0, qdelta), v_dotprod(t10, qw1)); + t1 = v_add(v_dotprod(t01, qw0, qdelta), v_dotprod(t11, qw1)); + t0 = v_shr(t0); + t1 = v_shr(t1); v_store(Iptr + x, v_pack(t0, t1)); v00 = v_reinterpret_as_s16(v_load(dsrc)); @@ -307,10 +307,10 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const v_zip(v00, v01, t00, t01); v_zip(v10, v11, t10, t11); - t0 = v_dotprod(t00, qw0, qdelta_d) + v_dotprod(t10, qw1); - t1 = v_dotprod(t01, qw0, qdelta_d) + v_dotprod(t11, qw1); - t0 = t0 >> W_BITS1; - t1 = t1 >> W_BITS1; + t0 = v_add(v_dotprod(t00, qw0, qdelta_d), v_dotprod(t10, qw1)); + t1 = v_add(v_dotprod(t01, qw0, qdelta_d), v_dotprod(t11, qw1)); + t0 = v_shr(t0); + t1 = v_shr(t1); v00 = v_pack(t0, t1); // Ix0 Iy0 Ix1 Iy1 ... v_store(dIptr, v00); @@ -332,10 +332,10 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const v_zip(v00, v01, t00, t01); v_zip(v10, v11, t10, t11); - t0 = v_dotprod(t00, qw0, qdelta_d) + v_dotprod(t10, qw1); - t1 = v_dotprod(t01, qw0, qdelta_d) + v_dotprod(t11, qw1); - t0 = t0 >> W_BITS1; - t1 = t1 >> W_BITS1; + t0 = v_add(v_dotprod(t00, qw0, qdelta_d), v_dotprod(t10, qw1)); + t1 = v_add(v_dotprod(t01, qw0, qdelta_d), v_dotprod(t11, qw1)); + t0 = v_shr(t0); + t1 = v_shr(t1); v00 = v_pack(t0, t1); // Ix0 Iy0 Ix1 Iy1 ... v_store(dIptr + 4*2, v00); @@ -548,18 +548,18 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const v_zip(v00, v01, t00, t01); v_zip(v10, v11, t10, t11); - t0 = v_dotprod(t00, qw0, qdelta) + v_dotprod(t10, qw1); - t1 = v_dotprod(t01, qw0, qdelta) + v_dotprod(t11, qw1); - t0 = t0 >> (W_BITS1-5); - t1 = t1 >> (W_BITS1-5); - diff0 = v_pack(t0, t1) - diff0; + t0 = v_add(v_dotprod(t00, qw0, qdelta), v_dotprod(t10, qw1)); + t1 = v_add(v_dotprod(t01, qw0, qdelta), v_dotprod(t11, qw1)); + t0 = v_shr(t0); + t1 = v_shr(t1); + diff0 = v_sub(v_pack(t0, t1), diff0); v_zip(diff0, diff0, diff2, diff1); // It0 It0 It1 It1 ... v00 = v_reinterpret_as_s16(v_load(dIptr)); // Ix0 Iy0 Ix1 Iy1 ... v01 = v_reinterpret_as_s16(v_load(dIptr + 8)); v_zip(v00, v01, v10, v11); v_zip(diff2, diff1, v00, v01); - qb0 += v_cvt_f32(v_dotprod(v00, v10)); - qb1 += v_cvt_f32(v_dotprod(v01, v11)); + qb0 = v_add(qb0, v_cvt_f32(v_dotprod(v00, v10))); + qb1 = v_add(qb1, v_cvt_f32(v_dotprod(v01, v11))); } #endif @@ -647,7 +647,7 @@ void cv::detail::LKTrackerInvoker::operator()(const Range& range) const #if CV_SIMD128 && !CV_NEON v_float32x4 qf0, qf1; - v_recombine(v_interleave_pairs(qb0 + qb1), v_setzero_f32(), qf0, qf1); + v_recombine(v_interleave_pairs(v_add(qb0, qb1)), v_setzero_f32(), qf0, qf1); ib1 += v_reduce_sum(qf0); ib2 += v_reduce_sum(qf1); #endif diff --git a/modules/video/src/optflowgf.cpp b/modules/video/src/optflowgf.cpp index 2b164b62d3..02e878a577 100644 --- a/modules/video/src/optflowgf.cpp +++ b/modules/video/src/optflowgf.cpp @@ -463,22 +463,22 @@ FarnebackUpdateFlow_GaussianBlur( const Mat& _R0, const Mat& _R1, const float *sptr0 = srow[m], *sptr1; v_float32x4 g4 = v_load(simd_kernel); v_float32x4 s0, s1, s2, s3; - s0 = v_load(sptr0 + x) * g4; - s1 = v_load(sptr0 + x + 4) * g4; - s2 = v_load(sptr0 + x + 8) * g4; - s3 = v_load(sptr0 + x + 12) * g4; + s0 = v_mul(v_load(sptr0 + x), g4); + s1 = v_mul(v_load(sptr0 + x + 4), g4); + s2 = v_mul(v_load(sptr0 + x + 8), g4); + s3 = v_mul(v_load(sptr0 + x + 12), g4); for( i = 1; i <= m; i++ ) { v_float32x4 x0, x1; sptr0 = srow[m+i], sptr1 = srow[m-i]; g4 = v_load(simd_kernel + i*4); - x0 = v_load(sptr0 + x) + v_load(sptr1 + x); - x1 = v_load(sptr0 + x + 4) + v_load(sptr1 + x + 4); + x0 = v_add(v_load(sptr0 + x), v_load(sptr1 + x)); + x1 = v_add(v_load(sptr0 + x + 4), v_load(sptr1 + x + 4)); s0 = v_muladd(x0, g4, s0); s1 = v_muladd(x1, g4, s1); - x0 = v_load(sptr0 + x + 8) + v_load(sptr1 + x + 8); - x1 = v_load(sptr0 + x + 12) + v_load(sptr1 + x + 12); + x0 = v_add(v_load(sptr0 + x + 8), v_load(sptr1 + x + 8)); + x1 = v_add(v_load(sptr0 + x + 12), v_load(sptr1 + x + 12)); s2 = v_muladd(x0, g4, s2); s3 = v_muladd(x1, g4, s3); } @@ -493,13 +493,13 @@ FarnebackUpdateFlow_GaussianBlur( const Mat& _R0, const Mat& _R1, { const float *sptr0 = srow[m], *sptr1; v_float32x4 g4 = v_load(simd_kernel); - v_float32x4 s0 = v_load(sptr0 + x) * g4; + v_float32x4 s0 = v_mul(v_load(sptr0 + x), g4); for( i = 1; i <= m; i++ ) { sptr0 = srow[m+i], sptr1 = srow[m-i]; g4 = v_load(simd_kernel + i*4); - v_float32x4 x0 = v_load(sptr0 + x) + v_load(sptr1 + x); + v_float32x4 x0 = v_add(v_load(sptr0 + x), v_load(sptr1 + x)); s0 = v_muladd(x0, g4, s0); } v_store(vsum + x, s0); @@ -528,14 +528,14 @@ FarnebackUpdateFlow_GaussianBlur( const Mat& _R0, const Mat& _R1, for( ; x <= width*5 - 8; x += 8 ) { v_float32x4 g4 = v_load(simd_kernel); - v_float32x4 s0 = v_load(vsum + x) * g4; - v_float32x4 s1 = v_load(vsum + x + 4) * g4; + v_float32x4 s0 = v_mul(v_load(vsum + x), g4); + v_float32x4 s1 = v_mul(v_load(vsum + x + 4), g4); for( i = 1; i <= m; i++ ) { g4 = v_load(simd_kernel + i*4); - v_float32x4 x0 = v_load(vsum + x - i*5) + v_load(vsum + x+ i*5); - v_float32x4 x1 = v_load(vsum + x - i*5 + 4) + v_load(vsum + x+ i*5 + 4); + v_float32x4 x0 = v_add(v_load(vsum + x - i * 5), v_load(vsum + x + i * 5)); + v_float32x4 x1 = v_add(v_load(vsum + x - i * 5 + 4), v_load(vsum + x + i * 5 + 4)); s0 = v_muladd(x0, g4, s0); s1 = v_muladd(x1, g4, s1); } diff --git a/modules/video/src/optical_flow_io.cpp b/modules/video/src/optical_flow_io.cpp index 8f9efedc33..ff34f25097 100644 --- a/modules/video/src/optical_flow_io.cpp +++ b/modules/video/src/optical_flow_io.cpp @@ -52,12 +52,12 @@ CV_EXPORTS_W Mat readOpticalFlow( const String& path ) Mat_ flow; std::ifstream file(path.c_str(), std::ios_base::binary); if ( !file.good() ) - return std::move(flow); // no file - return empty matrix + return Mat(); // no file - return empty matrix float tag; file.read((char*) &tag, sizeof(float)); if ( tag != FLOW_TAG_FLOAT ) - return std::move(flow); + return Mat(); int width, height; @@ -76,14 +76,14 @@ CV_EXPORTS_W Mat readOpticalFlow( const String& path ) if ( !file.good() ) { flow.release(); - return std::move(flow); + return Mat(); } flow(i, j) = u; } } file.close(); - return std::move(flow); + return Mat(flow); } CV_EXPORTS_W bool writeOpticalFlow( const String& path, InputArray flow ) diff --git a/modules/video/src/tracking/tracker_vit.cpp b/modules/video/src/tracking/tracker_vit.cpp index 5a6f3e9ac5..b807948033 100644 --- a/modules/video/src/tracking/tracker_vit.cpp +++ b/modules/video/src/tracking/tracker_vit.cpp @@ -45,6 +45,9 @@ public: { net = dnn::readNet(params.net); CV_Assert(!net.empty()); + + net.setPreferableBackend(params.backend); + net.setPreferableTarget(params.target); } void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; diff --git a/modules/video/src/variational_refinement.cpp b/modules/video/src/variational_refinement.cpp index cca30f1ce7..968bce6717 100644 --- a/modules/video/src/variational_refinement.cpp +++ b/modules/video/src/variational_refinement.cpp @@ -651,15 +651,15 @@ void VariationalRefinementImpl::ComputeDataTerm_ParBody::operator()(const Range pdU_vec = v_load(pdU + j); pdV_vec = v_load(pdV + j); - derivNorm_vec = pIx_vec * pIx_vec + pIy_vec * pIy_vec + zeta_vec; - Ik1z_vec = pIz_vec + pIx_vec * pdU_vec + pIy_vec * pdV_vec; - weight_vec = (delta_vec / v_sqrt(Ik1z_vec * Ik1z_vec / derivNorm_vec + eps_vec)) / derivNorm_vec; + derivNorm_vec = v_add(v_add(v_mul(pIx_vec, pIx_vec), v_mul(pIy_vec, pIy_vec)), zeta_vec); + Ik1z_vec = v_add(v_add(pIz_vec, v_mul(pIx_vec, pdU_vec)), v_mul(pIy_vec, pdV_vec)); + weight_vec = v_div(v_div(delta_vec, v_sqrt(v_add(v_div(v_mul(Ik1z_vec, Ik1z_vec), derivNorm_vec), eps_vec))), derivNorm_vec); - pa11_vec = weight_vec * (pIx_vec * pIx_vec) + zeta_vec; - pa12_vec = weight_vec * (pIx_vec * pIy_vec); - pa22_vec = weight_vec * (pIy_vec * pIy_vec) + zeta_vec; - pb1_vec = zero_vec - weight_vec * (pIz_vec * pIx_vec); - pb2_vec = zero_vec - weight_vec * (pIz_vec * pIy_vec); + pa11_vec = v_add(v_mul(weight_vec, v_mul(pIx_vec, pIx_vec)), zeta_vec); + pa12_vec = v_mul(weight_vec, v_mul(pIx_vec, pIy_vec)); + pa22_vec = v_add(v_mul(weight_vec, v_mul(pIy_vec, pIy_vec)), zeta_vec); + pb1_vec = v_sub(zero_vec, v_mul(weight_vec, v_mul(pIz_vec, pIx_vec))); + pb2_vec = v_sub(zero_vec, v_mul(weight_vec, v_mul(pIz_vec, pIy_vec))); pIxx_vec = v_load(pIxx + j); pIxy_vec = v_load(pIxy + j); @@ -667,18 +667,17 @@ void VariationalRefinementImpl::ComputeDataTerm_ParBody::operator()(const Range pIxz_vec = v_load(pIxz + j); pIyz_vec = v_load(pIyz + j); - derivNorm_vec = pIxx_vec * pIxx_vec + pIxy_vec * pIxy_vec + zeta_vec; - derivNorm2_vec = pIyy_vec * pIyy_vec + pIxy_vec * pIxy_vec + zeta_vec; - Ik1zx_vec = pIxz_vec + pIxx_vec * pdU_vec + pIxy_vec * pdV_vec; - Ik1zy_vec = pIyz_vec + pIxy_vec * pdU_vec + pIyy_vec * pdV_vec; - weight_vec = gamma_vec / v_sqrt(Ik1zx_vec * Ik1zx_vec / derivNorm_vec + - Ik1zy_vec * Ik1zy_vec / derivNorm2_vec + eps_vec); + derivNorm_vec = v_add(v_add(v_mul(pIxx_vec, pIxx_vec), v_mul(pIxy_vec, pIxy_vec)), zeta_vec); + derivNorm2_vec = v_add(v_add(v_mul(pIyy_vec, pIyy_vec), v_mul(pIxy_vec, pIxy_vec)), zeta_vec); + Ik1zx_vec = v_add(v_add(pIxz_vec, v_mul(pIxx_vec, pdU_vec)), v_mul(pIxy_vec, pdV_vec)); + Ik1zy_vec = v_add(v_add(pIyz_vec, v_mul(pIxy_vec, pdU_vec)), v_mul(pIyy_vec, pdV_vec)); + weight_vec = v_div(gamma_vec, v_sqrt(v_add(v_add(v_div(v_mul(Ik1zx_vec, Ik1zx_vec), derivNorm_vec), v_div(v_mul(Ik1zy_vec, Ik1zy_vec), derivNorm2_vec)), eps_vec))); - pa11_vec += weight_vec * (pIxx_vec * pIxx_vec / derivNorm_vec + pIxy_vec * pIxy_vec / derivNorm2_vec); - pa12_vec += weight_vec * (pIxx_vec * pIxy_vec / derivNorm_vec + pIxy_vec * pIyy_vec / derivNorm2_vec); - pa22_vec += weight_vec * (pIxy_vec * pIxy_vec / derivNorm_vec + pIyy_vec * pIyy_vec / derivNorm2_vec); - pb1_vec -= weight_vec * (pIxx_vec * pIxz_vec / derivNorm_vec + pIxy_vec * pIyz_vec / derivNorm2_vec); - pb2_vec -= weight_vec * (pIxy_vec * pIxz_vec / derivNorm_vec + pIyy_vec * pIyz_vec / derivNorm2_vec); + pa11_vec = v_add(pa11_vec, v_mul(weight_vec, v_add(v_div(v_mul(pIxx_vec, pIxx_vec), derivNorm_vec), v_div(v_mul(pIxy_vec, pIxy_vec), derivNorm2_vec)))); + pa12_vec = v_add(pa12_vec, v_mul(weight_vec, v_add(v_div(v_mul(pIxx_vec, pIxy_vec), derivNorm_vec), v_div(v_mul(pIxy_vec, pIyy_vec), derivNorm2_vec)))); + pa22_vec = v_add(pa22_vec, v_mul(weight_vec, v_add(v_div(v_mul(pIxy_vec, pIxy_vec), derivNorm_vec), v_div(v_mul(pIyy_vec, pIyy_vec), derivNorm2_vec)))); + pb1_vec = v_sub(pb1_vec, v_mul(weight_vec, v_add(v_div(v_mul(pIxx_vec, pIxz_vec), derivNorm_vec), v_div(v_mul(pIxy_vec, pIyz_vec), derivNorm2_vec)))); + pb2_vec = v_sub(pb2_vec, v_mul(weight_vec, v_add(v_div(v_mul(pIxy_vec, pIxz_vec), derivNorm_vec), v_div(v_mul(pIyy_vec, pIyz_vec), derivNorm2_vec)))); v_store(pa11 + j, pa11_vec); v_store(pa12 + j, pa12_vec); @@ -850,26 +849,26 @@ void VariationalRefinementImpl::ComputeSmoothnessTermHorPass_ParBody::operator() cW_u_vec = v_load(cW_u + j); cW_v_vec = v_load(cW_v + j); - ux_vec = v_load(cW_u_next + j) - cW_u_vec; - vx_vec = v_load(cW_v_next + j) - cW_v_vec; - uy_vec = v_load(cW_u_next_row + j) - cW_u_vec; - vy_vec = v_load(cW_v_next_row + j) - cW_v_vec; + ux_vec = v_sub(v_load(cW_u_next + j), cW_u_vec); + vx_vec = v_sub(v_load(cW_v_next + j), cW_v_vec); + uy_vec = v_sub(v_load(cW_u_next_row + j), cW_u_vec); + vy_vec = v_sub(v_load(cW_v_next_row + j), cW_v_vec); pWeight_vec = - alpha2_vec / v_sqrt(ux_vec * ux_vec + vx_vec * vx_vec + uy_vec * uy_vec + vy_vec * vy_vec + eps_vec); + v_div(alpha2_vec, v_sqrt(v_add(v_add(v_add(v_add(v_mul(ux_vec, ux_vec), v_mul(vx_vec, vx_vec)), v_mul(uy_vec, uy_vec)), v_mul(vy_vec, vy_vec)), eps_vec))); v_store(pWeight + j, pWeight_vec); - ux_vec = pWeight_vec * (v_load(pW_u_next + j) - v_load(pW_u + j)); - vx_vec = pWeight_vec * (v_load(pW_v_next + j) - v_load(pW_v + j)); + ux_vec = v_mul(pWeight_vec, v_sub(v_load(pW_u_next + j), v_load(pW_u + j))); + vx_vec = v_mul(pWeight_vec, v_sub(v_load(pW_v_next + j), v_load(pW_v + j))); - v_store(pA_u + j, v_load(pA_u + j) + pWeight_vec); - v_store(pA_v + j, v_load(pA_v + j) + pWeight_vec); - v_store(pB_u + j, v_load(pB_u + j) + ux_vec); - v_store(pB_v + j, v_load(pB_v + j) + vx_vec); + v_store(pA_u + j, v_add(v_load(pA_u + j), pWeight_vec)); + v_store(pA_v + j, v_add(v_load(pA_v + j), pWeight_vec)); + v_store(pB_u + j, v_add(v_load(pB_u + j), ux_vec)); + v_store(pB_v + j, v_add(v_load(pB_v + j), vx_vec)); - v_store(pA_u_next + j, v_load(pA_u_next + j) + pWeight_vec); - v_store(pA_v_next + j, v_load(pA_v_next + j) + pWeight_vec); - v_store(pB_u_next + j, v_load(pB_u_next + j) - ux_vec); - v_store(pB_v_next + j, v_load(pB_v_next + j) - vx_vec); + v_store(pA_u_next + j, v_add(v_load(pA_u_next + j), pWeight_vec)); + v_store(pA_v_next + j, v_add(v_load(pA_v_next + j), pWeight_vec)); + v_store(pB_u_next + j, v_sub(v_load(pB_u_next + j), ux_vec)); + v_store(pB_v_next + j, v_sub(v_load(pB_v_next + j), vx_vec)); } #endif for (; j < len - 1; j++) @@ -956,18 +955,18 @@ void VariationalRefinementImpl::ComputeSmoothnessTermVertPass_ParBody::operator( for (; j < len - 3; j += 4) { pWeight_vec = v_load(pWeight + j); - uy_vec = pWeight_vec * (v_load(pW_u_next_row + j) - v_load(pW_u + j)); - vy_vec = pWeight_vec * (v_load(pW_v_next_row + j) - v_load(pW_v + j)); + uy_vec = v_mul(pWeight_vec, v_sub(v_load(pW_u_next_row + j), v_load(pW_u + j))); + vy_vec = v_mul(pWeight_vec, v_sub(v_load(pW_v_next_row + j), v_load(pW_v + j))); - v_store(pA_u + j, v_load(pA_u + j) + pWeight_vec); - v_store(pA_v + j, v_load(pA_v + j) + pWeight_vec); - v_store(pB_u + j, v_load(pB_u + j) + uy_vec); - v_store(pB_v + j, v_load(pB_v + j) + vy_vec); + v_store(pA_u + j, v_add(v_load(pA_u + j), pWeight_vec)); + v_store(pA_v + j, v_add(v_load(pA_v + j), pWeight_vec)); + v_store(pB_u + j, v_add(v_load(pB_u + j), uy_vec)); + v_store(pB_v + j, v_add(v_load(pB_v + j), vy_vec)); - v_store(pA_u_next_row + j, v_load(pA_u_next_row + j) + pWeight_vec); - v_store(pA_v_next_row + j, v_load(pA_v_next_row + j) + pWeight_vec); - v_store(pB_u_next_row + j, v_load(pB_u_next_row + j) - uy_vec); - v_store(pB_v_next_row + j, v_load(pB_v_next_row + j) - vy_vec); + v_store(pA_u_next_row + j, v_add(v_load(pA_u_next_row + j), pWeight_vec)); + v_store(pA_v_next_row + j, v_add(v_load(pA_v_next_row + j), pWeight_vec)); + v_store(pB_u_next_row + j, v_sub(v_load(pB_u_next_row + j), uy_vec)); + v_store(pB_v_next_row + j, v_sub(v_load(pB_v_next_row + j), vy_vec)); } #endif for (; j < len; j++) @@ -1084,15 +1083,13 @@ void VariationalRefinementImpl::RedBlackSOR_ParBody::operator()(const Range &ran pdv_shifted_vec = v_reinterpret_as_f32( v_extract<3>(v_reinterpret_as_s32(pdv_prev_vec), v_reinterpret_as_s32(pdv_next_vec))); - sigmaU_vec = pW_shifted_vec * pdu_shifted_vec + pW_vec * pdu_next_vec + pW_prev_row_vec * pdu_prev_row_vec + - pW_vec * pdu_next_row_vec; - sigmaV_vec = pW_shifted_vec * pdv_shifted_vec + pW_vec * pdv_next_vec + pW_prev_row_vec * pdv_prev_row_vec + - pW_vec * pdv_next_row_vec; + sigmaU_vec = v_add(v_add(v_add(v_mul(pW_shifted_vec, pdu_shifted_vec), v_mul(pW_vec, pdu_next_vec)), v_mul(pW_prev_row_vec, pdu_prev_row_vec)), v_mul(pW_vec, pdu_next_row_vec)); + sigmaV_vec = v_add(v_add(v_add(v_mul(pW_shifted_vec, pdv_shifted_vec), v_mul(pW_vec, pdv_next_vec)), v_mul(pW_prev_row_vec, pdv_prev_row_vec)), v_mul(pW_vec, pdv_next_row_vec)); pdu_vec = v_load(pdu + j); pdv_vec = v_load(pdv + j); - pdu_vec += omega_vec * ((sigmaU_vec + v_load(pb1 + j) - pdv_vec * pa12_vec) / v_load(pa11 + j) - pdu_vec); - pdv_vec += omega_vec * ((sigmaV_vec + v_load(pb2 + j) - pdu_vec * pa12_vec) / v_load(pa22 + j) - pdv_vec); + pdu_vec = v_add(pdu_vec, v_mul(omega_vec, v_sub(v_div(v_sub(v_add(sigmaU_vec, v_load(pb1 + j)), v_mul(pdv_vec, pa12_vec)), v_load(pa11 + j)), pdu_vec))); + pdv_vec = v_add(pdv_vec, v_mul(omega_vec, v_sub(v_div(v_sub(v_add(sigmaV_vec, v_load(pb2 + j)), v_mul(pdu_vec, pa12_vec)), v_load(pa22 + j)), pdv_vec))); v_store(pdu + j, pdu_vec); v_store(pdv + j, pdv_vec); diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 7e9c5b304a..0e69c27731 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -219,10 +219,13 @@ enum VideoWriterProperties { VIDEOWRITER_PROP_NSTRIPES = 3, //!< Number of stripes for parallel encoding. -1 for auto detection. VIDEOWRITER_PROP_IS_COLOR = 4, //!< If it is not zero, the encoder will expect and encode color frames, otherwise it //!< will work with grayscale frames. - VIDEOWRITER_PROP_DEPTH = 5, //!< Defaults to CV_8U. + VIDEOWRITER_PROP_DEPTH = 5, //!< Defaults to \ref CV_8U. VIDEOWRITER_PROP_HW_ACCELERATION = 6, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in VideoWriter constructor / .open() method. Default value is backend-specific. VIDEOWRITER_PROP_HW_DEVICE = 7, //!< (**open-only**) Hardware device index (select GPU if multiple available). Device enumeration is acceleration type specific. VIDEOWRITER_PROP_HW_ACCELERATION_USE_OPENCL= 8, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between cv::UMat and HW accelerated encoder. + VIDEOWRITER_PROP_RAW_VIDEO = 9, //!< (**open-only**) Set to non-zero to enable encapsulation of an encoded raw video stream. Each raw encoded video frame should be passed to VideoWriter::write() as single row or column of a \ref CV_8UC1 Mat. \note If the key frame interval is not 1 then it must be manually specified by the user. This can either be performed during initialization passing \ref VIDEOWRITER_PROP_KEY_INTERVAL as one of the extra encoder params to \ref VideoWriter::VideoWriter(const String &, int, double, const Size &, const std::vector< int > ¶ms) or afterwards by setting the \ref VIDEOWRITER_PROP_KEY_FLAG with \ref VideoWriter::set() before writing each frame. FFMpeg backend only. + VIDEOWRITER_PROP_KEY_INTERVAL = 10, //!< (**open-only**) Set the key frame interval using raw video encapsulation (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). Defaults to 1 when not set. FFMpeg backend only. + VIDEOWRITER_PROP_KEY_FLAG = 11, //!< Set to non-zero to signal that the following frames are key frames or zero if not, when encapsulating raw video (\ref VIDEOWRITER_PROP_RAW_VIDEO != 0). FFMpeg backend only. #ifndef CV_DOXYGEN CV__VIDEOWRITER_PROP_LATEST #endif diff --git a/modules/videoio/src/cap_ffmpeg.cpp b/modules/videoio/src/cap_ffmpeg.cpp index 764bc33864..ba509a3c42 100644 --- a/modules/videoio/src/cap_ffmpeg.cpp +++ b/modules/videoio/src/cap_ffmpeg.cpp @@ -198,7 +198,11 @@ public: return ffmpegWriter->getProperty(propId); } - virtual bool setProperty(int, double) CV_OVERRIDE { return false; } + virtual bool setProperty(int propId, double value) CV_OVERRIDE { + if (!ffmpegWriter) + return 0; + return ffmpegWriter->setProperty(propId, value); + } virtual bool isOpened() const CV_OVERRIDE { return ffmpegWriter != 0; } protected: diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index e4431b323e..bf259b4daf 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -1447,7 +1447,8 @@ bool CvCapture_FFMPEG::processRawPacket() #else AVCodecContext* ctx = ic->streams[video_stream]->codec; int err = av_bitstream_filter_filter(bsfc, ctx, NULL, &packet_filtered.data, - &packet_filtered.size, packet.data, packet.size, packet_filtered.flags & AV_PKT_FLAG_KEY); + &packet_filtered.size, packet.data, packet.size, packet.flags & AV_PKT_FLAG_KEY); + if (packet.flags & AV_PKT_FLAG_KEY) packet_filtered.flags |= AV_PKT_FLAG_KEY; if (err < 0) { CV_WARN("Packet filtering failed"); @@ -2069,6 +2070,7 @@ struct CvVideoWriter_FFMPEG bool writeFrame( const unsigned char* data, int step, int width, int height, int cn, int origin ); bool writeHWFrame(cv::InputArray input); double getProperty(int propId) const; + bool setProperty(int, double); void init(); @@ -2092,6 +2094,9 @@ struct CvVideoWriter_FFMPEG VideoAccelerationType va_type; int hw_device; int use_opencl; + bool encode_video; + int idr_period; + bool key_frame; }; static const char * icvFFMPEGErrStr(int err) @@ -2157,6 +2162,9 @@ void CvVideoWriter_FFMPEG::init() hw_device = -1; use_opencl = 0; ok = false; + encode_video = true; + idr_period = 0; + key_frame = false; } /** @@ -2202,7 +2210,7 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc, AVStream *st, const AVCodec* codec, int w, int h, int bitrate, - double fps, AVPixelFormat pixel_format, int fourcc) + double fps, AVPixelFormat pixel_format, int fourcc, AVCodecID codec_id) { #ifdef CV_FFMPEG_CODECPAR AVCodecContext *c = avcodec_alloc_context3(codec); @@ -2213,9 +2221,7 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc, int frame_rate, frame_rate_base; - c->codec_id = codec->id; - c->codec_type = AVMEDIA_TYPE_VIDEO; - c->codec_tag = fourcc; + c->codec_id = codec ? codec->id : codec_id; #ifndef CV_FFMPEG_CODECPAR // Set per-codec defaults @@ -2225,6 +2231,9 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc, c->codec_id = c_id; #endif + c->codec_type = AVMEDIA_TYPE_VIDEO; + c->codec_tag = fourcc; + /* put sample parameters */ int64_t lbit_rate = (int64_t)bitrate; lbit_rate += (bitrate / 2); @@ -2323,6 +2332,29 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc, static const int OPENCV_NO_FRAMES_WRITTEN_CODE = 1000; +static int icv_av_encapsulate_video_FFMPEG(AVFormatContext* oc, AVStream* video_st, AVCodecContext* c, + uint8_t* data, int sz, const int frame_idx, const bool key_frame) +{ +#if LIBAVFORMAT_BUILD < CALC_FFMPEG_VERSION(57, 0, 0) + AVPacket pkt_; + av_init_packet(&pkt_); + AVPacket* pkt = &pkt_; +#else + AVPacket* pkt = av_packet_alloc(); +#endif + if(key_frame) + pkt->flags |= PKT_FLAG_KEY; + pkt->pts = frame_idx; + pkt->size = sz; + pkt->data = data; + av_packet_rescale_ts(pkt, c->time_base, video_st->time_base); + int ret = av_write_frame(oc, pkt); +#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(57, 0, 0) + av_packet_free(&pkt); +#endif + return ret; +} + static int icv_av_write_frame_FFMPEG( AVFormatContext * oc, AVStream * video_st, AVCodecContext * c, uint8_t *, uint32_t, AVFrame * picture, int frame_idx) @@ -2404,6 +2436,14 @@ static int icv_av_write_frame_FFMPEG( AVFormatContext * oc, AVStream * video_st, /// write a frame with FFMPEG bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int width, int height, int cn, int origin ) { + if (!encode_video) { + CV_Assert(cn == 1 && ((width > 0 && height == 1) || (width == 1 && height > 0 && step == 1))); + const bool set_key_frame = key_frame ? key_frame : idr_period ? frame_idx % idr_period == 0 : 1; + bool ret = icv_av_encapsulate_video_FFMPEG(oc, video_st, context, (uint8_t*)data, width, frame_idx, set_key_frame); + frame_idx++; + return ret; + } + // check parameters if (input_pix_fmt == AV_PIX_FMT_BGR24) { if (cn != 3) { @@ -2592,6 +2632,21 @@ double CvVideoWriter_FFMPEG::getProperty(int propId) const return 0; } +bool CvVideoWriter_FFMPEG::setProperty(int property_id, double value) +{ + if (!video_st) return false; + + switch (property_id) + { + case VIDEOWRITER_PROP_KEY_FLAG: + key_frame = static_cast(value); + break; + default: + return false; + } + return true; +} + /// close video output stream and free associated memory void CvVideoWriter_FFMPEG::close() { @@ -2601,17 +2656,19 @@ void CvVideoWriter_FFMPEG::close() // TODO -- do we need to account for latency here? /* write the trailer, if any */ - if (picture && ok && oc) + if ((!encode_video || picture) && ok && oc) { #if LIBAVFORMAT_BUILD < CALC_FFMPEG_VERSION(57, 0, 0) if (!(oc->oformat->flags & AVFMT_RAWPICTURE)) #endif { - for(;;) - { - int ret = icv_av_write_frame_FFMPEG( oc, video_st, context, outbuf, outbuf_size, NULL, frame_idx); - if( ret == OPENCV_NO_FRAMES_WRITTEN_CODE || ret < 0 ) - break; + if (encode_video) { + for (;;) + { + int ret = icv_av_write_frame_FFMPEG(oc, video_st, context, outbuf, outbuf_size, NULL, frame_idx); + if (ret == OPENCV_NO_FRAMES_WRITTEN_CODE || ret < 0) + break; + } } } av_write_trailer(oc); @@ -2720,6 +2777,8 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, close(); + encode_video = !params.get(VIDEOWRITER_PROP_RAW_VIDEO, false); + idr_period = params.get(VIDEOWRITER_PROP_KEY_INTERVAL, 0); const bool is_color = params.get(VIDEOWRITER_PROP_IS_COLOR, true); const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U); const bool is_supported = depth == CV_8U || (depth == CV_16U && !is_color); @@ -2770,13 +2829,15 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, if(fps <= 0) return false; - // we allow frames of odd width or height, but in this case we truncate - // the rightmost column/the bottom row. Probably, this should be handled more elegantly, - // but some internal functions inside FFMPEG swscale require even width/height. - width &= -2; - height &= -2; - if( width <= 0 || height <= 0 ) - return false; + if (encode_video) { + // we allow frames of odd width or height, but in this case we truncate + // the rightmost column/the bottom row. Probably, this should be handled more elegantly, + // but some internal functions inside FFMPEG swscale require even width/height. + width &= -2; + height &= -2; + if (width <= 0 || height <= 0) + return false; + } /* auto detect the output format from the name and fourcc code. */ @@ -3027,41 +3088,46 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, HWAccelIterator accel_iter(va_type, true/*isEncoder*/, dict); while (accel_iter.good()) { + AVPixelFormat hw_format = AV_PIX_FMT_NONE; + AVHWDeviceType hw_type = AV_HWDEVICE_TYPE_NONE; #else do { #endif + if (encode_video) { #if USE_AV_HW_CODECS - accel_iter.parse_next(); - AVHWDeviceType hw_type = accel_iter.hw_type(); - codec = NULL; - AVPixelFormat hw_format = AV_PIX_FMT_NONE; - if (hw_device_ctx) - av_buffer_unref(&hw_device_ctx); - if (hw_type != AV_HWDEVICE_TYPE_NONE) - { - codec = hw_find_codec(codec_id, hw_type, av_codec_is_encoder, accel_iter.disabled_codecs().c_str(), &hw_format); + accel_iter.parse_next(); + hw_type = accel_iter.hw_type(); + codec = NULL; + hw_format = AV_PIX_FMT_NONE; + if (hw_device_ctx) + av_buffer_unref(&hw_device_ctx); + if (hw_type != AV_HWDEVICE_TYPE_NONE) + { + codec = hw_find_codec(codec_id, hw_type, av_codec_is_encoder, accel_iter.disabled_codecs().c_str(), &hw_format); + if (!codec) + continue; + + hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname(), use_opencl != 0); + if (!hw_device_ctx) + continue; + } + else if (hw_type == AV_HWDEVICE_TYPE_NONE) +#endif + { + codec = avcodec_find_encoder(codec_id); + if (!codec) { + CV_LOG_ERROR(NULL, "Could not find encoder for codec_id=" << (int)codec_id << ", error: " + << icvFFMPEGErrStr(AVERROR_ENCODER_NOT_FOUND)); + } + } if (!codec) continue; + } - hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname(), use_opencl != 0); - if (!hw_device_ctx) - continue; - } - else if (hw_type == AV_HWDEVICE_TYPE_NONE) -#endif - { - codec = avcodec_find_encoder(codec_id); - if (!codec) { - CV_LOG_ERROR(NULL, "Could not find encoder for codec_id=" << (int)codec_id << ", error: " - << icvFFMPEGErrStr(AVERROR_ENCODER_NOT_FOUND)); - } - } - if (!codec) - continue; #if USE_AV_HW_CODECS - AVPixelFormat format = (hw_format != AV_PIX_FMT_NONE) ? hw_format : codec_pix_fmt; + AVPixelFormat format = (hw_format != AV_PIX_FMT_NONE) ? hw_format : codec_pix_fmt; #else - AVPixelFormat format = codec_pix_fmt; + AVPixelFormat format = codec_pix_fmt; #endif #ifdef CV_FFMPEG_CODECPAR @@ -3069,7 +3135,7 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, #endif context = icv_configure_video_stream_FFMPEG(oc, video_st, codec, width, height, (int) (bitrate + 0.5), - fps, format, fourcc); + fps, format, fourcc, codec_id); if (!context) { continue; @@ -3082,17 +3148,18 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, av_dump_format(oc, 0, filename, 1); #endif #endif - + if (encode_video) { #if USE_AV_HW_CODECS - if (hw_device_ctx) { - context->hw_device_ctx = av_buffer_ref(hw_device_ctx); - if (hw_format != AV_PIX_FMT_NONE) { - context->hw_frames_ctx = hw_create_frames(NULL, hw_device_ctx, width, height, hw_format); - if (!context->hw_frames_ctx) - continue; + if (hw_device_ctx) { + context->hw_device_ctx = av_buffer_ref(hw_device_ctx); + if (hw_format != AV_PIX_FMT_NONE) { + context->hw_frames_ctx = hw_create_frames(NULL, hw_device_ctx, width, height, hw_format); + if (!context->hw_frames_ctx) + continue; + } } - } #endif + } int64_t lbit_rate = (int64_t) context->bit_rate; lbit_rate += (int64_t)(bitrate / 2); @@ -3101,7 +3168,7 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, context->bit_rate = (int) lbit_rate; /* open the codec */ - err = avcodec_open2(context, codec, NULL); + err = !encode_video ? 0 : avcodec_open2(context, codec, NULL); if (err >= 0) { #if USE_AV_HW_CODECS va_type = hw_type_to_va_type(hw_type); @@ -3137,43 +3204,43 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, avcodec_parameters_from_context(video_st->codecpar, context); #endif - outbuf = NULL; - - + if (encode_video) { + outbuf = NULL; #if LIBAVFORMAT_BUILD < CALC_FFMPEG_VERSION(57, 0, 0) - if (!(oc->oformat->flags & AVFMT_RAWPICTURE)) + if (!(oc->oformat->flags & AVFMT_RAWPICTURE)) #endif - { - /* allocate output buffer */ - /* assume we will never get codec output with more than 4 bytes per pixel... */ - outbuf_size = width*height*4; - outbuf = (uint8_t *) av_malloc(outbuf_size); - } + { + /* allocate output buffer */ + /* assume we will never get codec output with more than 4 bytes per pixel... */ + outbuf_size = width * height * 4; + outbuf = (uint8_t*)av_malloc(outbuf_size); + } - bool need_color_convert; - AVPixelFormat sw_pix_fmt = context->pix_fmt; + bool need_color_convert; + AVPixelFormat sw_pix_fmt = context->pix_fmt; #if USE_AV_HW_CODECS - if (context->hw_frames_ctx) - sw_pix_fmt = ((AVHWFramesContext*)context->hw_frames_ctx->data)->sw_format; + if (context->hw_frames_ctx) + sw_pix_fmt = ((AVHWFramesContext*)context->hw_frames_ctx->data)->sw_format; #endif - need_color_convert = (sw_pix_fmt != input_pix_fmt); + need_color_convert = (sw_pix_fmt != input_pix_fmt); - /* allocate the encoded raw picture */ - picture = icv_alloc_picture_FFMPEG(sw_pix_fmt, context->width, context->height, need_color_convert); - if (!picture) { - return false; - } - - /* if the output format is not our input format, then a temporary - picture of the input format is needed too. It is then converted - to the required output format */ - input_picture = NULL; - if ( need_color_convert ) { - input_picture = icv_alloc_picture_FFMPEG(input_pix_fmt, context->width, context->height, false); - if (!input_picture) { + /* allocate the encoded raw picture */ + picture = icv_alloc_picture_FFMPEG(sw_pix_fmt, context->width, context->height, need_color_convert); + if (!picture) { return false; } + + /* if the output format is not our input format, then a temporary + picture of the input format is needed too. It is then converted + to the required output format */ + input_picture = NULL; + if (need_color_convert) { + input_picture = icv_alloc_picture_FFMPEG(input_pix_fmt, context->width, context->height, false); + if (!input_picture) { + return false; + } + } } /* open the output file, if needed */ diff --git a/modules/videoio/test/test_ffmpeg.cpp b/modules/videoio/test/test_ffmpeg.cpp index 0496b8c369..9f59480c9c 100644 --- a/modules/videoio/test/test_ffmpeg.cpp +++ b/modules/videoio/test/test_ffmpeg.cpp @@ -235,6 +235,206 @@ const videoio_container_params_t videoio_container_params[] = INSTANTIATE_TEST_CASE_P(/**/, videoio_container, testing::ValuesIn(videoio_container_params)); +typedef tuple videoio_encapsulate_params_t; +typedef testing::TestWithParam< videoio_encapsulate_params_t > videoio_encapsulate; + +TEST_P(videoio_encapsulate, write) +{ +#ifdef _WIN32 + throw SkipTestException("Test disabled until PR for raw video encapsulation is merged and windows dll is updated"); +#else + const VideoCaptureAPIs api = CAP_FFMPEG; + if (!videoio_registry::hasBackend(api)) + throw SkipTestException("FFmpeg backend was not found"); + + const string fileName = findDataFile(get<0>(GetParam())); + const string ext = get<1>(GetParam()); + const int idrPeriod = get<2>(GetParam()); + const int nFrames = get<3>(GetParam()); + const string fileNameOut = tempfile(cv::format("test_encapsulated_stream.%s", ext.c_str()).c_str()); + + // Use VideoWriter to encapsulate encoded video read with VideoReader + { + VideoCapture capRaw(fileName, api, { CAP_PROP_FORMAT, -1 }); + ASSERT_TRUE(capRaw.isOpened()); + const int width = static_cast(capRaw.get(CAP_PROP_FRAME_WIDTH)); + const int height = static_cast(capRaw.get(CAP_PROP_FRAME_HEIGHT)); + const double fps = capRaw.get(CAP_PROP_FPS); + const int codecExtradataIndex = static_cast(capRaw.get(CAP_PROP_CODEC_EXTRADATA_INDEX)); + Mat extraData; + capRaw.retrieve(extraData, codecExtradataIndex); + const int fourcc = static_cast(capRaw.get(CAP_PROP_FOURCC)); + const bool mpeg4 = (fourcc == fourccFromString("FMP4")); + + VideoWriter container(fileNameOut, api, fourcc, fps, { width, height }, { VideoWriterProperties::VIDEOWRITER_PROP_RAW_VIDEO, 1, VideoWriterProperties::VIDEOWRITER_PROP_KEY_INTERVAL, idrPeriod }); + ASSERT_TRUE(container.isOpened()); + Mat rawFrame; + for (int i = 0; i < nFrames; i++) { + ASSERT_TRUE(capRaw.read(rawFrame)); + ASSERT_FALSE(rawFrame.empty()); + if (i == 0 && mpeg4) { + Mat tmp = rawFrame.clone(); + const size_t newSzt = tmp.total() + extraData.total(); + const int newSz = static_cast(newSzt); + ASSERT_TRUE(newSzt == static_cast(newSz)); + rawFrame = Mat(1, newSz, CV_8UC1); + memcpy(rawFrame.data, extraData.data, extraData.total()); + memcpy(rawFrame.data + extraData.total(), tmp.data, tmp.total()); + } + container.write(rawFrame); + } + container.release(); + } + + std::cout << "Checking encapsulated video container: " << fileNameOut << std::endl; + + // Check encapsulated video container is "identical" to the original + { + VideoCapture capReference(fileName), capActual(fileNameOut), capActualRaw(fileNameOut, api, { CAP_PROP_FORMAT, -1 }); + ASSERT_TRUE(capReference.isOpened()); + ASSERT_TRUE(capActual.isOpened()); + ASSERT_TRUE(capActualRaw.isOpened()); + const double fpsReference = capReference.get(CAP_PROP_FPS); + const double fpsActual = capActual.get(CAP_PROP_FPS); + ASSERT_EQ(fpsReference, fpsActual); + const int nFramesActual = static_cast(capActual.get(CAP_PROP_FRAME_COUNT)); + ASSERT_EQ(nFrames, nFramesActual); + + Mat reference, actual; + for (int i = 0; i < nFrames; i++) { + ASSERT_TRUE(capReference.read(reference)); + ASSERT_FALSE(reference.empty()); + ASSERT_TRUE(capActual.read(actual)); + ASSERT_FALSE(actual.empty()); + ASSERT_EQ(0, cvtest::norm(reference, actual, NORM_INF)); + + ASSERT_TRUE(capActualRaw.grab()); + const bool keyFrameActual = capActualRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME) == 1.; + const bool keyFrameReference = idrPeriod ? i % idrPeriod == 0 : 1; + ASSERT_EQ(keyFrameReference, keyFrameActual); + } + } + + ASSERT_EQ(0, remove(fileNameOut.c_str())); +#endif +} + +const videoio_encapsulate_params_t videoio_encapsulate_params[] = +{ + videoio_encapsulate_params_t("video/big_buck_bunny.h264", "avi", 125, 125), + videoio_encapsulate_params_t("video/big_buck_bunny.h265", "mp4", 125, 125), + videoio_encapsulate_params_t("video/big_buck_bunny.wmv", "wmv", 12, 13), + videoio_encapsulate_params_t("video/big_buck_bunny.mp4", "mp4", 12, 13), + videoio_encapsulate_params_t("video/big_buck_bunny.mjpg.avi", "mp4", 0, 4), + videoio_encapsulate_params_t("video/big_buck_bunny.mov", "mp4", 12, 13), + videoio_encapsulate_params_t("video/big_buck_bunny.avi", "mp4", 125, 125), + videoio_encapsulate_params_t("video/big_buck_bunny.mpg", "mp4", 12, 13), + videoio_encapsulate_params_t("video/VID00003-20100701-2204.wmv", "wmv", 12, 13), + videoio_encapsulate_params_t("video/VID00003-20100701-2204.mpg", "mp4", 12,13), + videoio_encapsulate_params_t("video/VID00003-20100701-2204.avi", "mp4", 12, 13), + videoio_encapsulate_params_t("video/VID00003-20100701-2204.3GP", "mp4", 51, 52), + videoio_encapsulate_params_t("video/sample_sorenson.avi", "mp4", 12, 13), + videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libxvid.mp4", "mp4", 3, 4), + videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.mpeg2video.mp4", "mp4", 12, 13), + videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.mjpeg.mp4", "mp4", 0, 5), + videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libx264.mp4", "avi", 15, 15), + videoio_encapsulate_params_t("../cv/tracking/faceocc2/data/faceocc2.webm", "webm", 128, 129), + videoio_encapsulate_params_t("../cv/video/1920x1080.avi", "mp4", 12, 13), + videoio_encapsulate_params_t("../cv/video/768x576.avi", "avi", 15, 16) + // Not supported by with FFmpeg: + //videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libx265.mp4", "mp4", 15, 15), + //videoio_encapsulate_params_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "mp4", 15, 15), + +}; + +INSTANTIATE_TEST_CASE_P(/**/, videoio_encapsulate, testing::ValuesIn(videoio_encapsulate_params)); + +TEST(videoio_encapsulate_set_idr, write) +{ +#ifdef _WIN32 + throw SkipTestException("Test disabled until PR for raw video encapsulation is merged and windows dll is updated"); +#else + const VideoCaptureAPIs api = CAP_FFMPEG; + if (!videoio_registry::hasBackend(api)) + throw SkipTestException("FFmpeg backend was not found"); + + const string fileName = findDataFile("video/big_buck_bunny.mp4"); + const string ext = "mp4"; + const string fileNameOut = tempfile(cv::format("test_encapsulated_stream_set_idr.%s", ext.c_str()).c_str()); + + // Use VideoWriter to encapsulate encoded video read with VideoReader + { + VideoCapture capRaw(fileName, api, { CAP_PROP_FORMAT, -1 }); + ASSERT_TRUE(capRaw.isOpened()); + const int width = static_cast(capRaw.get(CAP_PROP_FRAME_WIDTH)); + const int height = static_cast(capRaw.get(CAP_PROP_FRAME_HEIGHT)); + const double fps = capRaw.get(CAP_PROP_FPS); + const int codecExtradataIndex = static_cast(capRaw.get(CAP_PROP_CODEC_EXTRADATA_INDEX)); + Mat extraData; + capRaw.retrieve(extraData, codecExtradataIndex); + const int fourcc = static_cast(capRaw.get(CAP_PROP_FOURCC)); + const bool mpeg4 = (fourcc == fourccFromString("FMP4")); + + VideoWriter container(fileNameOut, api, fourcc, fps, { width, height }, { VideoWriterProperties::VIDEOWRITER_PROP_RAW_VIDEO, 1 }); + ASSERT_TRUE(container.isOpened()); + Mat rawFrame; + int i = 0; + while (capRaw.read(rawFrame)) { + ASSERT_FALSE(rawFrame.empty()); + if (i == 0 && mpeg4) { + Mat tmp = rawFrame.clone(); + const size_t newSzt = tmp.total() + extraData.total(); + const int newSz = static_cast(newSzt); + ASSERT_TRUE(newSzt == static_cast(newSz)); + rawFrame = Mat(1, newSz, CV_8UC1); + memcpy(rawFrame.data, extraData.data, extraData.total()); + memcpy(rawFrame.data + extraData.total(), tmp.data, tmp.total()); + } + if (static_cast(capRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME))) + container.set(VideoWriterProperties::VIDEOWRITER_PROP_KEY_FLAG, 1); + else + container.set(VideoWriterProperties::VIDEOWRITER_PROP_KEY_FLAG, 0); + container.write(rawFrame); + i++; + } + container.release(); + } + + std::cout << "Checking encapsulated video container: " << fileNameOut << std::endl; + + // Check encapsulated video container is "identical" to the original + { + VideoCapture capReference(fileName), capReferenceRaw(fileName, api, { CAP_PROP_FORMAT, -1 }), capActual(fileNameOut), capActualRaw(fileNameOut, api, { CAP_PROP_FORMAT, -1 }); + ASSERT_TRUE(capReference.isOpened()); + ASSERT_TRUE(capActual.isOpened()); + ASSERT_TRUE(capReferenceRaw.isOpened()); + ASSERT_TRUE(capActualRaw.isOpened()); + const double fpsReference = capReference.get(CAP_PROP_FPS); + const double fpsActual = capActual.get(CAP_PROP_FPS); + ASSERT_EQ(fpsReference, fpsActual); + const int nFramesReference = static_cast(capReference.get(CAP_PROP_FRAME_COUNT)); + const int nFramesActual = static_cast(capActual.get(CAP_PROP_FRAME_COUNT)); + ASSERT_EQ(nFramesReference, nFramesActual); + + Mat reference, actual; + for (int i = 0; i < nFramesReference; i++) { + ASSERT_TRUE(capReference.read(reference)); + ASSERT_FALSE(reference.empty()); + ASSERT_TRUE(capActual.read(actual)); + ASSERT_FALSE(actual.empty()); + ASSERT_EQ(0, cvtest::norm(reference, actual, NORM_INF)); + ASSERT_TRUE(capReferenceRaw.grab()); + ASSERT_TRUE(capActualRaw.grab()); + const bool keyFrameReference = capActualRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME) == 1.; + const bool keyFrameActual = capActualRaw.get(CAP_PROP_LRF_HAS_KEY_FRAME) == 1.; + ASSERT_EQ(keyFrameReference, keyFrameActual); + } + } + + ASSERT_EQ(0, remove(fileNameOut.c_str())); +#endif +} + typedef tuple videoio_skip_params_t; typedef testing::TestWithParam< videoio_skip_params_t > videoio_skip; diff --git a/samples/cpp/simd_basic.cpp b/samples/cpp/simd_basic.cpp index 9af4d91cef..ef78c39a45 100644 --- a/samples/cpp/simd_basic.cpp +++ b/samples/cpp/simd_basic.cpp @@ -38,8 +38,8 @@ int main(int /*argc*/, char** /*argv*/) printf("================== arithm check =================\n"); v_uint8 a = vx_setall_u8(10); - v_uint8 c = a + vx_setall_u8(45); - printf("(vx_setall_u8(10) + vx_setall_u8(45)).get0() => %d\n", (int)c.get0()); + v_uint8 c = v_add(a, vx_setall_u8(45)); + printf("v_get0(vx_setall_u8(10) + vx_setall_u8(45)) => %d\n", (int)v_get0(c)); #else printf("\nSIMD intrinsics are not available. Check compilation target and passed build options.\n"); #endif diff --git a/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp b/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp index 9be4170d7b..52018461c3 100644 --- a/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp +++ b/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp @@ -85,7 +85,7 @@ void conv1dsimd(Mat src, Mat kernel, float *ans, int row = 0, int rowk = 0, int //! [convolution-1D-main] //! [convolution-1D-main-h1] - int step = v_float32().nlanes; + int step = VTraits::vlanes(); float *sptr = src_32.ptr(row), *kptr = kernel.ptr(rowk); for (int k = 0; k < ksize; k++) { @@ -96,7 +96,7 @@ void conv1dsimd(Mat src, Mat kernel, float *ans, int row = 0, int rowk = 0, int for (i = 0; i + step < len; i += step) { v_float32 window = vx_load(sptr + i + k); - v_float32 sum = vx_load(ans + i) + kernel_wide * window; + v_float32 sum = v_add(vx_load(ans + i), v_mul(kernel_wide, window)); v_store(ans + i, sum); } //! [convolution-1D-main-h2] @@ -122,7 +122,7 @@ void convolute_simd(Mat src, Mat &dst, Mat kernel) copyMakeBorder(src, src, sz, sz, 0, 0, BORDER_REPLICATE); - int step = v_float32().nlanes; + int step = VTraits::vlanes(); //! [convolution-2D-init] //! [convolution-2D-main] @@ -135,7 +135,7 @@ void convolute_simd(Mat src, Mat &dst, Mat kernel) int j; for (j = 0; j + step < cols; j += step) { - v_float32 sum = vx_load(&dst.ptr(i)[j]) + vx_load(&ans[j]); + v_float32 sum = v_add(vx_load(&dst.ptr(i)[j]), vx_load(&ans[j])); v_store(&dst.ptr(i)[j], sum); }