diff --git a/3rdparty/libwebp/CMakeLists.txt b/3rdparty/libwebp/CMakeLists.txt index 723575c8db..f3b6ebd0d6 100644 --- a/3rdparty/libwebp/CMakeLists.txt +++ b/3rdparty/libwebp/CMakeLists.txt @@ -9,8 +9,8 @@ if(ANDROID) ocv_include_directories(${CPUFEATURES_INCLUDE_DIRS}) endif() -file(GLOB lib_srcs src/dec/*.c src/demux/*.c src/dsp/*.c src/enc/*.c src/mux/*.c src/utils/*.c src/webp/*.c) -file(GLOB lib_hdrs src/dec/*.h src/demux/*.h src/dsp/*.h src/enc/*.h src/mux/*.h src/utils/*.h src/webp/*.h) +file(GLOB lib_srcs sharpyuv/*.c src/dec/*.c src/demux/*.c src/dsp/*.c src/enc/*.c src/mux/*.c src/utils/*.c src/webp/*.c) +file(GLOB lib_hdrs sharpyuv/*.h src/dec/*.h src/demux/*.h src/dsp/*.h src/enc/*.h src/mux/*.h src/utils/*.h src/webp/*.h) # FIXIT if(ANDROID AND ARMEABI_V7A AND NOT NEON) @@ -21,12 +21,6 @@ if(ANDROID AND ARMEABI_V7A AND NOT NEON) endforeach() endif() -# FIX for quant.h - requires C99 for() loops -ocv_check_flag_support(C "-std=c99" _varname "${CMAKE_C_FLAGS}") -if(${_varname}) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") -endif() - # ---------------------------------------------------------------------------------- # Define the library target: diff --git a/3rdparty/libwebp/patches/20190910-msa-asm-patch.diff b/3rdparty/libwebp/patches/20190910-msa-asm-patch.diff deleted file mode 100644 index 1be2135203..0000000000 --- a/3rdparty/libwebp/patches/20190910-msa-asm-patch.diff +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/3rdparty/libwebp/src/dsp/msa_macro.h b/3rdparty/libwebp/src/dsp/msa_macro.h -index de026a1d9e..a16c0bb300 100644 ---- a/3rdparty/libwebp/src/dsp/msa_macro.h -+++ b/3rdparty/libwebp/src/dsp/msa_macro.h -@@ -73,7 +73,7 @@ - static inline TYPE FUNC_NAME(const void* const psrc) { \ - const uint8_t* const psrc_m = (const uint8_t*)psrc; \ - TYPE val_m; \ -- asm volatile ( \ -+ __asm__ volatile ( \ - "" #INSTR " %[val_m], %[psrc_m] \n\t" \ - : [val_m] "=r" (val_m) \ - : [psrc_m] "m" (*psrc_m)); \ -@@ -86,7 +86,7 @@ - static inline void FUNC_NAME(TYPE val, void* const pdst) { \ - uint8_t* const pdst_m = (uint8_t*)pdst; \ - TYPE val_m = val; \ -- asm volatile ( \ -+ __asm__ volatile ( \ - " " #INSTR " %[val_m], %[pdst_m] \n\t" \ - : [pdst_m] "=m" (*pdst_m) \ - : [val_m] "r" (val_m)); \ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv.c b/3rdparty/libwebp/sharpyuv/sharpyuv.c new file mode 100644 index 0000000000..b94885a6c3 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv.c @@ -0,0 +1,565 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Sharp RGB to YUV conversion. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "sharpyuv/sharpyuv.h" + +#include +#include +#include +#include +#include + +#include "src/webp/types.h" +#include "sharpyuv/sharpyuv_cpu.h" +#include "sharpyuv/sharpyuv_dsp.h" +#include "sharpyuv/sharpyuv_gamma.h" + +//------------------------------------------------------------------------------ + +int SharpYuvGetVersion(void) { + return SHARPYUV_VERSION; +} + +//------------------------------------------------------------------------------ +// Sharp RGB->YUV conversion + +static const int kNumIterations = 4; + +#define YUV_FIX 16 // fixed-point precision for RGB->YUV +static const int kYuvHalf = 1 << (YUV_FIX - 1); + +// Max bit depth so that intermediate calculations fit in 16 bits. +static const int kMaxBitDepth = 14; + +// Returns the precision shift to use based on the input rgb_bit_depth. +static int GetPrecisionShift(int rgb_bit_depth) { + // Try to add 2 bits of precision if it fits in kMaxBitDepth. Otherwise remove + // bits if needed. + return ((rgb_bit_depth + 2) <= kMaxBitDepth) ? 2 + : (kMaxBitDepth - rgb_bit_depth); +} + +typedef int16_t fixed_t; // signed type with extra precision for UV +typedef uint16_t fixed_y_t; // unsigned type with extra precision for W + +//------------------------------------------------------------------------------ + +static uint8_t clip_8b(fixed_t v) { + return (!(v & ~0xff)) ? (uint8_t)v : (v < 0) ? 0u : 255u; +} + +static uint16_t clip(fixed_t v, int max) { + return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; +} + +static fixed_y_t clip_bit_depth(int y, int bit_depth) { + const int max = (1 << bit_depth) - 1; + return (!(y & ~max)) ? (fixed_y_t)y : (y < 0) ? 0 : max; +} + +//------------------------------------------------------------------------------ + +static int RGBToGray(int64_t r, int64_t g, int64_t b) { + const int64_t luma = 13933 * r + 46871 * g + 4732 * b + kYuvHalf; + return (int)(luma >> YUV_FIX); +} + +static uint32_t ScaleDown(uint16_t a, uint16_t b, uint16_t c, uint16_t d, + int rgb_bit_depth, + SharpYuvTransferFunctionType transfer_type) { + const int bit_depth = rgb_bit_depth + GetPrecisionShift(rgb_bit_depth); + const uint32_t A = SharpYuvGammaToLinear(a, bit_depth, transfer_type); + const uint32_t B = SharpYuvGammaToLinear(b, bit_depth, transfer_type); + const uint32_t C = SharpYuvGammaToLinear(c, bit_depth, transfer_type); + const uint32_t D = SharpYuvGammaToLinear(d, bit_depth, transfer_type); + return SharpYuvLinearToGamma((A + B + C + D + 2) >> 2, bit_depth, + transfer_type); +} + +static WEBP_INLINE void UpdateW(const fixed_y_t* src, fixed_y_t* dst, int w, + int rgb_bit_depth, + SharpYuvTransferFunctionType transfer_type) { + const int bit_depth = rgb_bit_depth + GetPrecisionShift(rgb_bit_depth); + int i; + for (i = 0; i < w; ++i) { + const uint32_t R = + SharpYuvGammaToLinear(src[0 * w + i], bit_depth, transfer_type); + const uint32_t G = + SharpYuvGammaToLinear(src[1 * w + i], bit_depth, transfer_type); + const uint32_t B = + SharpYuvGammaToLinear(src[2 * w + i], bit_depth, transfer_type); + const uint32_t Y = RGBToGray(R, G, B); + dst[i] = (fixed_y_t)SharpYuvLinearToGamma(Y, bit_depth, transfer_type); + } +} + +static void UpdateChroma(const fixed_y_t* src1, const fixed_y_t* src2, + fixed_t* dst, int uv_w, int rgb_bit_depth, + SharpYuvTransferFunctionType transfer_type) { + int i; + for (i = 0; i < uv_w; ++i) { + const int r = + ScaleDown(src1[0 * uv_w + 0], src1[0 * uv_w + 1], src2[0 * uv_w + 0], + src2[0 * uv_w + 1], rgb_bit_depth, transfer_type); + const int g = + ScaleDown(src1[2 * uv_w + 0], src1[2 * uv_w + 1], src2[2 * uv_w + 0], + src2[2 * uv_w + 1], rgb_bit_depth, transfer_type); + const int b = + ScaleDown(src1[4 * uv_w + 0], src1[4 * uv_w + 1], src2[4 * uv_w + 0], + src2[4 * uv_w + 1], rgb_bit_depth, transfer_type); + const int W = RGBToGray(r, g, b); + dst[0 * uv_w] = (fixed_t)(r - W); + dst[1 * uv_w] = (fixed_t)(g - W); + dst[2 * uv_w] = (fixed_t)(b - W); + dst += 1; + src1 += 2; + src2 += 2; + } +} + +static void StoreGray(const fixed_y_t* rgb, fixed_y_t* y, int w) { + int i; + assert(w > 0); + for (i = 0; i < w; ++i) { + y[i] = RGBToGray(rgb[0 * w + i], rgb[1 * w + i], rgb[2 * w + i]); + } +} + +//------------------------------------------------------------------------------ + +static WEBP_INLINE fixed_y_t Filter2(int A, int B, int W0, int bit_depth) { + const int v0 = (A * 3 + B + 2) >> 2; + return clip_bit_depth(v0 + W0, bit_depth); +} + +//------------------------------------------------------------------------------ + +static WEBP_INLINE int Shift(int v, int shift) { + return (shift >= 0) ? (v << shift) : (v >> -shift); +} + +static void ImportOneRow(const uint8_t* const r_ptr, + const uint8_t* const g_ptr, + const uint8_t* const b_ptr, + int rgb_step, + int rgb_bit_depth, + int pic_width, + fixed_y_t* const dst) { + // Convert the rgb_step from a number of bytes to a number of uint8_t or + // uint16_t values depending the bit depth. + const int step = (rgb_bit_depth > 8) ? rgb_step / 2 : rgb_step; + int i; + const int w = (pic_width + 1) & ~1; + for (i = 0; i < pic_width; ++i) { + const int off = i * step; + const int shift = GetPrecisionShift(rgb_bit_depth); + if (rgb_bit_depth == 8) { + dst[i + 0 * w] = Shift(r_ptr[off], shift); + dst[i + 1 * w] = Shift(g_ptr[off], shift); + dst[i + 2 * w] = Shift(b_ptr[off], shift); + } else { + dst[i + 0 * w] = Shift(((uint16_t*)r_ptr)[off], shift); + dst[i + 1 * w] = Shift(((uint16_t*)g_ptr)[off], shift); + dst[i + 2 * w] = Shift(((uint16_t*)b_ptr)[off], shift); + } + } + if (pic_width & 1) { // replicate rightmost pixel + dst[pic_width + 0 * w] = dst[pic_width + 0 * w - 1]; + dst[pic_width + 1 * w] = dst[pic_width + 1 * w - 1]; + dst[pic_width + 2 * w] = dst[pic_width + 2 * w - 1]; + } +} + +static void InterpolateTwoRows(const fixed_y_t* const best_y, + const fixed_t* prev_uv, + const fixed_t* cur_uv, + const fixed_t* next_uv, + int w, + fixed_y_t* out1, + fixed_y_t* out2, + int rgb_bit_depth) { + const int uv_w = w >> 1; + const int len = (w - 1) >> 1; // length to filter + int k = 3; + const int bit_depth = rgb_bit_depth + GetPrecisionShift(rgb_bit_depth); + while (k-- > 0) { // process each R/G/B segments in turn + // special boundary case for i==0 + out1[0] = Filter2(cur_uv[0], prev_uv[0], best_y[0], bit_depth); + out2[0] = Filter2(cur_uv[0], next_uv[0], best_y[w], bit_depth); + + SharpYuvFilterRow(cur_uv, prev_uv, len, best_y + 0 + 1, out1 + 1, + bit_depth); + SharpYuvFilterRow(cur_uv, next_uv, len, best_y + w + 1, out2 + 1, + bit_depth); + + // special boundary case for i == w - 1 when w is even + if (!(w & 1)) { + out1[w - 1] = Filter2(cur_uv[uv_w - 1], prev_uv[uv_w - 1], + best_y[w - 1 + 0], bit_depth); + out2[w - 1] = Filter2(cur_uv[uv_w - 1], next_uv[uv_w - 1], + best_y[w - 1 + w], bit_depth); + } + out1 += w; + out2 += w; + prev_uv += uv_w; + cur_uv += uv_w; + next_uv += uv_w; + } +} + +static WEBP_INLINE int RGBToYUVComponent(int r, int g, int b, + const int coeffs[4], int sfix) { + const int srounder = 1 << (YUV_FIX + sfix - 1); + const int luma = coeffs[0] * r + coeffs[1] * g + coeffs[2] * b + + coeffs[3] + srounder; + return (luma >> (YUV_FIX + sfix)); +} + +static int ConvertWRGBToYUV(const fixed_y_t* best_y, const fixed_t* best_uv, + uint8_t* y_ptr, int y_stride, uint8_t* u_ptr, + int u_stride, uint8_t* v_ptr, int v_stride, + int rgb_bit_depth, + int yuv_bit_depth, int width, int height, + const SharpYuvConversionMatrix* yuv_matrix) { + int i, j; + const fixed_t* const best_uv_base = best_uv; + const int w = (width + 1) & ~1; + const int h = (height + 1) & ~1; + const int uv_w = w >> 1; + const int uv_h = h >> 1; + const int sfix = GetPrecisionShift(rgb_bit_depth); + const int yuv_max = (1 << yuv_bit_depth) - 1; + + for (best_uv = best_uv_base, j = 0; j < height; ++j) { + for (i = 0; i < width; ++i) { + const int off = (i >> 1); + const int W = best_y[i]; + const int r = best_uv[off + 0 * uv_w] + W; + const int g = best_uv[off + 1 * uv_w] + W; + const int b = best_uv[off + 2 * uv_w] + W; + const int y = RGBToYUVComponent(r, g, b, yuv_matrix->rgb_to_y, sfix); + if (yuv_bit_depth <= 8) { + y_ptr[i] = clip_8b(y); + } else { + ((uint16_t*)y_ptr)[i] = clip(y, yuv_max); + } + } + best_y += w; + best_uv += (j & 1) * 3 * uv_w; + y_ptr += y_stride; + } + for (best_uv = best_uv_base, j = 0; j < uv_h; ++j) { + for (i = 0; i < uv_w; ++i) { + const int off = i; + // Note r, g and b values here are off by W, but a constant offset on all + // 3 components doesn't change the value of u and v with a YCbCr matrix. + const int r = best_uv[off + 0 * uv_w]; + const int g = best_uv[off + 1 * uv_w]; + const int b = best_uv[off + 2 * uv_w]; + const int u = RGBToYUVComponent(r, g, b, yuv_matrix->rgb_to_u, sfix); + const int v = RGBToYUVComponent(r, g, b, yuv_matrix->rgb_to_v, sfix); + if (yuv_bit_depth <= 8) { + u_ptr[i] = clip_8b(u); + v_ptr[i] = clip_8b(v); + } else { + ((uint16_t*)u_ptr)[i] = clip(u, yuv_max); + ((uint16_t*)v_ptr)[i] = clip(v, yuv_max); + } + } + best_uv += 3 * uv_w; + u_ptr += u_stride; + v_ptr += v_stride; + } + return 1; +} + +//------------------------------------------------------------------------------ +// Main function + +static void* SafeMalloc(uint64_t nmemb, size_t size) { + const uint64_t total_size = nmemb * (uint64_t)size; + if (total_size != (size_t)total_size) return NULL; + return malloc((size_t)total_size); +} + +#define SAFE_ALLOC(W, H, T) ((T*)SafeMalloc((W) * (H), sizeof(T))) + +static int DoSharpArgbToYuv(const uint8_t* r_ptr, const uint8_t* g_ptr, + const uint8_t* b_ptr, int rgb_step, int rgb_stride, + int rgb_bit_depth, uint8_t* y_ptr, int y_stride, + uint8_t* u_ptr, int u_stride, uint8_t* v_ptr, + int v_stride, int yuv_bit_depth, int width, + int height, + const SharpYuvConversionMatrix* yuv_matrix, + SharpYuvTransferFunctionType transfer_type) { + // we expand the right/bottom border if needed + const int w = (width + 1) & ~1; + const int h = (height + 1) & ~1; + const int uv_w = w >> 1; + const int uv_h = h >> 1; + uint64_t prev_diff_y_sum = ~0; + int j, iter; + + // TODO(skal): allocate one big memory chunk. But for now, it's easier + // for valgrind debugging to have several chunks. + fixed_y_t* const tmp_buffer = SAFE_ALLOC(w * 3, 2, fixed_y_t); // scratch + fixed_y_t* const best_y_base = SAFE_ALLOC(w, h, fixed_y_t); + fixed_y_t* const target_y_base = SAFE_ALLOC(w, h, fixed_y_t); + fixed_y_t* const best_rgb_y = SAFE_ALLOC(w, 2, fixed_y_t); + fixed_t* const best_uv_base = SAFE_ALLOC(uv_w * 3, uv_h, fixed_t); + fixed_t* const target_uv_base = SAFE_ALLOC(uv_w * 3, uv_h, fixed_t); + fixed_t* const best_rgb_uv = SAFE_ALLOC(uv_w * 3, 1, fixed_t); + fixed_y_t* best_y = best_y_base; + fixed_y_t* target_y = target_y_base; + fixed_t* best_uv = best_uv_base; + fixed_t* target_uv = target_uv_base; + const uint64_t diff_y_threshold = (uint64_t)(3.0 * w * h); + int ok; + assert(w > 0); + assert(h > 0); + + if (best_y_base == NULL || best_uv_base == NULL || + target_y_base == NULL || target_uv_base == NULL || + best_rgb_y == NULL || best_rgb_uv == NULL || + tmp_buffer == NULL) { + ok = 0; + goto End; + } + + // Import RGB samples to W/RGB representation. + for (j = 0; j < height; j += 2) { + const int is_last_row = (j == height - 1); + fixed_y_t* const src1 = tmp_buffer + 0 * w; + fixed_y_t* const src2 = tmp_buffer + 3 * w; + + // prepare two rows of input + ImportOneRow(r_ptr, g_ptr, b_ptr, rgb_step, rgb_bit_depth, width, + src1); + if (!is_last_row) { + ImportOneRow(r_ptr + rgb_stride, g_ptr + rgb_stride, b_ptr + rgb_stride, + rgb_step, rgb_bit_depth, width, src2); + } else { + memcpy(src2, src1, 3 * w * sizeof(*src2)); + } + StoreGray(src1, best_y + 0, w); + StoreGray(src2, best_y + w, w); + + UpdateW(src1, target_y, w, rgb_bit_depth, transfer_type); + UpdateW(src2, target_y + w, w, rgb_bit_depth, transfer_type); + UpdateChroma(src1, src2, target_uv, uv_w, rgb_bit_depth, transfer_type); + memcpy(best_uv, target_uv, 3 * uv_w * sizeof(*best_uv)); + best_y += 2 * w; + best_uv += 3 * uv_w; + target_y += 2 * w; + target_uv += 3 * uv_w; + r_ptr += 2 * rgb_stride; + g_ptr += 2 * rgb_stride; + b_ptr += 2 * rgb_stride; + } + + // Iterate and resolve clipping conflicts. + for (iter = 0; iter < kNumIterations; ++iter) { + const fixed_t* cur_uv = best_uv_base; + const fixed_t* prev_uv = best_uv_base; + uint64_t diff_y_sum = 0; + + best_y = best_y_base; + best_uv = best_uv_base; + target_y = target_y_base; + target_uv = target_uv_base; + for (j = 0; j < h; j += 2) { + fixed_y_t* const src1 = tmp_buffer + 0 * w; + fixed_y_t* const src2 = tmp_buffer + 3 * w; + { + const fixed_t* const next_uv = cur_uv + ((j < h - 2) ? 3 * uv_w : 0); + InterpolateTwoRows(best_y, prev_uv, cur_uv, next_uv, w, + src1, src2, rgb_bit_depth); + prev_uv = cur_uv; + cur_uv = next_uv; + } + + UpdateW(src1, best_rgb_y + 0 * w, w, rgb_bit_depth, transfer_type); + UpdateW(src2, best_rgb_y + 1 * w, w, rgb_bit_depth, transfer_type); + UpdateChroma(src1, src2, best_rgb_uv, uv_w, rgb_bit_depth, transfer_type); + + // update two rows of Y and one row of RGB + diff_y_sum += + SharpYuvUpdateY(target_y, best_rgb_y, best_y, 2 * w, + rgb_bit_depth + GetPrecisionShift(rgb_bit_depth)); + SharpYuvUpdateRGB(target_uv, best_rgb_uv, best_uv, 3 * uv_w); + + best_y += 2 * w; + best_uv += 3 * uv_w; + target_y += 2 * w; + target_uv += 3 * uv_w; + } + // test exit condition + if (iter > 0) { + if (diff_y_sum < diff_y_threshold) break; + if (diff_y_sum > prev_diff_y_sum) break; + } + prev_diff_y_sum = diff_y_sum; + } + + // final reconstruction + ok = ConvertWRGBToYUV(best_y_base, best_uv_base, y_ptr, y_stride, u_ptr, + u_stride, v_ptr, v_stride, rgb_bit_depth, yuv_bit_depth, + width, height, yuv_matrix); + + End: + free(best_y_base); + free(best_uv_base); + free(target_y_base); + free(target_uv_base); + free(best_rgb_y); + free(best_rgb_uv); + free(tmp_buffer); + return ok; +} +#undef SAFE_ALLOC + +#if defined(WEBP_USE_THREAD) && !defined(_WIN32) +#include // NOLINT + +#define LOCK_ACCESS \ + static pthread_mutex_t sharpyuv_lock = PTHREAD_MUTEX_INITIALIZER; \ + if (pthread_mutex_lock(&sharpyuv_lock)) return +#define UNLOCK_ACCESS_AND_RETURN \ + do { \ + (void)pthread_mutex_unlock(&sharpyuv_lock); \ + return; \ + } while (0) +#else // !(defined(WEBP_USE_THREAD) && !defined(_WIN32)) +#define LOCK_ACCESS do {} while (0) +#define UNLOCK_ACCESS_AND_RETURN return +#endif // defined(WEBP_USE_THREAD) && !defined(_WIN32) + +// Hidden exported init function. +// By default SharpYuvConvert calls it with SharpYuvGetCPUInfo. If needed, +// users can declare it as extern and call it with an alternate VP8CPUInfo +// function. +extern VP8CPUInfo SharpYuvGetCPUInfo; +SHARPYUV_EXTERN void SharpYuvInit(VP8CPUInfo cpu_info_func); +void SharpYuvInit(VP8CPUInfo cpu_info_func) { + static volatile VP8CPUInfo sharpyuv_last_cpuinfo_used = + (VP8CPUInfo)&sharpyuv_last_cpuinfo_used; + LOCK_ACCESS; + // Only update SharpYuvGetCPUInfo when called from external code to avoid a + // race on reading the value in SharpYuvConvert(). + if (cpu_info_func != (VP8CPUInfo)&SharpYuvGetCPUInfo) { + SharpYuvGetCPUInfo = cpu_info_func; + } + if (sharpyuv_last_cpuinfo_used == SharpYuvGetCPUInfo) { + UNLOCK_ACCESS_AND_RETURN; + } + + SharpYuvInitDsp(); + SharpYuvInitGammaTables(); + + sharpyuv_last_cpuinfo_used = SharpYuvGetCPUInfo; + UNLOCK_ACCESS_AND_RETURN; +} + +int SharpYuvConvert(const void* r_ptr, const void* g_ptr, const void* b_ptr, + int rgb_step, int rgb_stride, int rgb_bit_depth, + void* y_ptr, int y_stride, void* u_ptr, int u_stride, + void* v_ptr, int v_stride, int yuv_bit_depth, int width, + int height, const SharpYuvConversionMatrix* yuv_matrix) { + SharpYuvOptions options; + options.yuv_matrix = yuv_matrix; + options.transfer_type = kSharpYuvTransferFunctionSrgb; + return SharpYuvConvertWithOptions( + r_ptr, g_ptr, b_ptr, rgb_step, rgb_stride, rgb_bit_depth, y_ptr, y_stride, + u_ptr, u_stride, v_ptr, v_stride, yuv_bit_depth, width, height, &options); +} + +int SharpYuvOptionsInitInternal(const SharpYuvConversionMatrix* yuv_matrix, + SharpYuvOptions* options, int version) { + const int major = (version >> 24); + const int minor = (version >> 16) & 0xff; + if (options == NULL || yuv_matrix == NULL || + (major == SHARPYUV_VERSION_MAJOR && major == 0 && + minor != SHARPYUV_VERSION_MINOR) || + (major != SHARPYUV_VERSION_MAJOR)) { + return 0; + } + options->yuv_matrix = yuv_matrix; + options->transfer_type = kSharpYuvTransferFunctionSrgb; + return 1; +} + +int SharpYuvConvertWithOptions(const void* r_ptr, const void* g_ptr, + const void* b_ptr, int rgb_step, int rgb_stride, + int rgb_bit_depth, void* y_ptr, int y_stride, + void* u_ptr, int u_stride, void* v_ptr, + int v_stride, int yuv_bit_depth, int width, + int height, const SharpYuvOptions* options) { + const SharpYuvConversionMatrix* yuv_matrix = options->yuv_matrix; + SharpYuvTransferFunctionType transfer_type = options->transfer_type; + SharpYuvConversionMatrix scaled_matrix; + const int rgb_max = (1 << rgb_bit_depth) - 1; + const int rgb_round = 1 << (rgb_bit_depth - 1); + const int yuv_max = (1 << yuv_bit_depth) - 1; + const int sfix = GetPrecisionShift(rgb_bit_depth); + + if (width < 1 || height < 1 || width == INT_MAX || height == INT_MAX || + r_ptr == NULL || g_ptr == NULL || b_ptr == NULL || y_ptr == NULL || + u_ptr == NULL || v_ptr == NULL) { + return 0; + } + if (rgb_bit_depth != 8 && rgb_bit_depth != 10 && rgb_bit_depth != 12 && + rgb_bit_depth != 16) { + return 0; + } + if (yuv_bit_depth != 8 && yuv_bit_depth != 10 && yuv_bit_depth != 12) { + return 0; + } + if (rgb_bit_depth > 8 && (rgb_step % 2 != 0 || rgb_stride %2 != 0)) { + // Step/stride should be even for uint16_t buffers. + return 0; + } + if (yuv_bit_depth > 8 && + (y_stride % 2 != 0 || u_stride % 2 != 0 || v_stride % 2 != 0)) { + // Stride should be even for uint16_t buffers. + return 0; + } + // The address of the function pointer is used to avoid a read race. + SharpYuvInit((VP8CPUInfo)&SharpYuvGetCPUInfo); + + // Add scaling factor to go from rgb_bit_depth to yuv_bit_depth, to the + // rgb->yuv conversion matrix. + if (rgb_bit_depth == yuv_bit_depth) { + memcpy(&scaled_matrix, yuv_matrix, sizeof(scaled_matrix)); + } else { + int i; + for (i = 0; i < 3; ++i) { + scaled_matrix.rgb_to_y[i] = + (yuv_matrix->rgb_to_y[i] * yuv_max + rgb_round) / rgb_max; + scaled_matrix.rgb_to_u[i] = + (yuv_matrix->rgb_to_u[i] * yuv_max + rgb_round) / rgb_max; + scaled_matrix.rgb_to_v[i] = + (yuv_matrix->rgb_to_v[i] * yuv_max + rgb_round) / rgb_max; + } + } + // Also incorporate precision change scaling. + scaled_matrix.rgb_to_y[3] = Shift(yuv_matrix->rgb_to_y[3], sfix); + scaled_matrix.rgb_to_u[3] = Shift(yuv_matrix->rgb_to_u[3], sfix); + scaled_matrix.rgb_to_v[3] = Shift(yuv_matrix->rgb_to_v[3], sfix); + + return DoSharpArgbToYuv(r_ptr, g_ptr, b_ptr, rgb_step, rgb_stride, + rgb_bit_depth, y_ptr, y_stride, u_ptr, u_stride, + v_ptr, v_stride, yuv_bit_depth, width, height, + &scaled_matrix, transfer_type); +} + +//------------------------------------------------------------------------------ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv.h b/3rdparty/libwebp/sharpyuv/sharpyuv.h new file mode 100644 index 0000000000..23a69ce39c --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv.h @@ -0,0 +1,174 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Sharp RGB to YUV conversion. + +#ifndef WEBP_SHARPYUV_SHARPYUV_H_ +#define WEBP_SHARPYUV_SHARPYUV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef SHARPYUV_EXTERN +#ifdef WEBP_EXTERN +#define SHARPYUV_EXTERN WEBP_EXTERN +#else +// This explicitly marks library functions and allows for changing the +// signature for e.g., Windows DLL builds. +#if defined(__GNUC__) && __GNUC__ >= 4 +#define SHARPYUV_EXTERN extern __attribute__((visibility("default"))) +#else +#if defined(_MSC_VER) && defined(WEBP_DLL) +#define SHARPYUV_EXTERN __declspec(dllexport) +#else +#define SHARPYUV_EXTERN extern +#endif /* _MSC_VER && WEBP_DLL */ +#endif /* __GNUC__ >= 4 */ +#endif /* WEBP_EXTERN */ +#endif /* SHARPYUV_EXTERN */ + +#ifndef SHARPYUV_INLINE +#ifdef WEBP_INLINE +#define SHARPYUV_INLINE WEBP_INLINE +#else +#ifndef _MSC_VER +#if defined(__cplusplus) || !defined(__STRICT_ANSI__) || \ + (defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) +#define SHARPYUV_INLINE inline +#else +#define SHARPYUV_INLINE +#endif +#else +#define SHARPYUV_INLINE __forceinline +#endif /* _MSC_VER */ +#endif /* WEBP_INLINE */ +#endif /* SHARPYUV_INLINE */ + +// SharpYUV API version following the convention from semver.org +#define SHARPYUV_VERSION_MAJOR 0 +#define SHARPYUV_VERSION_MINOR 4 +#define SHARPYUV_VERSION_PATCH 0 +// Version as a uint32_t. The major number is the high 8 bits. +// The minor number is the middle 8 bits. The patch number is the low 16 bits. +#define SHARPYUV_MAKE_VERSION(MAJOR, MINOR, PATCH) \ + (((MAJOR) << 24) | ((MINOR) << 16) | (PATCH)) +#define SHARPYUV_VERSION \ + SHARPYUV_MAKE_VERSION(SHARPYUV_VERSION_MAJOR, SHARPYUV_VERSION_MINOR, \ + SHARPYUV_VERSION_PATCH) + +// Returns the library's version number, packed in hexadecimal. See +// SHARPYUV_VERSION. +SHARPYUV_EXTERN int SharpYuvGetVersion(void); + +// RGB to YUV conversion matrix, in 16 bit fixed point. +// y = rgb_to_y[0] * r + rgb_to_y[1] * g + rgb_to_y[2] * b + rgb_to_y[3] +// u = rgb_to_u[0] * r + rgb_to_u[1] * g + rgb_to_u[2] * b + rgb_to_u[3] +// v = rgb_to_v[0] * r + rgb_to_v[1] * g + rgb_to_v[2] * b + rgb_to_v[3] +// Then y, u and v values are divided by 1<<16 and rounded. +typedef struct { + int rgb_to_y[4]; + int rgb_to_u[4]; + int rgb_to_v[4]; +} SharpYuvConversionMatrix; + +typedef struct SharpYuvOptions SharpYuvOptions; + +// Enums for transfer functions, as defined in H.273, +// https://www.itu.int/rec/T-REC-H.273-202107-I/en +typedef enum SharpYuvTransferFunctionType { + // 0 is reserved + kSharpYuvTransferFunctionBt709 = 1, + // 2 is unspecified + // 3 is reserved + kSharpYuvTransferFunctionBt470M = 4, + kSharpYuvTransferFunctionBt470Bg = 5, + kSharpYuvTransferFunctionBt601 = 6, + kSharpYuvTransferFunctionSmpte240 = 7, + kSharpYuvTransferFunctionLinear = 8, + kSharpYuvTransferFunctionLog100 = 9, + kSharpYuvTransferFunctionLog100_Sqrt10 = 10, + kSharpYuvTransferFunctionIec61966 = 11, + kSharpYuvTransferFunctionBt1361 = 12, + kSharpYuvTransferFunctionSrgb = 13, + kSharpYuvTransferFunctionBt2020_10Bit = 14, + kSharpYuvTransferFunctionBt2020_12Bit = 15, + kSharpYuvTransferFunctionSmpte2084 = 16, // PQ + kSharpYuvTransferFunctionSmpte428 = 17, + kSharpYuvTransferFunctionHlg = 18, + kSharpYuvTransferFunctionNum +} SharpYuvTransferFunctionType; + +// Converts RGB to YUV420 using a downsampling algorithm that minimizes +// artefacts caused by chroma subsampling. +// This is slower than standard downsampling (averaging of 4 UV values). +// Assumes that the image will be upsampled using a bilinear filter. If nearest +// neighbor is used instead, the upsampled image might look worse than with +// standard downsampling. +// r_ptr, g_ptr, b_ptr: pointers to the source r, g and b channels. Should point +// to uint8_t buffers if rgb_bit_depth is 8, or uint16_t buffers otherwise. +// rgb_step: distance in bytes between two horizontally adjacent pixels on the +// r, g and b channels. If rgb_bit_depth is > 8, it should be a +// multiple of 2. +// rgb_stride: distance in bytes between two vertically adjacent pixels on the +// r, g, and b channels. If rgb_bit_depth is > 8, it should be a +// multiple of 2. +// rgb_bit_depth: number of bits for each r/g/b value. One of: 8, 10, 12, 16. +// Note: 16 bit input is truncated to 14 bits before conversion to yuv. +// yuv_bit_depth: number of bits for each y/u/v value. One of: 8, 10, 12. +// y_ptr, u_ptr, v_ptr: pointers to the destination y, u and v channels. Should +// point to uint8_t buffers if yuv_bit_depth is 8, or uint16_t buffers +// otherwise. +// y_stride, u_stride, v_stride: distance in bytes between two vertically +// adjacent pixels on the y, u and v channels. If yuv_bit_depth > 8, they +// should be multiples of 2. +// width, height: width and height of the image in pixels +// This function calls SharpYuvConvertWithOptions with a default transfer +// function of kSharpYuvTransferFunctionSrgb. +SHARPYUV_EXTERN int SharpYuvConvert(const void* r_ptr, const void* g_ptr, + const void* b_ptr, int rgb_step, + int rgb_stride, int rgb_bit_depth, + void* y_ptr, int y_stride, void* u_ptr, + int u_stride, void* v_ptr, int v_stride, + int yuv_bit_depth, int width, int height, + const SharpYuvConversionMatrix* yuv_matrix); + +struct SharpYuvOptions { + // This matrix cannot be NULL and can be initialized by + // SharpYuvComputeConversionMatrix. + const SharpYuvConversionMatrix* yuv_matrix; + SharpYuvTransferFunctionType transfer_type; +}; + +// Internal, version-checked, entry point +SHARPYUV_EXTERN int SharpYuvOptionsInitInternal(const SharpYuvConversionMatrix*, + SharpYuvOptions*, int); + +// Should always be called, to initialize a fresh SharpYuvOptions +// structure before modification. SharpYuvOptionsInit() must have succeeded +// before using the 'options' object. +static SHARPYUV_INLINE int SharpYuvOptionsInit( + const SharpYuvConversionMatrix* yuv_matrix, SharpYuvOptions* options) { + return SharpYuvOptionsInitInternal(yuv_matrix, options, SHARPYUV_VERSION); +} + +SHARPYUV_EXTERN int SharpYuvConvertWithOptions( + const void* r_ptr, const void* g_ptr, const void* b_ptr, int rgb_step, + int rgb_stride, int rgb_bit_depth, void* y_ptr, int y_stride, void* u_ptr, + int u_stride, void* v_ptr, int v_stride, int yuv_bit_depth, int width, + int height, const SharpYuvOptions* options); + +// TODO(b/194336375): Add YUV444 to YUV420 conversion. Maybe also add 422 +// support (it's rarely used in practice, especially for images). + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_SHARPYUV_SHARPYUV_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.c b/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.c new file mode 100644 index 0000000000..29425a0c49 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.c @@ -0,0 +1,14 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +#include "sharpyuv/sharpyuv_cpu.h" + +// Include src/dsp/cpu.c to create SharpYuvGetCPUInfo from VP8GetCPUInfo. The +// function pointer is renamed in sharpyuv_cpu.h. +#include "src/dsp/cpu.c" diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.h b/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.h new file mode 100644 index 0000000000..176ca3eb16 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_cpu.h @@ -0,0 +1,22 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +#ifndef WEBP_SHARPYUV_SHARPYUV_CPU_H_ +#define WEBP_SHARPYUV_SHARPYUV_CPU_H_ + +#include "sharpyuv/sharpyuv.h" + +// Avoid exporting SharpYuvGetCPUInfo in shared object / DLL builds. +// SharpYuvInit() replaces the use of the function pointer. +#undef WEBP_EXTERN +#define WEBP_EXTERN extern +#define VP8GetCPUInfo SharpYuvGetCPUInfo +#include "src/dsp/cpu.h" + +#endif // WEBP_SHARPYUV_SHARPYUV_CPU_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_csp.c b/3rdparty/libwebp/sharpyuv/sharpyuv_csp.c new file mode 100644 index 0000000000..0ad22be945 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_csp.c @@ -0,0 +1,110 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Colorspace utilities. + +#include "sharpyuv/sharpyuv_csp.h" + +#include +#include +#include + +static int ToFixed16(float f) { return (int)floor(f * (1 << 16) + 0.5f); } + +void SharpYuvComputeConversionMatrix(const SharpYuvColorSpace* yuv_color_space, + SharpYuvConversionMatrix* matrix) { + const float kr = yuv_color_space->kr; + const float kb = yuv_color_space->kb; + const float kg = 1.0f - kr - kb; + const float cr = 0.5f / (1.0f - kb); + const float cb = 0.5f / (1.0f - kr); + + const int shift = yuv_color_space->bit_depth - 8; + + const float denom = (float)((1 << yuv_color_space->bit_depth) - 1); + float scale_y = 1.0f; + float add_y = 0.0f; + float scale_u = cr; + float scale_v = cb; + float add_uv = (float)(128 << shift); + assert(yuv_color_space->bit_depth >= 8); + + if (yuv_color_space->range == kSharpYuvRangeLimited) { + scale_y *= (219 << shift) / denom; + scale_u *= (224 << shift) / denom; + scale_v *= (224 << shift) / denom; + add_y = (float)(16 << shift); + } + + matrix->rgb_to_y[0] = ToFixed16(kr * scale_y); + matrix->rgb_to_y[1] = ToFixed16(kg * scale_y); + matrix->rgb_to_y[2] = ToFixed16(kb * scale_y); + matrix->rgb_to_y[3] = ToFixed16(add_y); + + matrix->rgb_to_u[0] = ToFixed16(-kr * scale_u); + matrix->rgb_to_u[1] = ToFixed16(-kg * scale_u); + matrix->rgb_to_u[2] = ToFixed16((1 - kb) * scale_u); + matrix->rgb_to_u[3] = ToFixed16(add_uv); + + matrix->rgb_to_v[0] = ToFixed16((1 - kr) * scale_v); + matrix->rgb_to_v[1] = ToFixed16(-kg * scale_v); + matrix->rgb_to_v[2] = ToFixed16(-kb * scale_v); + matrix->rgb_to_v[3] = ToFixed16(add_uv); +} + +// Matrices are in YUV_FIX fixed point precision. +// WebP's matrix, similar but not identical to kRec601LimitedMatrix. +static const SharpYuvConversionMatrix kWebpMatrix = { + {16839, 33059, 6420, 16 << 16}, + {-9719, -19081, 28800, 128 << 16}, + {28800, -24116, -4684, 128 << 16}, +}; +// Kr=0.2990f Kb=0.1140f bits=8 range=kSharpYuvRangeLimited +static const SharpYuvConversionMatrix kRec601LimitedMatrix = { + {16829, 33039, 6416, 16 << 16}, + {-9714, -19071, 28784, 128 << 16}, + {28784, -24103, -4681, 128 << 16}, +}; +// Kr=0.2990f Kb=0.1140f bits=8 range=kSharpYuvRangeFull +static const SharpYuvConversionMatrix kRec601FullMatrix = { + {19595, 38470, 7471, 0}, + {-11058, -21710, 32768, 128 << 16}, + {32768, -27439, -5329, 128 << 16}, +}; +// Kr=0.2126f Kb=0.0722f bits=8 range=kSharpYuvRangeLimited +static const SharpYuvConversionMatrix kRec709LimitedMatrix = { + {11966, 40254, 4064, 16 << 16}, + {-6596, -22189, 28784, 128 << 16}, + {28784, -26145, -2639, 128 << 16}, +}; +// Kr=0.2126f Kb=0.0722f bits=8 range=kSharpYuvRangeFull +static const SharpYuvConversionMatrix kRec709FullMatrix = { + {13933, 46871, 4732, 0}, + {-7509, -25259, 32768, 128 << 16}, + {32768, -29763, -3005, 128 << 16}, +}; + +const SharpYuvConversionMatrix* SharpYuvGetConversionMatrix( + SharpYuvMatrixType matrix_type) { + switch (matrix_type) { + case kSharpYuvMatrixWebp: + return &kWebpMatrix; + case kSharpYuvMatrixRec601Limited: + return &kRec601LimitedMatrix; + case kSharpYuvMatrixRec601Full: + return &kRec601FullMatrix; + case kSharpYuvMatrixRec709Limited: + return &kRec709LimitedMatrix; + case kSharpYuvMatrixRec709Full: + return &kRec709FullMatrix; + case kSharpYuvMatrixNum: + return NULL; + } + return NULL; +} diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_csp.h b/3rdparty/libwebp/sharpyuv/sharpyuv_csp.h new file mode 100644 index 0000000000..3214e3ac60 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_csp.h @@ -0,0 +1,60 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Colorspace utilities. + +#ifndef WEBP_SHARPYUV_SHARPYUV_CSP_H_ +#define WEBP_SHARPYUV_SHARPYUV_CSP_H_ + +#include "sharpyuv/sharpyuv.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Range of YUV values. +typedef enum { + kSharpYuvRangeFull, // YUV values between [0;255] (for 8 bit) + kSharpYuvRangeLimited // Y in [16;235], YUV in [16;240] (for 8 bit) +} SharpYuvRange; + +// Constants that define a YUV color space. +typedef struct { + // Kr and Kb are defined such that: + // Y = Kr * r + Kg * g + Kb * b where Kg = 1 - Kr - Kb. + float kr; + float kb; + int bit_depth; // 8, 10 or 12 + SharpYuvRange range; +} SharpYuvColorSpace; + +// Fills in 'matrix' for the given YUVColorSpace. +SHARPYUV_EXTERN void SharpYuvComputeConversionMatrix( + const SharpYuvColorSpace* yuv_color_space, + SharpYuvConversionMatrix* matrix); + +// Enums for precomputed conversion matrices. +typedef enum { + kSharpYuvMatrixWebp = 0, + kSharpYuvMatrixRec601Limited, + kSharpYuvMatrixRec601Full, + kSharpYuvMatrixRec709Limited, + kSharpYuvMatrixRec709Full, + kSharpYuvMatrixNum +} SharpYuvMatrixType; + +// Returns a pointer to a matrix for one of the predefined colorspaces. +SHARPYUV_EXTERN const SharpYuvConversionMatrix* SharpYuvGetConversionMatrix( + SharpYuvMatrixType matrix_type); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_SHARPYUV_SHARPYUV_CSP_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.c b/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.c new file mode 100644 index 0000000000..0da3efc0b8 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.c @@ -0,0 +1,104 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions for Sharp YUV. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "sharpyuv/sharpyuv_dsp.h" + +#include +#include + +#include "sharpyuv/sharpyuv_cpu.h" + +//----------------------------------------------------------------------------- + +#if !WEBP_NEON_OMIT_C_CODE +static uint16_t clip(int v, int max) { + return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; +} + +static uint64_t SharpYuvUpdateY_C(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len, int bit_depth) { + uint64_t diff = 0; + int i; + const int max_y = (1 << bit_depth) - 1; + for (i = 0; i < len; ++i) { + const int diff_y = ref[i] - src[i]; + const int new_y = (int)dst[i] + diff_y; + dst[i] = clip(new_y, max_y); + diff += (uint64_t)abs(diff_y); + } + return diff; +} + +static void SharpYuvUpdateRGB_C(const int16_t* ref, const int16_t* src, + int16_t* dst, int len) { + int i; + for (i = 0; i < len; ++i) { + const int diff_uv = ref[i] - src[i]; + dst[i] += diff_uv; + } +} + +static void SharpYuvFilterRow_C(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth) { + int i; + const int max_y = (1 << bit_depth) - 1; + for (i = 0; i < len; ++i, ++A, ++B) { + const int v0 = (A[0] * 9 + A[1] * 3 + B[0] * 3 + B[1] + 8) >> 4; + const int v1 = (A[1] * 9 + A[0] * 3 + B[1] * 3 + B[0] + 8) >> 4; + out[2 * i + 0] = clip(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip(best_y[2 * i + 1] + v1, max_y); + } +} +#endif // !WEBP_NEON_OMIT_C_CODE + +//----------------------------------------------------------------------------- + +uint64_t (*SharpYuvUpdateY)(const uint16_t* src, const uint16_t* ref, + uint16_t* dst, int len, int bit_depth); +void (*SharpYuvUpdateRGB)(const int16_t* src, const int16_t* ref, int16_t* dst, + int len); +void (*SharpYuvFilterRow)(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth); + +extern VP8CPUInfo SharpYuvGetCPUInfo; +extern void InitSharpYuvSSE2(void); +extern void InitSharpYuvNEON(void); + +void SharpYuvInitDsp(void) { +#if !WEBP_NEON_OMIT_C_CODE + SharpYuvUpdateY = SharpYuvUpdateY_C; + SharpYuvUpdateRGB = SharpYuvUpdateRGB_C; + SharpYuvFilterRow = SharpYuvFilterRow_C; +#endif + + if (SharpYuvGetCPUInfo != NULL) { +#if defined(WEBP_HAVE_SSE2) + if (SharpYuvGetCPUInfo(kSSE2)) { + InitSharpYuvSSE2(); + } +#endif // WEBP_HAVE_SSE2 + } + +#if defined(WEBP_HAVE_NEON) + if (WEBP_NEON_OMIT_C_CODE || + (SharpYuvGetCPUInfo != NULL && SharpYuvGetCPUInfo(kNEON))) { + InitSharpYuvNEON(); + } +#endif // WEBP_HAVE_NEON + + assert(SharpYuvUpdateY != NULL); + assert(SharpYuvUpdateRGB != NULL); + assert(SharpYuvFilterRow != NULL); +} diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.h b/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.h new file mode 100644 index 0000000000..805fbadbf6 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_dsp.h @@ -0,0 +1,28 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions for Sharp YUV. + +#ifndef WEBP_SHARPYUV_SHARPYUV_DSP_H_ +#define WEBP_SHARPYUV_SHARPYUV_DSP_H_ + +#include "sharpyuv/sharpyuv_cpu.h" +#include "src/webp/types.h" + +extern uint64_t (*SharpYuvUpdateY)(const uint16_t* src, const uint16_t* ref, + uint16_t* dst, int len, int bit_depth); +extern void (*SharpYuvUpdateRGB)(const int16_t* src, const int16_t* ref, + int16_t* dst, int len); +extern void (*SharpYuvFilterRow)(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth); + +void SharpYuvInitDsp(void); + +#endif // WEBP_SHARPYUV_SHARPYUV_DSP_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.c b/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.c new file mode 100644 index 0000000000..fecadc6480 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.c @@ -0,0 +1,419 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Gamma correction utilities. + +#include "sharpyuv/sharpyuv_gamma.h" + +#include +#include +#include + +#include "src/webp/types.h" + +// Gamma correction compensates loss of resolution during chroma subsampling. +// Size of pre-computed table for converting from gamma to linear. +#define GAMMA_TO_LINEAR_TAB_BITS 10 +#define GAMMA_TO_LINEAR_TAB_SIZE (1 << GAMMA_TO_LINEAR_TAB_BITS) +static uint32_t kGammaToLinearTabS[GAMMA_TO_LINEAR_TAB_SIZE + 2]; +#define LINEAR_TO_GAMMA_TAB_BITS 9 +#define LINEAR_TO_GAMMA_TAB_SIZE (1 << LINEAR_TO_GAMMA_TAB_BITS) +static uint32_t kLinearToGammaTabS[LINEAR_TO_GAMMA_TAB_SIZE + 2]; + +static const double kGammaF = 1. / 0.45; +#define GAMMA_TO_LINEAR_BITS 16 + +static volatile int kGammaTablesSOk = 0; +void SharpYuvInitGammaTables(void) { + assert(GAMMA_TO_LINEAR_BITS <= 16); + if (!kGammaTablesSOk) { + int v; + const double a = 0.09929682680944; + const double thresh = 0.018053968510807; + const double final_scale = 1 << GAMMA_TO_LINEAR_BITS; + // Precompute gamma to linear table. + { + const double norm = 1. / GAMMA_TO_LINEAR_TAB_SIZE; + const double a_rec = 1. / (1. + a); + for (v = 0; v <= GAMMA_TO_LINEAR_TAB_SIZE; ++v) { + const double g = norm * v; + double value; + if (g <= thresh * 4.5) { + value = g / 4.5; + } else { + value = pow(a_rec * (g + a), kGammaF); + } + kGammaToLinearTabS[v] = (uint32_t)(value * final_scale + .5); + } + // to prevent small rounding errors to cause read-overflow: + kGammaToLinearTabS[GAMMA_TO_LINEAR_TAB_SIZE + 1] = + kGammaToLinearTabS[GAMMA_TO_LINEAR_TAB_SIZE]; + } + // Precompute linear to gamma table. + { + const double scale = 1. / LINEAR_TO_GAMMA_TAB_SIZE; + for (v = 0; v <= LINEAR_TO_GAMMA_TAB_SIZE; ++v) { + const double g = scale * v; + double value; + if (g <= thresh) { + value = 4.5 * g; + } else { + value = (1. + a) * pow(g, 1. / kGammaF) - a; + } + kLinearToGammaTabS[v] = + (uint32_t)(final_scale * value + 0.5); + } + // to prevent small rounding errors to cause read-overflow: + kLinearToGammaTabS[LINEAR_TO_GAMMA_TAB_SIZE + 1] = + kLinearToGammaTabS[LINEAR_TO_GAMMA_TAB_SIZE]; + } + kGammaTablesSOk = 1; + } +} + +static WEBP_INLINE int Shift(int v, int shift) { + return (shift >= 0) ? (v << shift) : (v >> -shift); +} + +static WEBP_INLINE uint32_t FixedPointInterpolation(int v, uint32_t* tab, + int tab_pos_shift_right, + int tab_value_shift) { + const uint32_t tab_pos = Shift(v, -tab_pos_shift_right); + // fractional part, in 'tab_pos_shift' fixed-point precision + const uint32_t x = v - (tab_pos << tab_pos_shift_right); // fractional part + // v0 / v1 are in kGammaToLinearBits fixed-point precision (range [0..1]) + const uint32_t v0 = Shift(tab[tab_pos + 0], tab_value_shift); + const uint32_t v1 = Shift(tab[tab_pos + 1], tab_value_shift); + // Final interpolation. + const uint32_t v2 = (v1 - v0) * x; // note: v1 >= v0. + const int half = + (tab_pos_shift_right > 0) ? 1 << (tab_pos_shift_right - 1) : 0; + const uint32_t result = v0 + ((v2 + half) >> tab_pos_shift_right); + return result; +} + +static uint32_t ToLinearSrgb(uint16_t v, int bit_depth) { + const int shift = GAMMA_TO_LINEAR_TAB_BITS - bit_depth; + if (shift > 0) { + return kGammaToLinearTabS[v << shift]; + } + return FixedPointInterpolation(v, kGammaToLinearTabS, -shift, 0); +} + +static uint16_t FromLinearSrgb(uint32_t value, int bit_depth) { + return FixedPointInterpolation( + value, kLinearToGammaTabS, + (GAMMA_TO_LINEAR_BITS - LINEAR_TO_GAMMA_TAB_BITS), + bit_depth - GAMMA_TO_LINEAR_BITS); +} + +//////////////////////////////////////////////////////////////////////////////// + +#define CLAMP(x, low, high) \ + (((x) < (low)) ? (low) : (((high) < (x)) ? (high) : (x))) +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) + +static WEBP_INLINE float Roundf(float x) { + if (x < 0) + return (float)ceil((double)(x - 0.5f)); + else + return (float)floor((double)(x + 0.5f)); +} + +static WEBP_INLINE float Powf(float base, float exp) { + return (float)pow((double)base, (double)exp); +} + +static WEBP_INLINE float Log10f(float x) { return (float)log10((double)x); } + +static float ToLinear709(float gamma) { + if (gamma < 0.f) { + return 0.f; + } else if (gamma < 4.5f * 0.018053968510807f) { + return gamma / 4.5f; + } else if (gamma < 1.f) { + return Powf((gamma + 0.09929682680944f) / 1.09929682680944f, 1.f / 0.45f); + } + return 1.f; +} + +static float FromLinear709(float linear) { + if (linear < 0.f) { + return 0.f; + } else if (linear < 0.018053968510807f) { + return linear * 4.5f; + } else if (linear < 1.f) { + return 1.09929682680944f * Powf(linear, 0.45f) - 0.09929682680944f; + } + return 1.f; +} + +static float ToLinear470M(float gamma) { + return Powf(CLAMP(gamma, 0.f, 1.f), 1.f / 2.2f); +} + +static float FromLinear470M(float linear) { + return Powf(CLAMP(linear, 0.f, 1.f), 2.2f); +} + +static float ToLinear470Bg(float gamma) { + return Powf(CLAMP(gamma, 0.f, 1.f), 1.f / 2.8f); +} + +static float FromLinear470Bg(float linear) { + return Powf(CLAMP(linear, 0.f, 1.f), 2.8f); +} + +static float ToLinearSmpte240(float gamma) { + if (gamma < 0.f) { + return 0.f; + } else if (gamma < 4.f * 0.022821585529445f) { + return gamma / 4.f; + } else if (gamma < 1.f) { + return Powf((gamma + 0.111572195921731f) / 1.111572195921731f, 1.f / 0.45f); + } + return 1.f; +} + +static float FromLinearSmpte240(float linear) { + if (linear < 0.f) { + return 0.f; + } else if (linear < 0.022821585529445f) { + return linear * 4.f; + } else if (linear < 1.f) { + return 1.111572195921731f * Powf(linear, 0.45f) - 0.111572195921731f; + } + return 1.f; +} + +static float ToLinearLog100(float gamma) { + return (gamma < 0.01f) ? 0.0f : 1.0f + Log10f(MIN(gamma, 1.f)) / 2.0f; +} + +static float FromLinearLog100(float linear) { + // The function is non-bijective so choose the middle of [0, 0.01]. + const float mid_interval = 0.01f / 2.f; + return (linear <= 0.0f) ? mid_interval + : Powf(10.0f, 2.f * (MIN(linear, 1.f) - 1.0f)); +} + +static float ToLinearLog100Sqrt10(float gamma) { + return (gamma < 0.00316227766f) ? 0.0f + : 1.0f + Log10f(MIN(gamma, 1.f)) / 2.5f; +} + +static float FromLinearLog100Sqrt10(float linear) { + // The function is non-bijective so choose the middle of [0, 0.00316227766f[. + const float mid_interval = 0.00316227766f / 2.f; + return (linear < 0.0f) ? mid_interval + : Powf(10.0f, 2.5f * (MIN(linear, 1.f) - 1.0f)); +} + +static float ToLinearIec61966(float gamma) { + if (gamma <= -4.5f * 0.018053968510807f) { + return Powf((-gamma + 0.09929682680944f) / -1.09929682680944f, 1.f / 0.45f); + } else if (gamma < 4.5f * 0.018053968510807f) { + return gamma / 4.5f; + } + return Powf((gamma + 0.09929682680944f) / 1.09929682680944f, 1.f / 0.45f); +} + +static float FromLinearIec61966(float linear) { + if (linear <= -0.018053968510807f) { + return -1.09929682680944f * Powf(-linear, 0.45f) + 0.09929682680944f; + } else if (linear < 0.018053968510807f) { + return linear * 4.5f; + } + return 1.09929682680944f * Powf(linear, 0.45f) - 0.09929682680944f; +} + +static float ToLinearBt1361(float gamma) { + if (gamma < -0.25f) { + return -0.25f; + } else if (gamma < 0.f) { + return Powf((gamma - 0.02482420670236f) / -0.27482420670236f, 1.f / 0.45f) / + -4.f; + } else if (gamma < 4.5f * 0.018053968510807f) { + return gamma / 4.5f; + } else if (gamma < 1.f) { + return Powf((gamma + 0.09929682680944f) / 1.09929682680944f, 1.f / 0.45f); + } + return 1.f; +} + +static float FromLinearBt1361(float linear) { + if (linear < -0.25f) { + return -0.25f; + } else if (linear < 0.f) { + return -0.27482420670236f * Powf(-4.f * linear, 0.45f) + 0.02482420670236f; + } else if (linear < 0.018053968510807f) { + return linear * 4.5f; + } else if (linear < 1.f) { + return 1.09929682680944f * Powf(linear, 0.45f) - 0.09929682680944f; + } + return 1.f; +} + +static float ToLinearPq(float gamma) { + if (gamma > 0.f) { + const float pow_gamma = Powf(gamma, 32.f / 2523.f); + const float num = MAX(pow_gamma - 107.f / 128.f, 0.0f); + const float den = MAX(2413.f / 128.f - 2392.f / 128.f * pow_gamma, FLT_MIN); + return Powf(num / den, 4096.f / 653.f); + } + return 0.f; +} + +static float FromLinearPq(float linear) { + if (linear > 0.f) { + const float pow_linear = Powf(linear, 653.f / 4096.f); + const float num = 107.f / 128.f + 2413.f / 128.f * pow_linear; + const float den = 1.0f + 2392.f / 128.f * pow_linear; + return Powf(num / den, 2523.f / 32.f); + } + return 0.f; +} + +static float ToLinearSmpte428(float gamma) { + return Powf(0.91655527974030934f * MAX(gamma, 0.f), 1.f / 2.6f); +} + +static float FromLinearSmpte428(float linear) { + return Powf(MAX(linear, 0.f), 2.6f) / 0.91655527974030934f; +} + +// Conversion in BT.2100 requires RGB info. Simplify to gamma correction here. +static float ToLinearHlg(float gamma) { + if (gamma < 0.f) { + return 0.f; + } else if (gamma <= 0.5f) { + return Powf((gamma * gamma) * (1.f / 3.f), 1.2f); + } + return Powf((expf((gamma - 0.55991073f) / 0.17883277f) + 0.28466892f) / 12.0f, + 1.2f); +} + +static float FromLinearHlg(float linear) { + linear = Powf(linear, 1.f / 1.2f); + if (linear < 0.f) { + return 0.f; + } else if (linear <= (1.f / 12.f)) { + return sqrtf(3.f * linear); + } + return 0.17883277f * logf(12.f * linear - 0.28466892f) + 0.55991073f; +} + +uint32_t SharpYuvGammaToLinear(uint16_t v, int bit_depth, + SharpYuvTransferFunctionType transfer_type) { + float v_float, linear; + if (transfer_type == kSharpYuvTransferFunctionSrgb) { + return ToLinearSrgb(v, bit_depth); + } + v_float = (float)v / ((1 << bit_depth) - 1); + switch (transfer_type) { + case kSharpYuvTransferFunctionBt709: + case kSharpYuvTransferFunctionBt601: + case kSharpYuvTransferFunctionBt2020_10Bit: + case kSharpYuvTransferFunctionBt2020_12Bit: + linear = ToLinear709(v_float); + break; + case kSharpYuvTransferFunctionBt470M: + linear = ToLinear470M(v_float); + break; + case kSharpYuvTransferFunctionBt470Bg: + linear = ToLinear470Bg(v_float); + break; + case kSharpYuvTransferFunctionSmpte240: + linear = ToLinearSmpte240(v_float); + break; + case kSharpYuvTransferFunctionLinear: + return v; + case kSharpYuvTransferFunctionLog100: + linear = ToLinearLog100(v_float); + break; + case kSharpYuvTransferFunctionLog100_Sqrt10: + linear = ToLinearLog100Sqrt10(v_float); + break; + case kSharpYuvTransferFunctionIec61966: + linear = ToLinearIec61966(v_float); + break; + case kSharpYuvTransferFunctionBt1361: + linear = ToLinearBt1361(v_float); + break; + case kSharpYuvTransferFunctionSmpte2084: + linear = ToLinearPq(v_float); + break; + case kSharpYuvTransferFunctionSmpte428: + linear = ToLinearSmpte428(v_float); + break; + case kSharpYuvTransferFunctionHlg: + linear = ToLinearHlg(v_float); + break; + default: + assert(0); + linear = 0; + break; + } + return (uint32_t)Roundf(linear * ((1 << 16) - 1)); +} + +uint16_t SharpYuvLinearToGamma(uint32_t v, int bit_depth, + SharpYuvTransferFunctionType transfer_type) { + float v_float, linear; + if (transfer_type == kSharpYuvTransferFunctionSrgb) { + return FromLinearSrgb(v, bit_depth); + } + v_float = (float)v / ((1 << 16) - 1); + switch (transfer_type) { + case kSharpYuvTransferFunctionBt709: + case kSharpYuvTransferFunctionBt601: + case kSharpYuvTransferFunctionBt2020_10Bit: + case kSharpYuvTransferFunctionBt2020_12Bit: + linear = FromLinear709(v_float); + break; + case kSharpYuvTransferFunctionBt470M: + linear = FromLinear470M(v_float); + break; + case kSharpYuvTransferFunctionBt470Bg: + linear = FromLinear470Bg(v_float); + break; + case kSharpYuvTransferFunctionSmpte240: + linear = FromLinearSmpte240(v_float); + break; + case kSharpYuvTransferFunctionLinear: + return v; + case kSharpYuvTransferFunctionLog100: + linear = FromLinearLog100(v_float); + break; + case kSharpYuvTransferFunctionLog100_Sqrt10: + linear = FromLinearLog100Sqrt10(v_float); + break; + case kSharpYuvTransferFunctionIec61966: + linear = FromLinearIec61966(v_float); + break; + case kSharpYuvTransferFunctionBt1361: + linear = FromLinearBt1361(v_float); + break; + case kSharpYuvTransferFunctionSmpte2084: + linear = FromLinearPq(v_float); + break; + case kSharpYuvTransferFunctionSmpte428: + linear = FromLinearSmpte428(v_float); + break; + case kSharpYuvTransferFunctionHlg: + linear = FromLinearHlg(v_float); + break; + default: + assert(0); + linear = 0; + break; + } + return (uint16_t)Roundf(linear * ((1 << bit_depth) - 1)); +} diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.h b/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.h new file mode 100644 index 0000000000..b8ba7e9870 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_gamma.h @@ -0,0 +1,38 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Gamma correction utilities. + +#ifndef WEBP_SHARPYUV_SHARPYUV_GAMMA_H_ +#define WEBP_SHARPYUV_SHARPYUV_GAMMA_H_ + +#include "sharpyuv/sharpyuv.h" +#include "src/webp/types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Initializes precomputed tables. Must be called once before calling +// SharpYuvGammaToLinear or SharpYuvLinearToGamma. +void SharpYuvInitGammaTables(void); + +// Converts a 'bit_depth'-bit gamma color value to a 16-bit linear value. +uint32_t SharpYuvGammaToLinear(uint16_t v, int bit_depth, + SharpYuvTransferFunctionType transfer_type); + +// Converts a 16-bit linear color value to a 'bit_depth'-bit gamma value. +uint16_t SharpYuvLinearToGamma(uint32_t value, int bit_depth, + SharpYuvTransferFunctionType transfer_type); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // WEBP_SHARPYUV_SHARPYUV_GAMMA_H_ diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_neon.c b/3rdparty/libwebp/sharpyuv/sharpyuv_neon.c new file mode 100644 index 0000000000..5840914865 --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_neon.c @@ -0,0 +1,181 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions for Sharp YUV. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "sharpyuv/sharpyuv_dsp.h" + +#if defined(WEBP_USE_NEON) +#include +#include +#include + +static uint16_t clip_NEON(int v, int max) { + return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; +} + +static uint64_t SharpYuvUpdateY_NEON(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const int16x8_t zero = vdupq_n_s16(0); + const int16x8_t max = vdupq_n_s16(max_y); + uint64x2_t sum = vdupq_n_u64(0); + uint64_t diff; + + for (i = 0; i + 8 <= len; i += 8) { + const int16x8_t A = vreinterpretq_s16_u16(vld1q_u16(ref + i)); + const int16x8_t B = vreinterpretq_s16_u16(vld1q_u16(src + i)); + const int16x8_t C = vreinterpretq_s16_u16(vld1q_u16(dst + i)); + const int16x8_t D = vsubq_s16(A, B); // diff_y + const int16x8_t F = vaddq_s16(C, D); // new_y + const uint16x8_t H = + vreinterpretq_u16_s16(vmaxq_s16(vminq_s16(F, max), zero)); + const int16x8_t I = vabsq_s16(D); // abs(diff_y) + vst1q_u16(dst + i, H); + sum = vpadalq_u32(sum, vpaddlq_u16(vreinterpretq_u16_s16(I))); + } + diff = vgetq_lane_u64(sum, 0) + vgetq_lane_u64(sum, 1); + for (; i < len; ++i) { + const int diff_y = ref[i] - src[i]; + const int new_y = (int)(dst[i]) + diff_y; + dst[i] = clip_NEON(new_y, max_y); + diff += (uint64_t)(abs(diff_y)); + } + return diff; +} + +static void SharpYuvUpdateRGB_NEON(const int16_t* ref, const int16_t* src, + int16_t* dst, int len) { + int i; + for (i = 0; i + 8 <= len; i += 8) { + const int16x8_t A = vld1q_s16(ref + i); + const int16x8_t B = vld1q_s16(src + i); + const int16x8_t C = vld1q_s16(dst + i); + const int16x8_t D = vsubq_s16(A, B); // diff_uv + const int16x8_t E = vaddq_s16(C, D); // new_uv + vst1q_s16(dst + i, E); + } + for (; i < len; ++i) { + const int diff_uv = ref[i] - src[i]; + dst[i] += diff_uv; + } +} + +static void SharpYuvFilterRow16_NEON(const int16_t* A, const int16_t* B, + int len, const uint16_t* best_y, + uint16_t* out, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const int16x8_t max = vdupq_n_s16(max_y); + const int16x8_t zero = vdupq_n_s16(0); + for (i = 0; i + 8 <= len; i += 8) { + const int16x8_t a0 = vld1q_s16(A + i + 0); + const int16x8_t a1 = vld1q_s16(A + i + 1); + const int16x8_t b0 = vld1q_s16(B + i + 0); + const int16x8_t b1 = vld1q_s16(B + i + 1); + const int16x8_t a0b1 = vaddq_s16(a0, b1); + const int16x8_t a1b0 = vaddq_s16(a1, b0); + const int16x8_t a0a1b0b1 = vaddq_s16(a0b1, a1b0); // A0+A1+B0+B1 + const int16x8_t a0b1_2 = vaddq_s16(a0b1, a0b1); // 2*(A0+B1) + const int16x8_t a1b0_2 = vaddq_s16(a1b0, a1b0); // 2*(A1+B0) + const int16x8_t c0 = vshrq_n_s16(vaddq_s16(a0b1_2, a0a1b0b1), 3); + const int16x8_t c1 = vshrq_n_s16(vaddq_s16(a1b0_2, a0a1b0b1), 3); + const int16x8_t e0 = vrhaddq_s16(c1, a0); + const int16x8_t e1 = vrhaddq_s16(c0, a1); + const int16x8x2_t f = vzipq_s16(e0, e1); + const int16x8_t g0 = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i + 0)); + const int16x8_t g1 = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i + 8)); + const int16x8_t h0 = vaddq_s16(g0, f.val[0]); + const int16x8_t h1 = vaddq_s16(g1, f.val[1]); + const int16x8_t i0 = vmaxq_s16(vminq_s16(h0, max), zero); + const int16x8_t i1 = vmaxq_s16(vminq_s16(h1, max), zero); + vst1q_u16(out + 2 * i + 0, vreinterpretq_u16_s16(i0)); + vst1q_u16(out + 2 * i + 8, vreinterpretq_u16_s16(i1)); + } + for (; i < len; ++i) { + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_NEON(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip_NEON(best_y[2 * i + 1] + v1, max_y); + } +} + +static void SharpYuvFilterRow32_NEON(const int16_t* A, const int16_t* B, + int len, const uint16_t* best_y, + uint16_t* out, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const uint16x8_t max = vdupq_n_u16(max_y); + for (i = 0; i + 4 <= len; i += 4) { + const int16x4_t a0 = vld1_s16(A + i + 0); + const int16x4_t a1 = vld1_s16(A + i + 1); + const int16x4_t b0 = vld1_s16(B + i + 0); + const int16x4_t b1 = vld1_s16(B + i + 1); + const int32x4_t a0b1 = vaddl_s16(a0, b1); + const int32x4_t a1b0 = vaddl_s16(a1, b0); + const int32x4_t a0a1b0b1 = vaddq_s32(a0b1, a1b0); // A0+A1+B0+B1 + const int32x4_t a0b1_2 = vaddq_s32(a0b1, a0b1); // 2*(A0+B1) + const int32x4_t a1b0_2 = vaddq_s32(a1b0, a1b0); // 2*(A1+B0) + const int32x4_t c0 = vshrq_n_s32(vaddq_s32(a0b1_2, a0a1b0b1), 3); + const int32x4_t c1 = vshrq_n_s32(vaddq_s32(a1b0_2, a0a1b0b1), 3); + const int32x4_t e0 = vrhaddq_s32(c1, vmovl_s16(a0)); + const int32x4_t e1 = vrhaddq_s32(c0, vmovl_s16(a1)); + const int32x4x2_t f = vzipq_s32(e0, e1); + + const int16x8_t g = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i)); + const int32x4_t h0 = vaddw_s16(f.val[0], vget_low_s16(g)); + const int32x4_t h1 = vaddw_s16(f.val[1], vget_high_s16(g)); + const uint16x8_t i_16 = vcombine_u16(vqmovun_s32(h0), vqmovun_s32(h1)); + const uint16x8_t i_clamped = vminq_u16(i_16, max); + vst1q_u16(out + 2 * i + 0, i_clamped); + } + for (; i < len; ++i) { + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_NEON(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip_NEON(best_y[2 * i + 1] + v1, max_y); + } +} + +static void SharpYuvFilterRow_NEON(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth) { + if (bit_depth <= 10) { + SharpYuvFilterRow16_NEON(A, B, len, best_y, out, bit_depth); + } else { + SharpYuvFilterRow32_NEON(A, B, len, best_y, out, bit_depth); + } +} + +//------------------------------------------------------------------------------ + +extern void InitSharpYuvNEON(void); + +WEBP_TSAN_IGNORE_FUNCTION void InitSharpYuvNEON(void) { + SharpYuvUpdateY = SharpYuvUpdateY_NEON; + SharpYuvUpdateRGB = SharpYuvUpdateRGB_NEON; + SharpYuvFilterRow = SharpYuvFilterRow_NEON; +} + +#else // !WEBP_USE_NEON + +extern void InitSharpYuvNEON(void); + +void InitSharpYuvNEON(void) {} + +#endif // WEBP_USE_NEON diff --git a/3rdparty/libwebp/sharpyuv/sharpyuv_sse2.c b/3rdparty/libwebp/sharpyuv/sharpyuv_sse2.c new file mode 100644 index 0000000000..9744d1bb6c --- /dev/null +++ b/3rdparty/libwebp/sharpyuv/sharpyuv_sse2.c @@ -0,0 +1,201 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Speed-critical functions for Sharp YUV. +// +// Author: Skal (pascal.massimino@gmail.com) + +#include "sharpyuv/sharpyuv_dsp.h" + +#if defined(WEBP_USE_SSE2) +#include +#include + +static uint16_t clip_SSE2(int v, int max) { + return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; +} + +static uint64_t SharpYuvUpdateY_SSE2(const uint16_t* ref, const uint16_t* src, + uint16_t* dst, int len, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + uint64_t diff = 0; + uint32_t tmp[4]; + int i; + const __m128i zero = _mm_setzero_si128(); + const __m128i max = _mm_set1_epi16(max_y); + const __m128i one = _mm_set1_epi16(1); + __m128i sum = zero; + + for (i = 0; i + 8 <= len; i += 8) { + const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); + const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); + const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); + const __m128i D = _mm_sub_epi16(A, B); // diff_y + const __m128i E = _mm_cmpgt_epi16(zero, D); // sign (-1 or 0) + const __m128i F = _mm_add_epi16(C, D); // new_y + const __m128i G = _mm_or_si128(E, one); // -1 or 1 + const __m128i H = _mm_max_epi16(_mm_min_epi16(F, max), zero); + const __m128i I = _mm_madd_epi16(D, G); // sum(abs(...)) + _mm_storeu_si128((__m128i*)(dst + i), H); + sum = _mm_add_epi32(sum, I); + } + _mm_storeu_si128((__m128i*)tmp, sum); + diff = tmp[3] + tmp[2] + tmp[1] + tmp[0]; + for (; i < len; ++i) { + const int diff_y = ref[i] - src[i]; + const int new_y = (int)dst[i] + diff_y; + dst[i] = clip_SSE2(new_y, max_y); + diff += (uint64_t)abs(diff_y); + } + return diff; +} + +static void SharpYuvUpdateRGB_SSE2(const int16_t* ref, const int16_t* src, + int16_t* dst, int len) { + int i = 0; + for (i = 0; i + 8 <= len; i += 8) { + const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); + const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); + const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); + const __m128i D = _mm_sub_epi16(A, B); // diff_uv + const __m128i E = _mm_add_epi16(C, D); // new_uv + _mm_storeu_si128((__m128i*)(dst + i), E); + } + for (; i < len; ++i) { + const int diff_uv = ref[i] - src[i]; + dst[i] += diff_uv; + } +} + +static void SharpYuvFilterRow16_SSE2(const int16_t* A, const int16_t* B, + int len, const uint16_t* best_y, + uint16_t* out, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const __m128i kCst8 = _mm_set1_epi16(8); + const __m128i max = _mm_set1_epi16(max_y); + const __m128i zero = _mm_setzero_si128(); + for (i = 0; i + 8 <= len; i += 8) { + const __m128i a0 = _mm_loadu_si128((const __m128i*)(A + i + 0)); + const __m128i a1 = _mm_loadu_si128((const __m128i*)(A + i + 1)); + const __m128i b0 = _mm_loadu_si128((const __m128i*)(B + i + 0)); + const __m128i b1 = _mm_loadu_si128((const __m128i*)(B + i + 1)); + const __m128i a0b1 = _mm_add_epi16(a0, b1); + const __m128i a1b0 = _mm_add_epi16(a1, b0); + const __m128i a0a1b0b1 = _mm_add_epi16(a0b1, a1b0); // A0+A1+B0+B1 + const __m128i a0a1b0b1_8 = _mm_add_epi16(a0a1b0b1, kCst8); + const __m128i a0b1_2 = _mm_add_epi16(a0b1, a0b1); // 2*(A0+B1) + const __m128i a1b0_2 = _mm_add_epi16(a1b0, a1b0); // 2*(A1+B0) + const __m128i c0 = _mm_srai_epi16(_mm_add_epi16(a0b1_2, a0a1b0b1_8), 3); + const __m128i c1 = _mm_srai_epi16(_mm_add_epi16(a1b0_2, a0a1b0b1_8), 3); + const __m128i d0 = _mm_add_epi16(c1, a0); + const __m128i d1 = _mm_add_epi16(c0, a1); + const __m128i e0 = _mm_srai_epi16(d0, 1); + const __m128i e1 = _mm_srai_epi16(d1, 1); + const __m128i f0 = _mm_unpacklo_epi16(e0, e1); + const __m128i f1 = _mm_unpackhi_epi16(e0, e1); + const __m128i g0 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); + const __m128i g1 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 8)); + const __m128i h0 = _mm_add_epi16(g0, f0); + const __m128i h1 = _mm_add_epi16(g1, f1); + const __m128i i0 = _mm_max_epi16(_mm_min_epi16(h0, max), zero); + const __m128i i1 = _mm_max_epi16(_mm_min_epi16(h1, max), zero); + _mm_storeu_si128((__m128i*)(out + 2 * i + 0), i0); + _mm_storeu_si128((__m128i*)(out + 2 * i + 8), i1); + } + for (; i < len; ++i) { + // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = + // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 + // We reuse the common sub-expressions. + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_SSE2(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip_SSE2(best_y[2 * i + 1] + v1, max_y); + } +} + +static WEBP_INLINE __m128i s16_to_s32(__m128i in) { + return _mm_srai_epi32(_mm_unpacklo_epi16(in, in), 16); +} + +static void SharpYuvFilterRow32_SSE2(const int16_t* A, const int16_t* B, + int len, const uint16_t* best_y, + uint16_t* out, int bit_depth) { + const int max_y = (1 << bit_depth) - 1; + int i; + const __m128i kCst8 = _mm_set1_epi32(8); + const __m128i max = _mm_set1_epi16(max_y); + const __m128i zero = _mm_setzero_si128(); + for (i = 0; i + 4 <= len; i += 4) { + const __m128i a0 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(A + i + 0))); + const __m128i a1 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(A + i + 1))); + const __m128i b0 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(B + i + 0))); + const __m128i b1 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(B + i + 1))); + const __m128i a0b1 = _mm_add_epi32(a0, b1); + const __m128i a1b0 = _mm_add_epi32(a1, b0); + const __m128i a0a1b0b1 = _mm_add_epi32(a0b1, a1b0); // A0+A1+B0+B1 + const __m128i a0a1b0b1_8 = _mm_add_epi32(a0a1b0b1, kCst8); + const __m128i a0b1_2 = _mm_add_epi32(a0b1, a0b1); // 2*(A0+B1) + const __m128i a1b0_2 = _mm_add_epi32(a1b0, a1b0); // 2*(A1+B0) + const __m128i c0 = _mm_srai_epi32(_mm_add_epi32(a0b1_2, a0a1b0b1_8), 3); + const __m128i c1 = _mm_srai_epi32(_mm_add_epi32(a1b0_2, a0a1b0b1_8), 3); + const __m128i d0 = _mm_add_epi32(c1, a0); + const __m128i d1 = _mm_add_epi32(c0, a1); + const __m128i e0 = _mm_srai_epi32(d0, 1); + const __m128i e1 = _mm_srai_epi32(d1, 1); + const __m128i f0 = _mm_unpacklo_epi32(e0, e1); + const __m128i f1 = _mm_unpackhi_epi32(e0, e1); + const __m128i g = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); + const __m128i h_16 = _mm_add_epi16(g, _mm_packs_epi32(f0, f1)); + const __m128i final = _mm_max_epi16(_mm_min_epi16(h_16, max), zero); + _mm_storeu_si128((__m128i*)(out + 2 * i + 0), final); + } + for (; i < len; ++i) { + // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = + // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 + // We reuse the common sub-expressions. + const int a0b1 = A[i + 0] + B[i + 1]; + const int a1b0 = A[i + 1] + B[i + 0]; + const int a0a1b0b1 = a0b1 + a1b0 + 8; + const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; + const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; + out[2 * i + 0] = clip_SSE2(best_y[2 * i + 0] + v0, max_y); + out[2 * i + 1] = clip_SSE2(best_y[2 * i + 1] + v1, max_y); + } +} + +static void SharpYuvFilterRow_SSE2(const int16_t* A, const int16_t* B, int len, + const uint16_t* best_y, uint16_t* out, + int bit_depth) { + if (bit_depth <= 10) { + SharpYuvFilterRow16_SSE2(A, B, len, best_y, out, bit_depth); + } else { + SharpYuvFilterRow32_SSE2(A, B, len, best_y, out, bit_depth); + } +} + +//------------------------------------------------------------------------------ + +extern void InitSharpYuvSSE2(void); + +WEBP_TSAN_IGNORE_FUNCTION void InitSharpYuvSSE2(void) { + SharpYuvUpdateY = SharpYuvUpdateY_SSE2; + SharpYuvUpdateRGB = SharpYuvUpdateRGB_SSE2; + SharpYuvFilterRow = SharpYuvFilterRow_SSE2; +} +#else // !WEBP_USE_SSE2 + +extern void InitSharpYuvSSE2(void); + +void InitSharpYuvSSE2(void) {} + +#endif // WEBP_USE_SSE2 diff --git a/3rdparty/libwebp/src/dec/alpha_dec.c b/3rdparty/libwebp/src/dec/alpha_dec.c index bce735bfc2..663255c42f 100644 --- a/3rdparty/libwebp/src/dec/alpha_dec.c +++ b/3rdparty/libwebp/src/dec/alpha_dec.c @@ -117,21 +117,12 @@ static int ALPHDecode(VP8Decoder* const dec, int row, int num_rows) { const uint8_t* deltas = dec->alpha_data_ + ALPHA_HEADER_LEN + row * width; uint8_t* dst = dec->alpha_plane_ + row * width; assert(deltas <= &dec->alpha_data_[dec->alpha_data_size_]); - if (alph_dec->filter_ != WEBP_FILTER_NONE) { - assert(WebPUnfilters[alph_dec->filter_] != NULL); - for (y = 0; y < num_rows; ++y) { - WebPUnfilters[alph_dec->filter_](prev_line, deltas, dst, width); - prev_line = dst; - dst += width; - deltas += width; - } - } else { - for (y = 0; y < num_rows; ++y) { - memcpy(dst, deltas, width * sizeof(*dst)); - prev_line = dst; - dst += width; - deltas += width; - } + assert(WebPUnfilters[alph_dec->filter_] != NULL); + for (y = 0; y < num_rows; ++y) { + WebPUnfilters[alph_dec->filter_](prev_line, deltas, dst, width); + prev_line = dst; + dst += width; + deltas += width; } dec->alpha_prev_line_ = prev_line; } else { // alph_dec->method_ == ALPHA_LOSSLESS_COMPRESSION @@ -155,7 +146,8 @@ static int AllocateAlphaPlane(VP8Decoder* const dec, const VP8Io* const io) { dec->alpha_plane_mem_ = (uint8_t*)WebPSafeMalloc(alpha_size, sizeof(*dec->alpha_plane_)); if (dec->alpha_plane_mem_ == NULL) { - return 0; + return VP8SetError(dec, VP8_STATUS_OUT_OF_MEMORY, + "Alpha decoder initialization failed."); } dec->alpha_plane_ = dec->alpha_plane_mem_; dec->alpha_prev_line_ = NULL; @@ -183,16 +175,25 @@ const uint8_t* VP8DecompressAlphaRows(VP8Decoder* const dec, assert(dec != NULL && io != NULL); if (row < 0 || num_rows <= 0 || row + num_rows > height) { - return NULL; // sanity check. + return NULL; } if (!dec->is_alpha_decoded_) { if (dec->alph_dec_ == NULL) { // Initialize decoder. dec->alph_dec_ = ALPHNew(); - if (dec->alph_dec_ == NULL) return NULL; + if (dec->alph_dec_ == NULL) { + VP8SetError(dec, VP8_STATUS_OUT_OF_MEMORY, + "Alpha decoder initialization failed."); + return NULL; + } if (!AllocateAlphaPlane(dec, io)) goto Error; if (!ALPHInit(dec->alph_dec_, dec->alpha_data_, dec->alpha_data_size_, io, dec->alpha_plane_)) { + VP8LDecoder* const vp8l_dec = dec->alph_dec_->vp8l_dec_; + VP8SetError(dec, + (vp8l_dec == NULL) ? VP8_STATUS_OUT_OF_MEMORY + : vp8l_dec->status_, + "Alpha decoder initialization failed."); goto Error; } // if we allowed use of alpha dithering, check whether it's needed at all diff --git a/3rdparty/libwebp/src/dec/buffer_dec.c b/3rdparty/libwebp/src/dec/buffer_dec.c index 3cd94eb4d9..11ce76f19e 100644 --- a/3rdparty/libwebp/src/dec/buffer_dec.c +++ b/3rdparty/libwebp/src/dec/buffer_dec.c @@ -75,7 +75,7 @@ static VP8StatusCode CheckDecBuffer(const WebPDecBuffer* const buffer) { const WebPRGBABuffer* const buf = &buffer->u.RGBA; const int stride = abs(buf->stride); const uint64_t size = - MIN_BUFFER_SIZE(width * kModeBpp[mode], height, stride); + MIN_BUFFER_SIZE((uint64_t)width * kModeBpp[mode], height, stride); ok &= (size <= buf->size); ok &= (stride >= width * kModeBpp[mode]); ok &= (buf->rgba != NULL); @@ -102,7 +102,7 @@ static VP8StatusCode AllocateBuffer(WebPDecBuffer* const buffer) { int stride; uint64_t size; - if ((uint64_t)w * kModeBpp[mode] >= (1ull << 32)) { + if ((uint64_t)w * kModeBpp[mode] >= (1ull << 31)) { return VP8_STATUS_INVALID_PARAM; } stride = w * kModeBpp[mode]; @@ -117,7 +117,6 @@ static VP8StatusCode AllocateBuffer(WebPDecBuffer* const buffer) { } total_size = size + 2 * uv_size + a_size; - // Security/sanity checks output = (uint8_t*)WebPSafeMalloc(total_size, sizeof(*output)); if (output == NULL) { return VP8_STATUS_OUT_OF_MEMORY; @@ -156,11 +155,11 @@ VP8StatusCode WebPFlipBuffer(WebPDecBuffer* const buffer) { } if (WebPIsRGBMode(buffer->colorspace)) { WebPRGBABuffer* const buf = &buffer->u.RGBA; - buf->rgba += (buffer->height - 1) * buf->stride; + buf->rgba += (int64_t)(buffer->height - 1) * buf->stride; buf->stride = -buf->stride; } else { WebPYUVABuffer* const buf = &buffer->u.YUVA; - const int H = buffer->height; + const int64_t H = buffer->height; buf->y += (H - 1) * buf->y_stride; buf->y_stride = -buf->y_stride; buf->u += ((H - 1) >> 1) * buf->u_stride; @@ -188,8 +187,7 @@ VP8StatusCode WebPAllocateDecBuffer(int width, int height, const int ch = options->crop_height; const int x = options->crop_left & ~1; const int y = options->crop_top & ~1; - if (x < 0 || y < 0 || cw <= 0 || ch <= 0 || - x + cw > width || y + ch > height) { + if (!WebPCheckCropDimensions(width, height, x, y, cw, ch)) { return VP8_STATUS_INVALID_PARAM; // out of frame boundary. } width = cw; diff --git a/3rdparty/libwebp/src/dec/frame_dec.c b/3rdparty/libwebp/src/dec/frame_dec.c index 04609a8e56..91ca1f8609 100644 --- a/3rdparty/libwebp/src/dec/frame_dec.c +++ b/3rdparty/libwebp/src/dec/frame_dec.c @@ -705,7 +705,7 @@ static int AllocateMemory(VP8Decoder* const dec) { + cache_size + alpha_size + WEBP_ALIGN_CST; uint8_t* mem; - if (needed != (size_t)needed) return 0; // check for overflow + if (!CheckSizeOverflow(needed)) return 0; // check for overflow if (needed > dec->mem_size_) { WebPSafeFree(dec->mem_); dec->mem_size_ = 0; diff --git a/3rdparty/libwebp/src/dec/io_dec.c b/3rdparty/libwebp/src/dec/io_dec.c index 29dc6345df..5ef6298886 100644 --- a/3rdparty/libwebp/src/dec/io_dec.c +++ b/3rdparty/libwebp/src/dec/io_dec.c @@ -298,46 +298,57 @@ static int InitYUVRescaler(const VP8Io* const io, WebPDecParams* const p) { const int uv_out_height = (out_height + 1) >> 1; const int uv_in_width = (io->mb_w + 1) >> 1; const int uv_in_height = (io->mb_h + 1) >> 1; - const size_t work_size = 2 * out_width; // scratch memory for luma rescaler + // scratch memory for luma rescaler + const size_t work_size = 2 * (size_t)out_width; const size_t uv_work_size = 2 * uv_out_width; // and for each u/v ones - size_t tmp_size, rescaler_size; + uint64_t total_size; + size_t rescaler_size; rescaler_t* work; WebPRescaler* scalers; const int num_rescalers = has_alpha ? 4 : 3; - tmp_size = (work_size + 2 * uv_work_size) * sizeof(*work); + total_size = ((uint64_t)work_size + 2 * uv_work_size) * sizeof(*work); if (has_alpha) { - tmp_size += work_size * sizeof(*work); + total_size += (uint64_t)work_size * sizeof(*work); } rescaler_size = num_rescalers * sizeof(*p->scaler_y) + WEBP_ALIGN_CST; + total_size += rescaler_size; + if (!CheckSizeOverflow(total_size)) { + return 0; + } - p->memory = WebPSafeMalloc(1ULL, tmp_size + rescaler_size); + p->memory = WebPSafeMalloc(1ULL, (size_t)total_size); if (p->memory == NULL) { return 0; // memory error } work = (rescaler_t*)p->memory; - scalers = (WebPRescaler*)WEBP_ALIGN((const uint8_t*)work + tmp_size); + scalers = (WebPRescaler*)WEBP_ALIGN( + (const uint8_t*)work + total_size - rescaler_size); p->scaler_y = &scalers[0]; p->scaler_u = &scalers[1]; p->scaler_v = &scalers[2]; p->scaler_a = has_alpha ? &scalers[3] : NULL; - WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h, - buf->y, out_width, out_height, buf->y_stride, 1, - work); - WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height, - buf->u, uv_out_width, uv_out_height, buf->u_stride, 1, - work + work_size); - WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height, - buf->v, uv_out_width, uv_out_height, buf->v_stride, 1, - work + work_size + uv_work_size); + if (!WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h, + buf->y, out_width, out_height, buf->y_stride, 1, + work) || + !WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height, + buf->u, uv_out_width, uv_out_height, buf->u_stride, 1, + work + work_size) || + !WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height, + buf->v, uv_out_width, uv_out_height, buf->v_stride, 1, + work + work_size + uv_work_size)) { + return 0; + } p->emit = EmitRescaledYUV; if (has_alpha) { - WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h, - buf->a, out_width, out_height, buf->a_stride, 1, - work + work_size + 2 * uv_work_size); + if (!WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h, + buf->a, out_width, out_height, buf->a_stride, 1, + work + work_size + 2 * uv_work_size)) { + return 0; + } p->emit_alpha = EmitRescaledAlphaYUV; WebPInitAlphaProcessing(); } @@ -480,51 +491,58 @@ static int InitRGBRescaler(const VP8Io* const io, WebPDecParams* const p) { const int out_height = io->scaled_height; const int uv_in_width = (io->mb_w + 1) >> 1; const int uv_in_height = (io->mb_h + 1) >> 1; - const size_t work_size = 2 * out_width; // scratch memory for one rescaler + // scratch memory for one rescaler + const size_t work_size = 2 * (size_t)out_width; rescaler_t* work; // rescalers work area uint8_t* tmp; // tmp storage for scaled YUV444 samples before RGB conversion - size_t tmp_size1, tmp_size2, total_size, rescaler_size; + uint64_t tmp_size1, tmp_size2, total_size; + size_t rescaler_size; WebPRescaler* scalers; const int num_rescalers = has_alpha ? 4 : 3; - tmp_size1 = 3 * work_size; - tmp_size2 = 3 * out_width; - if (has_alpha) { - tmp_size1 += work_size; - tmp_size2 += out_width; - } + tmp_size1 = (uint64_t)num_rescalers * work_size; + tmp_size2 = (uint64_t)num_rescalers * out_width; total_size = tmp_size1 * sizeof(*work) + tmp_size2 * sizeof(*tmp); rescaler_size = num_rescalers * sizeof(*p->scaler_y) + WEBP_ALIGN_CST; + total_size += rescaler_size; + if (!CheckSizeOverflow(total_size)) { + return 0; + } - p->memory = WebPSafeMalloc(1ULL, total_size + rescaler_size); + p->memory = WebPSafeMalloc(1ULL, (size_t)total_size); if (p->memory == NULL) { return 0; // memory error } work = (rescaler_t*)p->memory; tmp = (uint8_t*)(work + tmp_size1); - scalers = (WebPRescaler*)WEBP_ALIGN((const uint8_t*)work + total_size); + scalers = (WebPRescaler*)WEBP_ALIGN( + (const uint8_t*)work + total_size - rescaler_size); p->scaler_y = &scalers[0]; p->scaler_u = &scalers[1]; p->scaler_v = &scalers[2]; p->scaler_a = has_alpha ? &scalers[3] : NULL; - WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h, - tmp + 0 * out_width, out_width, out_height, 0, 1, - work + 0 * work_size); - WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height, - tmp + 1 * out_width, out_width, out_height, 0, 1, - work + 1 * work_size); - WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height, - tmp + 2 * out_width, out_width, out_height, 0, 1, - work + 2 * work_size); + if (!WebPRescalerInit(p->scaler_y, io->mb_w, io->mb_h, + tmp + 0 * out_width, out_width, out_height, 0, 1, + work + 0 * work_size) || + !WebPRescalerInit(p->scaler_u, uv_in_width, uv_in_height, + tmp + 1 * out_width, out_width, out_height, 0, 1, + work + 1 * work_size) || + !WebPRescalerInit(p->scaler_v, uv_in_width, uv_in_height, + tmp + 2 * out_width, out_width, out_height, 0, 1, + work + 2 * work_size)) { + return 0; + } p->emit = EmitRescaledRGB; WebPInitYUV444Converters(); if (has_alpha) { - WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h, - tmp + 3 * out_width, out_width, out_height, 0, 1, - work + 3 * work_size); + if (!WebPRescalerInit(p->scaler_a, io->mb_w, io->mb_h, + tmp + 3 * out_width, out_width, out_height, 0, 1, + work + 3 * work_size)) { + return 0; + } p->emit_alpha = EmitRescaledAlphaRGB; if (p->output->colorspace == MODE_RGBA_4444 || p->output->colorspace == MODE_rgbA_4444) { diff --git a/3rdparty/libwebp/src/dec/tree_dec.c b/3rdparty/libwebp/src/dec/tree_dec.c index 1c6fdea27c..2434605953 100644 --- a/3rdparty/libwebp/src/dec/tree_dec.c +++ b/3rdparty/libwebp/src/dec/tree_dec.c @@ -12,10 +12,11 @@ // Author: Skal (pascal.massimino@gmail.com) #include "src/dec/vp8i_dec.h" +#include "src/dsp/cpu.h" #include "src/utils/bit_reader_inl_utils.h" #if !defined(USE_GENERIC_TREE) -#if !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) +#if !defined(__arm__) && !defined(_M_ARM) && !WEBP_AARCH64 // using a table is ~1-2% slower on ARM. Prefer the coded-tree approach then. #define USE_GENERIC_TREE 1 // ALTERNATE_CODE #else diff --git a/3rdparty/libwebp/src/dec/vp8_dec.c b/3rdparty/libwebp/src/dec/vp8_dec.c index 8f73697478..20b92e84c4 100644 --- a/3rdparty/libwebp/src/dec/vp8_dec.c +++ b/3rdparty/libwebp/src/dec/vp8_dec.c @@ -335,7 +335,7 @@ int VP8GetHeaders(VP8Decoder* const dec, VP8Io* const io) { io->scaled_width = io->width; io->scaled_height = io->height; - io->mb_w = io->width; // sanity check + io->mb_w = io->width; // for soundness io->mb_h = io->height; // ditto VP8ResetProba(&dec->proba_); @@ -403,7 +403,7 @@ static const uint8_t kZigzag[16] = { 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15 }; -// See section 13-2: http://tools.ietf.org/html/rfc6386#section-13.2 +// See section 13-2: https://datatracker.ietf.org/doc/html/rfc6386#section-13.2 static int GetLargeValue(VP8BitReader* const br, const uint8_t* const p) { int v; if (!VP8GetBit(br, p[3], "coeffs")) { @@ -494,6 +494,8 @@ static int GetCoeffsAlt(VP8BitReader* const br, return 16; } +extern VP8CPUInfo VP8GetCPUInfo; + WEBP_DSP_INIT_FUNC(InitGetCoeffs) { if (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kSlowSSSE3)) { GetCoeffs = GetCoeffsAlt; diff --git a/3rdparty/libwebp/src/dec/vp8i_dec.h b/3rdparty/libwebp/src/dec/vp8i_dec.h index a0c0af1579..1ae4ff62f2 100644 --- a/3rdparty/libwebp/src/dec/vp8i_dec.h +++ b/3rdparty/libwebp/src/dec/vp8i_dec.h @@ -31,8 +31,8 @@ extern "C" { // version numbers #define DEC_MAJ_VERSION 1 -#define DEC_MIN_VERSION 2 -#define DEC_REV_VERSION 0 +#define DEC_MIN_VERSION 3 +#define DEC_REV_VERSION 1 // YUV-cache parameters. Cache is 32-bytes wide (= one cacheline). // Constraints are: We need to store one 16x16 block of luma samples (y), diff --git a/3rdparty/libwebp/src/dec/vp8l_dec.c b/3rdparty/libwebp/src/dec/vp8l_dec.c index 2d603b4379..11c00ea964 100644 --- a/3rdparty/libwebp/src/dec/vp8l_dec.c +++ b/3rdparty/libwebp/src/dec/vp8l_dec.c @@ -12,6 +12,7 @@ // Authors: Vikas Arora (vikaas.arora@gmail.com) // Jyrki Alakuijala (jyrki@google.com) +#include #include #include "src/dec/alphai_dec.h" @@ -84,7 +85,7 @@ static const uint8_t kCodeToPlane[CODE_TO_PLANE_CODES] = { // to 256 (green component values) + 24 (length prefix values) // + color_cache_size (between 0 and 2048). // All values computed for 8-bit first level lookup with Mark Adler's tool: -// http://www.hdfgroup.org/ftp/lib-external/zlib/zlib-1.2.5/examples/enough.c +// https://github.com/madler/zlib/blob/v1.2.5/examples/enough.c #define FIXED_TABLE_SIZE (630 * 3 + 410) static const uint16_t kTableSize[12] = { FIXED_TABLE_SIZE + 654, @@ -101,6 +102,14 @@ static const uint16_t kTableSize[12] = { FIXED_TABLE_SIZE + 2704 }; +static int VP8LSetError(VP8LDecoder* const dec, VP8StatusCode error) { + // The oldest error reported takes precedence over the new one. + if (dec->status_ == VP8_STATUS_OK || dec->status_ == VP8_STATUS_SUSPENDED) { + dec->status_ = error; + } + return 0; +} + static int DecodeImageStream(int xsize, int ysize, int is_level0, VP8LDecoder* const dec, @@ -178,7 +187,7 @@ static WEBP_INLINE int PlaneCodeToDistance(int xsize, int plane_code) { //------------------------------------------------------------------------------ // Decodes the next Huffman code from bit-stream. -// FillBitWindow(br) needs to be called at minimum every second call +// VP8LFillBitWindow(br) needs to be called at minimum every second call // to ReadSymbol, in order to pre-fetch enough bits. static WEBP_INLINE int ReadSymbol(const HuffmanCode* table, VP8LBitReader* const br) { @@ -253,11 +262,11 @@ static int ReadHuffmanCodeLengths( int symbol; int max_symbol; int prev_code_len = DEFAULT_CODE_LENGTH; - HuffmanCode table[1 << LENGTHS_TABLE_BITS]; + HuffmanTables tables; - if (!VP8LBuildHuffmanTable(table, LENGTHS_TABLE_BITS, - code_length_code_lengths, - NUM_CODE_LENGTH_CODES)) { + if (!VP8LHuffmanTablesAllocate(1 << LENGTHS_TABLE_BITS, &tables) || + !VP8LBuildHuffmanTable(&tables, LENGTHS_TABLE_BITS, + code_length_code_lengths, NUM_CODE_LENGTH_CODES)) { goto End; } @@ -277,7 +286,7 @@ static int ReadHuffmanCodeLengths( int code_len; if (max_symbol-- == 0) break; VP8LFillBitWindow(br); - p = &table[VP8LPrefetchBits(br) & LENGTHS_TABLE_MASK]; + p = &tables.curr_segment->start[VP8LPrefetchBits(br) & LENGTHS_TABLE_MASK]; VP8LSetBitPos(br, br->bit_pos_ + p->bits); code_len = p->value; if (code_len < kCodeLengthLiterals) { @@ -300,14 +309,16 @@ static int ReadHuffmanCodeLengths( ok = 1; End: - if (!ok) dec->status_ = VP8_STATUS_BITSTREAM_ERROR; + VP8LHuffmanTablesDeallocate(&tables); + if (!ok) return VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); return ok; } // 'code_lengths' is pre-allocated temporary buffer, used for creating Huffman // tree. static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, - int* const code_lengths, HuffmanCode* const table) { + int* const code_lengths, + HuffmanTables* const table) { int ok = 0; int size = 0; VP8LBitReader* const br = &dec->br_; @@ -321,7 +332,7 @@ static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, // The first code is either 1 bit or 8 bit code. int symbol = VP8LReadBits(br, (first_symbol_len_code == 0) ? 1 : 8); code_lengths[symbol] = 1; - // The second code (if present), is always 8 bit long. + // The second code (if present), is always 8 bits long. if (num_symbols == 2) { symbol = VP8LReadBits(br, 8); code_lengths[symbol] = 1; @@ -331,10 +342,7 @@ static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, int i; int code_length_code_lengths[NUM_CODE_LENGTH_CODES] = { 0 }; const int num_codes = VP8LReadBits(br, 4) + 4; - if (num_codes > NUM_CODE_LENGTH_CODES) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; - return 0; - } + assert(num_codes <= NUM_CODE_LENGTH_CODES); for (i = 0; i < num_codes; ++i) { code_length_code_lengths[kCodeLengthCodeOrder[i]] = VP8LReadBits(br, 3); @@ -349,36 +357,35 @@ static int ReadHuffmanCode(int alphabet_size, VP8LDecoder* const dec, code_lengths, alphabet_size); } if (!ok || size == 0) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; - return 0; + return VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); } return size; } static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, int color_cache_bits, int allow_recursion) { - int i, j; + int i; VP8LBitReader* const br = &dec->br_; VP8LMetadata* const hdr = &dec->hdr_; uint32_t* huffman_image = NULL; HTreeGroup* htree_groups = NULL; - HuffmanCode* huffman_tables = NULL; - HuffmanCode* huffman_table = NULL; + HuffmanTables* huffman_tables = &hdr->huffman_tables_; int num_htree_groups = 1; int num_htree_groups_max = 1; - int max_alphabet_size = 0; - int* code_lengths = NULL; - const int table_size = kTableSize[color_cache_bits]; int* mapping = NULL; int ok = 0; + // Check the table has been 0 initialized (through InitMetadata). + assert(huffman_tables->root.start == NULL); + assert(huffman_tables->curr_segment == NULL); + if (allow_recursion && VP8LReadBits(br, 1)) { // use meta Huffman codes. const int huffman_precision = VP8LReadBits(br, 3) + 2; const int huffman_xsize = VP8LSubSampleSize(xsize, huffman_precision); const int huffman_ysize = VP8LSubSampleSize(ysize, huffman_precision); const int huffman_pixs = huffman_xsize * huffman_ysize; - if (!DecodeImageStream(huffman_xsize, huffman_ysize, 0, dec, + if (!DecodeImageStream(huffman_xsize, huffman_ysize, /*is_level0=*/0, dec, &huffman_image)) { goto Error; } @@ -402,7 +409,7 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, // values [0, num_htree_groups) mapping = (int*)WebPSafeMalloc(num_htree_groups_max, sizeof(*mapping)); if (mapping == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; + VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto Error; } // -1 means a value is unmapped, and therefore unused in the Huffman @@ -421,29 +428,55 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, if (br->eos_) goto Error; - // Find maximum alphabet size for the htree group. - for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { - int alphabet_size = kAlphabetSize[j]; - if (j == 0 && color_cache_bits > 0) { - alphabet_size += 1 << color_cache_bits; - } - if (max_alphabet_size < alphabet_size) { - max_alphabet_size = alphabet_size; - } + if (!ReadHuffmanCodesHelper(color_cache_bits, num_htree_groups, + num_htree_groups_max, mapping, dec, + huffman_tables, &htree_groups)) { + goto Error; } + ok = 1; - code_lengths = (int*)WebPSafeCalloc((uint64_t)max_alphabet_size, - sizeof(*code_lengths)); - huffman_tables = (HuffmanCode*)WebPSafeMalloc(num_htree_groups * table_size, - sizeof(*huffman_tables)); - htree_groups = VP8LHtreeGroupsNew(num_htree_groups); + // All OK. Finalize pointers. + hdr->huffman_image_ = huffman_image; + hdr->num_htree_groups_ = num_htree_groups; + hdr->htree_groups_ = htree_groups; - if (htree_groups == NULL || code_lengths == NULL || huffman_tables == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; + Error: + WebPSafeFree(mapping); + if (!ok) { + WebPSafeFree(huffman_image); + VP8LHuffmanTablesDeallocate(huffman_tables); + VP8LHtreeGroupsFree(htree_groups); + } + return ok; +} + +int ReadHuffmanCodesHelper(int color_cache_bits, int num_htree_groups, + int num_htree_groups_max, const int* const mapping, + VP8LDecoder* const dec, + HuffmanTables* const huffman_tables, + HTreeGroup** const htree_groups) { + int i, j, ok = 0; + const int max_alphabet_size = + kAlphabetSize[0] + ((color_cache_bits > 0) ? 1 << color_cache_bits : 0); + const int table_size = kTableSize[color_cache_bits]; + int* code_lengths = NULL; + + if ((mapping == NULL && num_htree_groups != num_htree_groups_max) || + num_htree_groups > num_htree_groups_max) { + goto Error; + } + + code_lengths = + (int*)WebPSafeCalloc((uint64_t)max_alphabet_size, sizeof(*code_lengths)); + *htree_groups = VP8LHtreeGroupsNew(num_htree_groups); + + if (*htree_groups == NULL || code_lengths == NULL || + !VP8LHuffmanTablesAllocate(num_htree_groups * table_size, + huffman_tables)) { + VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto Error; } - huffman_table = huffman_tables; for (i = 0; i < num_htree_groups_max; ++i) { // If the index "i" is unused in the Huffman image, just make sure the // coefficients are valid but do not store them. @@ -460,7 +493,7 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, } } else { HTreeGroup* const htree_group = - &htree_groups[(mapping == NULL) ? i : mapping[i]]; + &(*htree_groups)[(mapping == NULL) ? i : mapping[i]]; HuffmanCode** const htrees = htree_group->htrees; int size; int total_size = 0; @@ -468,19 +501,20 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, int max_bits = 0; for (j = 0; j < HUFFMAN_CODES_PER_META_CODE; ++j) { int alphabet_size = kAlphabetSize[j]; - htrees[j] = huffman_table; if (j == 0 && color_cache_bits > 0) { alphabet_size += (1 << color_cache_bits); } - size = ReadHuffmanCode(alphabet_size, dec, code_lengths, huffman_table); + size = + ReadHuffmanCode(alphabet_size, dec, code_lengths, huffman_tables); + htrees[j] = huffman_tables->curr_segment->curr_table; if (size == 0) { goto Error; } if (is_trivial_literal && kLiteralMap[j] == 1) { - is_trivial_literal = (huffman_table->bits == 0); + is_trivial_literal = (htrees[j]->bits == 0); } - total_size += huffman_table->bits; - huffman_table += size; + total_size += htrees[j]->bits; + huffman_tables->curr_segment->curr_table += size; if (j <= ALPHA) { int local_max_bits = code_lengths[0]; int k; @@ -511,19 +545,12 @@ static int ReadHuffmanCodes(VP8LDecoder* const dec, int xsize, int ysize, } ok = 1; - // All OK. Finalize pointers. - hdr->huffman_image_ = huffman_image; - hdr->num_htree_groups_ = num_htree_groups; - hdr->htree_groups_ = htree_groups; - hdr->huffman_tables_ = huffman_tables; - Error: WebPSafeFree(code_lengths); - WebPSafeFree(mapping); if (!ok) { - WebPSafeFree(huffman_image); - WebPSafeFree(huffman_tables); - VP8LHtreeGroupsFree(htree_groups); + VP8LHuffmanTablesDeallocate(huffman_tables); + VP8LHtreeGroupsFree(*htree_groups); + *htree_groups = NULL; } return ok; } @@ -547,8 +574,7 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) { scaled_data_size * sizeof(*scaled_data); uint8_t* memory = (uint8_t*)WebPSafeMalloc(memory_size, sizeof(*memory)); if (memory == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - return 0; + return VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); } assert(dec->rescaler_memory == NULL); dec->rescaler_memory = memory; @@ -559,8 +585,11 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) { memory += work_size * sizeof(*work); scaled_data = (uint32_t*)memory; - WebPRescalerInit(dec->rescaler, in_width, in_height, (uint8_t*)scaled_data, - out_width, out_height, 0, num_channels, work); + if (!WebPRescalerInit(dec->rescaler, in_width, in_height, + (uint8_t*)scaled_data, out_width, out_height, + 0, num_channels, work)) { + return 0; + } return 1; } #endif // WEBP_REDUCE_SIZE @@ -574,13 +603,14 @@ static int AllocateAndInitRescaler(VP8LDecoder* const dec, VP8Io* const io) { static int Export(WebPRescaler* const rescaler, WEBP_CSP_MODE colorspace, int rgba_stride, uint8_t* const rgba) { uint32_t* const src = (uint32_t*)rescaler->dst; + uint8_t* dst = rgba; const int dst_width = rescaler->dst_width; int num_lines_out = 0; while (WebPRescalerHasPendingOutput(rescaler)) { - uint8_t* const dst = rgba + num_lines_out * rgba_stride; WebPRescalerExportRow(rescaler); WebPMultARGBRow(src, dst_width, 1); VP8LConvertFromBGRA(src, dst_width, colorspace, dst); + dst += rgba_stride; ++num_lines_out; } return num_lines_out; @@ -594,8 +624,8 @@ static int EmitRescaledRowsRGBA(const VP8LDecoder* const dec, int num_lines_in = 0; int num_lines_out = 0; while (num_lines_in < mb_h) { - uint8_t* const row_in = in + num_lines_in * in_stride; - uint8_t* const row_out = out + num_lines_out * out_stride; + uint8_t* const row_in = in + (uint64_t)num_lines_in * in_stride; + uint8_t* const row_out = out + (uint64_t)num_lines_out * out_stride; const int lines_left = mb_h - num_lines_in; const int needed_lines = WebPRescaleNeededLines(dec->rescaler, lines_left); int lines_imported; @@ -796,7 +826,8 @@ static void ProcessRows(VP8LDecoder* const dec, int row) { const WebPDecBuffer* const output = dec->output_; if (WebPIsRGBMode(output->colorspace)) { // convert to RGBA const WebPRGBABuffer* const buf = &output->u.RGBA; - uint8_t* const rgba = buf->rgba + dec->last_out_row_ * buf->stride; + uint8_t* const rgba = + buf->rgba + (int64_t)dec->last_out_row_ * buf->stride; const int num_rows_out = #if !defined(WEBP_REDUCE_SIZE) io->use_scaling ? @@ -1077,12 +1108,10 @@ static int DecodeAlphaData(VP8LDecoder* const dec, uint8_t* const data, End: br->eos_ = VP8LIsEndOfStream(br); if (!ok || (br->eos_ && pos < end)) { - ok = 0; - dec->status_ = br->eos_ ? VP8_STATUS_SUSPENDED - : VP8_STATUS_BITSTREAM_ERROR; - } else { - dec->last_pixel_ = pos; + return VP8LSetError( + dec, br->eos_ ? VP8_STATUS_SUSPENDED : VP8_STATUS_BITSTREAM_ERROR); } + dec->last_pixel_ = pos; return ok; } @@ -1232,9 +1261,20 @@ static int DecodeImageData(VP8LDecoder* const dec, uint32_t* const data, } br->eos_ = VP8LIsEndOfStream(br); - if (dec->incremental_ && br->eos_ && src < src_end) { + // In incremental decoding: + // br->eos_ && src < src_last: if 'br' reached the end of the buffer and + // 'src_last' has not been reached yet, there is not enough data. 'dec' has to + // be reset until there is more data. + // !br->eos_ && src < src_last: this cannot happen as either the buffer is + // fully read, either enough has been read to reach 'src_last'. + // src >= src_last: 'src_last' is reached, all is fine. 'src' can actually go + // beyond 'src_last' in case the image is cropped and an LZ77 goes further. + // The buffer might have been enough or there is some left. 'br->eos_' does + // not matter. + assert(!dec->incremental_ || (br->eos_ && src < src_last) || src >= src_last); + if (dec->incremental_ && br->eos_ && src < src_last) { RestoreState(dec); - } else if (!br->eos_) { + } else if ((dec->incremental_ && src >= src_last) || !br->eos_) { // Process the remaining rows corresponding to last row-block. if (process_func != NULL) { process_func(dec, row > last_row ? last_row : row); @@ -1249,8 +1289,7 @@ static int DecodeImageData(VP8LDecoder* const dec, uint32_t* const data, return 1; Error: - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; - return 0; + return VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); } // ----------------------------------------------------------------------------- @@ -1276,7 +1315,7 @@ static int ExpandColorMap(int num_colors, VP8LTransform* const transform) { uint8_t* const new_data = (uint8_t*)new_color_map; new_color_map[0] = transform->data_[0]; for (i = 4; i < 4 * num_colors; ++i) { - // Equivalent to AddPixelEq(), on a byte-basis. + // Equivalent to VP8LAddPixels(), on a byte-basis. new_data[i] = (data[i] + new_data[i - 4]) & 0xff; } for (; i < 4 * final_num_colors; ++i) { @@ -1317,7 +1356,7 @@ static int ReadTransform(int* const xsize, int const* ysize, transform->bits_), VP8LSubSampleSize(transform->ysize_, transform->bits_), - 0, dec, &transform->data_); + /*is_level0=*/0, dec, &transform->data_); break; case COLOR_INDEXING_TRANSFORM: { const int num_colors = VP8LReadBits(br, 8) + 1; @@ -1327,11 +1366,14 @@ static int ReadTransform(int* const xsize, int const* ysize, : 3; *xsize = VP8LSubSampleSize(transform->xsize_, bits); transform->bits_ = bits; - ok = DecodeImageStream(num_colors, 1, 0, dec, &transform->data_); - ok = ok && ExpandColorMap(num_colors, transform); + ok = DecodeImageStream(num_colors, /*ysize=*/1, /*is_level0=*/0, dec, + &transform->data_); + if (ok && !ExpandColorMap(num_colors, transform)) { + return VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); + } break; } - case SUBTRACT_GREEN: + case SUBTRACT_GREEN_TRANSFORM: break; default: assert(0); // can't happen @@ -1353,7 +1395,7 @@ static void ClearMetadata(VP8LMetadata* const hdr) { assert(hdr != NULL); WebPSafeFree(hdr->huffman_image_); - WebPSafeFree(hdr->huffman_tables_); + VP8LHuffmanTablesDeallocate(&hdr->huffman_tables_); VP8LHtreeGroupsFree(hdr->htree_groups_); VP8LColorCacheClear(&hdr->color_cache_); VP8LColorCacheClear(&hdr->saved_color_cache_); @@ -1434,7 +1476,7 @@ static int DecodeImageStream(int xsize, int ysize, color_cache_bits = VP8LReadBits(br, 4); ok = (color_cache_bits >= 1 && color_cache_bits <= MAX_CACHE_BITS); if (!ok) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; + VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); goto End; } } @@ -1443,7 +1485,7 @@ static int DecodeImageStream(int xsize, int ysize, ok = ok && ReadHuffmanCodes(dec, transform_xsize, transform_ysize, color_cache_bits, is_level0); if (!ok) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; + VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); goto End; } @@ -1451,8 +1493,7 @@ static int DecodeImageStream(int xsize, int ysize, if (color_cache_bits > 0) { hdr->color_cache_size_ = 1 << color_cache_bits; if (!VP8LColorCacheInit(&hdr->color_cache_, color_cache_bits)) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - ok = 0; + ok = VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto End; } } else { @@ -1469,8 +1510,7 @@ static int DecodeImageStream(int xsize, int ysize, const uint64_t total_size = (uint64_t)transform_xsize * transform_ysize; data = (uint32_t*)WebPSafeMalloc(total_size, sizeof(*data)); if (data == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - ok = 0; + ok = VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto End; } } @@ -1514,9 +1554,8 @@ static int AllocateInternalBuffers32b(VP8LDecoder* const dec, int final_width) { assert(dec->width_ <= final_width); dec->pixels_ = (uint32_t*)WebPSafeMalloc(total_num_pixels, sizeof(uint32_t)); if (dec->pixels_ == NULL) { - dec->argb_cache_ = NULL; // for sanity check - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - return 0; + dec->argb_cache_ = NULL; // for soundness + return VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); } dec->argb_cache_ = dec->pixels_ + num_pixels + cache_top_pixels; return 1; @@ -1524,11 +1563,10 @@ static int AllocateInternalBuffers32b(VP8LDecoder* const dec, int final_width) { static int AllocateInternalBuffers8b(VP8LDecoder* const dec) { const uint64_t total_num_pixels = (uint64_t)dec->width_ * dec->height_; - dec->argb_cache_ = NULL; // for sanity check + dec->argb_cache_ = NULL; // for soundness dec->pixels_ = (uint32_t*)WebPSafeMalloc(total_num_pixels, sizeof(uint8_t)); if (dec->pixels_ == NULL) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; - return 0; + return VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); } return 1; } @@ -1583,7 +1621,8 @@ int VP8LDecodeAlphaHeader(ALPHDecoder* const alph_dec, dec->status_ = VP8_STATUS_OK; VP8LInitBitReader(&dec->br_, data, data_size); - if (!DecodeImageStream(alph_dec->width_, alph_dec->height_, 1, dec, NULL)) { + if (!DecodeImageStream(alph_dec->width_, alph_dec->height_, /*is_level0=*/1, + dec, /*decoded_data=*/NULL)) { goto Err; } @@ -1638,22 +1677,24 @@ int VP8LDecodeHeader(VP8LDecoder* const dec, VP8Io* const io) { if (dec == NULL) return 0; if (io == NULL) { - dec->status_ = VP8_STATUS_INVALID_PARAM; - return 0; + return VP8LSetError(dec, VP8_STATUS_INVALID_PARAM); } dec->io_ = io; dec->status_ = VP8_STATUS_OK; VP8LInitBitReader(&dec->br_, io->data, io->data_size); if (!ReadImageInfo(&dec->br_, &width, &height, &has_alpha)) { - dec->status_ = VP8_STATUS_BITSTREAM_ERROR; + VP8LSetError(dec, VP8_STATUS_BITSTREAM_ERROR); goto Error; } dec->state_ = READ_DIM; io->width = width; io->height = height; - if (!DecodeImageStream(width, height, 1, dec, NULL)) goto Error; + if (!DecodeImageStream(width, height, /*is_level0=*/1, dec, + /*decoded_data=*/NULL)) { + goto Error; + } return 1; Error: @@ -1666,10 +1707,9 @@ int VP8LDecodeImage(VP8LDecoder* const dec) { VP8Io* io = NULL; WebPDecParams* params = NULL; - // Sanity checks. if (dec == NULL) return 0; - assert(dec->hdr_.huffman_tables_ != NULL); + assert(dec->hdr_.huffman_tables_.root.start != NULL); assert(dec->hdr_.htree_groups_ != NULL); assert(dec->hdr_.num_htree_groups_ > 0); @@ -1684,7 +1724,7 @@ int VP8LDecodeImage(VP8LDecoder* const dec) { assert(dec->output_ != NULL); if (!WebPIoInitFromOptions(params->options, io, MODE_BGRA)) { - dec->status_ = VP8_STATUS_INVALID_PARAM; + VP8LSetError(dec, VP8_STATUS_INVALID_PARAM); goto Err; } @@ -1694,7 +1734,7 @@ int VP8LDecodeImage(VP8LDecoder* const dec) { if (io->use_scaling && !AllocateAndInitRescaler(dec, io)) goto Err; #else if (io->use_scaling) { - dec->status_ = VP8_STATUS_INVALID_PARAM; + VP8LSetError(dec, VP8_STATUS_INVALID_PARAM); goto Err; } #endif @@ -1712,7 +1752,7 @@ int VP8LDecodeImage(VP8LDecoder* const dec) { dec->hdr_.saved_color_cache_.colors_ == NULL) { if (!VP8LColorCacheInit(&dec->hdr_.saved_color_cache_, dec->hdr_.color_cache_.hash_bits_)) { - dec->status_ = VP8_STATUS_OUT_OF_MEMORY; + VP8LSetError(dec, VP8_STATUS_OUT_OF_MEMORY); goto Err; } } diff --git a/3rdparty/libwebp/src/dec/vp8li_dec.h b/3rdparty/libwebp/src/dec/vp8li_dec.h index 72b2e86120..b057573f6c 100644 --- a/3rdparty/libwebp/src/dec/vp8li_dec.h +++ b/3rdparty/libwebp/src/dec/vp8li_dec.h @@ -51,7 +51,7 @@ typedef struct { uint32_t* huffman_image_; int num_htree_groups_; HTreeGroup* htree_groups_; - HuffmanCode* huffman_tables_; + HuffmanTables huffman_tables_; } VP8LMetadata; typedef struct VP8LDecoder VP8LDecoder; @@ -126,6 +126,19 @@ void VP8LClear(VP8LDecoder* const dec); // Clears and deallocate a lossless decoder instance. void VP8LDelete(VP8LDecoder* const dec); +// Helper function for reading the different Huffman codes and storing them in +// 'huffman_tables' and 'htree_groups'. +// If mapping is NULL 'num_htree_groups_max' must equal 'num_htree_groups'. +// If it is not NULL, it maps 'num_htree_groups_max' indices to the +// 'num_htree_groups' groups. If 'num_htree_groups_max' > 'num_htree_groups', +// some of those indices map to -1. This is used for non-balanced codes to +// limit memory usage. +int ReadHuffmanCodesHelper(int color_cache_bits, int num_htree_groups, + int num_htree_groups_max, const int* const mapping, + VP8LDecoder* const dec, + HuffmanTables* const huffman_tables, + HTreeGroup** const htree_groups); + //------------------------------------------------------------------------------ #ifdef __cplusplus diff --git a/3rdparty/libwebp/src/dec/webp_dec.c b/3rdparty/libwebp/src/dec/webp_dec.c index 42d098874d..f557868b99 100644 --- a/3rdparty/libwebp/src/dec/webp_dec.c +++ b/3rdparty/libwebp/src/dec/webp_dec.c @@ -179,7 +179,7 @@ static VP8StatusCode ParseOptionalChunks(const uint8_t** const data, return VP8_STATUS_BITSTREAM_ERROR; // Not a valid chunk size. } // For odd-sized chunk-payload, there's one byte padding at the end. - disk_chunk_size = (CHUNK_HEADER_SIZE + chunk_size + 1) & ~1; + disk_chunk_size = (CHUNK_HEADER_SIZE + chunk_size + 1) & ~1u; total_size += disk_chunk_size; // Check that total bytes skipped so far does not exceed riff_size. @@ -658,19 +658,26 @@ uint8_t* WebPDecodeBGRA(const uint8_t* data, size_t data_size, uint8_t* WebPDecodeYUV(const uint8_t* data, size_t data_size, int* width, int* height, uint8_t** u, uint8_t** v, int* stride, int* uv_stride) { - WebPDecBuffer output; // only to preserve the side-infos - uint8_t* const out = Decode(MODE_YUV, data, data_size, - width, height, &output); - - if (out != NULL) { - const WebPYUVABuffer* const buf = &output.u.YUVA; - *u = buf->u; - *v = buf->v; - *stride = buf->y_stride; - *uv_stride = buf->u_stride; - assert(buf->u_stride == buf->v_stride); + // data, width and height are checked by Decode(). + if (u == NULL || v == NULL || stride == NULL || uv_stride == NULL) { + return NULL; + } + + { + WebPDecBuffer output; // only to preserve the side-infos + uint8_t* const out = Decode(MODE_YUV, data, data_size, + width, height, &output); + + if (out != NULL) { + const WebPYUVABuffer* const buf = &output.u.YUVA; + *u = buf->u; + *v = buf->v; + *stride = buf->y_stride; + *uv_stride = buf->u_stride; + assert(buf->u_stride == buf->v_stride); + } + return out; } - return out; } static void DefaultFeatures(WebPBitstreamFeatures* const features) { @@ -785,6 +792,13 @@ VP8StatusCode WebPDecode(const uint8_t* data, size_t data_size, //------------------------------------------------------------------------------ // Cropping and rescaling. +int WebPCheckCropDimensions(int image_width, int image_height, + int x, int y, int w, int h) { + return !(x < 0 || y < 0 || w <= 0 || h <= 0 || + x >= image_width || w > image_width || w > image_width - x || + y >= image_height || h > image_height || h > image_height - y); +} + int WebPIoInitFromOptions(const WebPDecoderOptions* const options, VP8Io* const io, WEBP_CSP_MODE src_colorspace) { const int W = io->width; @@ -792,7 +806,7 @@ int WebPIoInitFromOptions(const WebPDecoderOptions* const options, int x = 0, y = 0, w = W, h = H; // Cropping - io->use_cropping = (options != NULL) && (options->use_cropping > 0); + io->use_cropping = (options != NULL) && options->use_cropping; if (io->use_cropping) { w = options->crop_width; h = options->crop_height; @@ -802,7 +816,7 @@ int WebPIoInitFromOptions(const WebPDecoderOptions* const options, x &= ~1; y &= ~1; } - if (x < 0 || y < 0 || w <= 0 || h <= 0 || x + w > W || y + h > H) { + if (!WebPCheckCropDimensions(W, H, x, y, w, h)) { return 0; // out of frame boundary error } } @@ -814,7 +828,7 @@ int WebPIoInitFromOptions(const WebPDecoderOptions* const options, io->mb_h = h; // Scaling - io->use_scaling = (options != NULL) && (options->use_scaling > 0); + io->use_scaling = (options != NULL) && options->use_scaling; if (io->use_scaling) { int scaled_width = options->scaled_width; int scaled_height = options->scaled_height; @@ -835,8 +849,8 @@ int WebPIoInitFromOptions(const WebPDecoderOptions* const options, if (io->use_scaling) { // disable filter (only for large downscaling ratio). - io->bypass_filtering = (io->scaled_width < W * 3 / 4) && - (io->scaled_height < H * 3 / 4); + io->bypass_filtering |= (io->scaled_width < W * 3 / 4) && + (io->scaled_height < H * 3 / 4); io->fancy_upsampling = 0; } return 1; diff --git a/3rdparty/libwebp/src/dec/webpi_dec.h b/3rdparty/libwebp/src/dec/webpi_dec.h index 24baff5d27..3b97388c71 100644 --- a/3rdparty/libwebp/src/dec/webpi_dec.h +++ b/3rdparty/libwebp/src/dec/webpi_dec.h @@ -77,6 +77,10 @@ VP8StatusCode WebPParseHeaders(WebPHeaderStructure* const headers); //------------------------------------------------------------------------------ // Misc utils +// Returns true if crop dimensions are within image bounds. +int WebPCheckCropDimensions(int image_width, int image_height, + int x, int y, int w, int h); + // Initializes VP8Io with custom setup, io and teardown functions. The default // hooks will use the supplied 'params' as io->opaque handle. void WebPInitCustomIo(WebPDecParams* const params, VP8Io* const io); diff --git a/3rdparty/libwebp/src/demux/anim_decode.c b/3rdparty/libwebp/src/demux/anim_decode.c index 3dcacc35d6..e077ffb536 100644 --- a/3rdparty/libwebp/src/demux/anim_decode.c +++ b/3rdparty/libwebp/src/demux/anim_decode.c @@ -23,6 +23,14 @@ #define NUM_CHANNELS 4 +// Channel extraction from a uint32_t representation of a uint8_t RGBA/BGRA +// buffer. +#ifdef WORDS_BIGENDIAN +#define CHANNEL_SHIFT(i) (24 - (i) * 8) +#else +#define CHANNEL_SHIFT(i) ((i) * 8) +#endif + typedef void (*BlendRowFunc)(uint32_t* const, const uint32_t* const, int); static void BlendPixelRowNonPremult(uint32_t* const src, const uint32_t* const dst, int num_pixels); @@ -87,11 +95,19 @@ WebPAnimDecoder* WebPAnimDecoderNewInternal( int abi_version) { WebPAnimDecoderOptions options; WebPAnimDecoder* dec = NULL; + WebPBitstreamFeatures features; if (webp_data == NULL || WEBP_ABI_IS_INCOMPATIBLE(abi_version, WEBP_DEMUX_ABI_VERSION)) { return NULL; } + // Validate the bitstream before doing expensive allocations. The demuxer may + // be more tolerant than the decoder. + if (WebPGetFeatures(webp_data->bytes, webp_data->size, &features) != + VP8_STATUS_OK) { + return NULL; + } + // Note: calloc() so that the pointer members are initialized to NULL. dec = (WebPAnimDecoder*)WebPSafeCalloc(1ULL, sizeof(*dec)); if (dec == NULL) goto Error; @@ -145,7 +161,7 @@ static int ZeroFillCanvas(uint8_t* buf, uint32_t canvas_width, uint32_t canvas_height) { const uint64_t size = (uint64_t)canvas_width * canvas_height * NUM_CHANNELS * sizeof(*buf); - if (size != (size_t)size) return 0; + if (!CheckSizeOverflow(size)) return 0; memset(buf, 0, (size_t)size); return 1; } @@ -166,7 +182,7 @@ static void ZeroFillFrameRect(uint8_t* buf, int buf_stride, int x_offset, static int CopyCanvas(const uint8_t* src, uint8_t* dst, uint32_t width, uint32_t height) { const uint64_t size = (uint64_t)width * height * NUM_CHANNELS; - if (size != (size_t)size) return 0; + if (!CheckSizeOverflow(size)) return 0; assert(src != NULL && dst != NULL); memcpy(dst, src, (size_t)size); return 1; @@ -201,35 +217,35 @@ static uint8_t BlendChannelNonPremult(uint32_t src, uint8_t src_a, const uint8_t dst_channel = (dst >> shift) & 0xff; const uint32_t blend_unscaled = src_channel * src_a + dst_channel * dst_a; assert(blend_unscaled < (1ULL << 32) / scale); - return (blend_unscaled * scale) >> 24; + return (blend_unscaled * scale) >> CHANNEL_SHIFT(3); } // Blend 'src' over 'dst' assuming they are NOT pre-multiplied by alpha. static uint32_t BlendPixelNonPremult(uint32_t src, uint32_t dst) { - const uint8_t src_a = (src >> 24) & 0xff; + const uint8_t src_a = (src >> CHANNEL_SHIFT(3)) & 0xff; if (src_a == 0) { return dst; } else { - const uint8_t dst_a = (dst >> 24) & 0xff; + const uint8_t dst_a = (dst >> CHANNEL_SHIFT(3)) & 0xff; // This is the approximate integer arithmetic for the actual formula: // dst_factor_a = (dst_a * (255 - src_a)) / 255. const uint8_t dst_factor_a = (dst_a * (256 - src_a)) >> 8; const uint8_t blend_a = src_a + dst_factor_a; const uint32_t scale = (1UL << 24) / blend_a; - const uint8_t blend_r = - BlendChannelNonPremult(src, src_a, dst, dst_factor_a, scale, 0); - const uint8_t blend_g = - BlendChannelNonPremult(src, src_a, dst, dst_factor_a, scale, 8); - const uint8_t blend_b = - BlendChannelNonPremult(src, src_a, dst, dst_factor_a, scale, 16); + const uint8_t blend_r = BlendChannelNonPremult( + src, src_a, dst, dst_factor_a, scale, CHANNEL_SHIFT(0)); + const uint8_t blend_g = BlendChannelNonPremult( + src, src_a, dst, dst_factor_a, scale, CHANNEL_SHIFT(1)); + const uint8_t blend_b = BlendChannelNonPremult( + src, src_a, dst, dst_factor_a, scale, CHANNEL_SHIFT(2)); assert(src_a + dst_factor_a < 256); - return (blend_r << 0) | - (blend_g << 8) | - (blend_b << 16) | - ((uint32_t)blend_a << 24); + return ((uint32_t)blend_r << CHANNEL_SHIFT(0)) | + ((uint32_t)blend_g << CHANNEL_SHIFT(1)) | + ((uint32_t)blend_b << CHANNEL_SHIFT(2)) | + ((uint32_t)blend_a << CHANNEL_SHIFT(3)); } } @@ -239,7 +255,7 @@ static void BlendPixelRowNonPremult(uint32_t* const src, const uint32_t* const dst, int num_pixels) { int i; for (i = 0; i < num_pixels; ++i) { - const uint8_t src_alpha = (src[i] >> 24) & 0xff; + const uint8_t src_alpha = (src[i] >> CHANNEL_SHIFT(3)) & 0xff; if (src_alpha != 0xff) { src[i] = BlendPixelNonPremult(src[i], dst[i]); } @@ -256,7 +272,7 @@ static WEBP_INLINE uint32_t ChannelwiseMultiply(uint32_t pix, uint32_t scale) { // Blend 'src' over 'dst' assuming they are pre-multiplied by alpha. static uint32_t BlendPixelPremult(uint32_t src, uint32_t dst) { - const uint8_t src_a = (src >> 24) & 0xff; + const uint8_t src_a = (src >> CHANNEL_SHIFT(3)) & 0xff; return src + ChannelwiseMultiply(dst, 256 - src_a); } @@ -266,7 +282,7 @@ static void BlendPixelRowPremult(uint32_t* const src, const uint32_t* const dst, int num_pixels) { int i; for (i = 0; i < num_pixels; ++i) { - const uint8_t src_alpha = (src[i] >> 24) & 0xff; + const uint8_t src_alpha = (src[i] >> CHANNEL_SHIFT(3)) & 0xff; if (src_alpha != 0xff) { src[i] = BlendPixelPremult(src[i], dst[i]); } diff --git a/3rdparty/libwebp/src/demux/demux.c b/3rdparty/libwebp/src/demux/demux.c index 860e2ce761..fd45a2500e 100644 --- a/3rdparty/libwebp/src/demux/demux.c +++ b/3rdparty/libwebp/src/demux/demux.c @@ -24,8 +24,8 @@ #include "src/webp/format_constants.h" #define DMUX_MAJ_VERSION 1 -#define DMUX_MIN_VERSION 2 -#define DMUX_REV_VERSION 0 +#define DMUX_MIN_VERSION 3 +#define DMUX_REV_VERSION 1 typedef struct { size_t start_; // start location of the data @@ -221,12 +221,16 @@ static ParseStatus StoreFrame(int frame_num, uint32_t min_size, const size_t chunk_start_offset = mem->start_; const uint32_t fourcc = ReadLE32(mem); const uint32_t payload_size = ReadLE32(mem); - const uint32_t payload_size_padded = payload_size + (payload_size & 1); - const size_t payload_available = (payload_size_padded > MemDataSize(mem)) - ? MemDataSize(mem) : payload_size_padded; - const size_t chunk_size = CHUNK_HEADER_SIZE + payload_available; + uint32_t payload_size_padded; + size_t payload_available; + size_t chunk_size; if (payload_size > MAX_CHUNK_PAYLOAD) return PARSE_ERROR; + + payload_size_padded = payload_size + (payload_size & 1); + payload_available = (payload_size_padded > MemDataSize(mem)) + ? MemDataSize(mem) : payload_size_padded; + chunk_size = CHUNK_HEADER_SIZE + payload_available; if (SizeIsInvalid(mem, payload_size_padded)) return PARSE_ERROR; if (payload_size_padded > MemDataSize(mem)) status = PARSE_NEED_MORE_DATA; @@ -451,9 +455,11 @@ static ParseStatus ParseVP8XChunks(WebPDemuxer* const dmux) { const size_t chunk_start_offset = mem->start_; const uint32_t fourcc = ReadLE32(mem); const uint32_t chunk_size = ReadLE32(mem); - const uint32_t chunk_size_padded = chunk_size + (chunk_size & 1); + uint32_t chunk_size_padded; if (chunk_size > MAX_CHUNK_PAYLOAD) return PARSE_ERROR; + + chunk_size_padded = chunk_size + (chunk_size & 1); if (SizeIsInvalid(mem, chunk_size_padded)) return PARSE_ERROR; switch (fourcc) { @@ -608,7 +614,6 @@ static int IsValidExtendedFormat(const WebPDemuxer* const dmux) { while (f != NULL) { const int cur_frame_set = f->frame_num_; - int frame_count = 0; // Check frame properties. for (; f != NULL && f->frame_num_ == cur_frame_set; f = f->next_) { @@ -643,8 +648,6 @@ static int IsValidExtendedFormat(const WebPDemuxer* const dmux) { dmux->canvas_width_, dmux->canvas_height_)) { return 0; } - - ++frame_count; } } return 1; diff --git a/3rdparty/libwebp/src/dsp/alpha_processing.c b/3rdparty/libwebp/src/dsp/alpha_processing.c index 3a27990ddc..1d152f24da 100644 --- a/3rdparty/libwebp/src/dsp/alpha_processing.c +++ b/3rdparty/libwebp/src/dsp/alpha_processing.c @@ -157,7 +157,8 @@ void WebPMultARGBRow_C(uint32_t* const ptr, int width, int inverse) { } } -void WebPMultRow_C(uint8_t* const ptr, const uint8_t* const alpha, +void WebPMultRow_C(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse) { int x; for (x = 0; x < width; ++x) { @@ -178,7 +179,8 @@ void WebPMultRow_C(uint8_t* const ptr, const uint8_t* const alpha, #undef MFIX void (*WebPMultARGBRow)(uint32_t* const ptr, int width, int inverse); -void (*WebPMultRow)(uint8_t* const ptr, const uint8_t* const alpha, +void (*WebPMultRow)(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse); //------------------------------------------------------------------------------ @@ -193,8 +195,8 @@ void WebPMultARGBRows(uint8_t* ptr, int stride, int width, int num_rows, } } -void WebPMultRows(uint8_t* ptr, int stride, - const uint8_t* alpha, int alpha_stride, +void WebPMultRows(uint8_t* WEBP_RESTRICT ptr, int stride, + const uint8_t* WEBP_RESTRICT alpha, int alpha_stride, int width, int num_rows, int inverse) { int n; for (n = 0; n < num_rows; ++n) { @@ -290,9 +292,9 @@ static void ApplyAlphaMultiply_16b_C(uint8_t* rgba4444, } #if !WEBP_NEON_OMIT_C_CODE -static int DispatchAlpha_C(const uint8_t* alpha, int alpha_stride, +static int DispatchAlpha_C(const uint8_t* WEBP_RESTRICT alpha, int alpha_stride, int width, int height, - uint8_t* dst, int dst_stride) { + uint8_t* WEBP_RESTRICT dst, int dst_stride) { uint32_t alpha_mask = 0xff; int i, j; @@ -309,9 +311,10 @@ static int DispatchAlpha_C(const uint8_t* alpha, int alpha_stride, return (alpha_mask != 0xff); } -static void DispatchAlphaToGreen_C(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride) { +static void DispatchAlphaToGreen_C(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint32_t* WEBP_RESTRICT dst, + int dst_stride) { int i, j; for (j = 0; j < height; ++j) { for (i = 0; i < width; ++i) { @@ -322,9 +325,9 @@ static void DispatchAlphaToGreen_C(const uint8_t* alpha, int alpha_stride, } } -static int ExtractAlpha_C(const uint8_t* argb, int argb_stride, +static int ExtractAlpha_C(const uint8_t* WEBP_RESTRICT argb, int argb_stride, int width, int height, - uint8_t* alpha, int alpha_stride) { + uint8_t* WEBP_RESTRICT alpha, int alpha_stride) { uint8_t alpha_mask = 0xff; int i, j; @@ -340,7 +343,8 @@ static int ExtractAlpha_C(const uint8_t* argb, int argb_stride, return (alpha_mask == 0xff); } -static void ExtractGreen_C(const uint32_t* argb, uint8_t* alpha, int size) { +static void ExtractGreen_C(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size) { int i; for (i = 0; i < size; ++i) alpha[i] = argb[i] >> 8; } @@ -372,8 +376,11 @@ static WEBP_INLINE uint32_t MakeARGB32(int a, int r, int g, int b) { } #ifdef WORDS_BIGENDIAN -static void PackARGB_C(const uint8_t* a, const uint8_t* r, const uint8_t* g, - const uint8_t* b, int len, uint32_t* out) { +static void PackARGB_C(const uint8_t* WEBP_RESTRICT a, + const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, uint32_t* WEBP_RESTRICT out) { int i; for (i = 0; i < len; ++i) { out[i] = MakeARGB32(a[4 * i], r[4 * i], g[4 * i], b[4 * i]); @@ -381,8 +388,10 @@ static void PackARGB_C(const uint8_t* a, const uint8_t* r, const uint8_t* g, } #endif -static void PackRGB_C(const uint8_t* r, const uint8_t* g, const uint8_t* b, - int len, int step, uint32_t* out) { +static void PackRGB_C(const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, int step, uint32_t* WEBP_RESTRICT out) { int i, offset = 0; for (i = 0; i < len; ++i) { out[i] = MakeARGB32(0xff, r[offset], g[offset], b[offset]); @@ -392,16 +401,22 @@ static void PackRGB_C(const uint8_t* r, const uint8_t* g, const uint8_t* b, void (*WebPApplyAlphaMultiply)(uint8_t*, int, int, int, int); void (*WebPApplyAlphaMultiply4444)(uint8_t*, int, int, int); -int (*WebPDispatchAlpha)(const uint8_t*, int, int, int, uint8_t*, int); -void (*WebPDispatchAlphaToGreen)(const uint8_t*, int, int, int, uint32_t*, int); -int (*WebPExtractAlpha)(const uint8_t*, int, int, int, uint8_t*, int); -void (*WebPExtractGreen)(const uint32_t* argb, uint8_t* alpha, int size); +int (*WebPDispatchAlpha)(const uint8_t* WEBP_RESTRICT, int, int, int, + uint8_t* WEBP_RESTRICT, int); +void (*WebPDispatchAlphaToGreen)(const uint8_t* WEBP_RESTRICT, int, int, int, + uint32_t* WEBP_RESTRICT, int); +int (*WebPExtractAlpha)(const uint8_t* WEBP_RESTRICT, int, int, int, + uint8_t* WEBP_RESTRICT, int); +void (*WebPExtractGreen)(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size); #ifdef WORDS_BIGENDIAN void (*WebPPackARGB)(const uint8_t* a, const uint8_t* r, const uint8_t* g, const uint8_t* b, int, uint32_t*); #endif -void (*WebPPackRGB)(const uint8_t* r, const uint8_t* g, const uint8_t* b, - int len, int step, uint32_t* out); +void (*WebPPackRGB)(const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, int step, uint32_t* WEBP_RESTRICT out); int (*WebPHasAlpha8b)(const uint8_t* src, int length); int (*WebPHasAlpha32b)(const uint8_t* src, int length); @@ -410,6 +425,7 @@ void (*WebPAlphaReplace)(uint32_t* src, int length, uint32_t color); //------------------------------------------------------------------------------ // Init function +extern VP8CPUInfo VP8GetCPUInfo; extern void WebPInitAlphaProcessingMIPSdspR2(void); extern void WebPInitAlphaProcessingSSE2(void); extern void WebPInitAlphaProcessingSSE41(void); @@ -438,10 +454,10 @@ WEBP_DSP_INIT_FUNC(WebPInitAlphaProcessing) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitAlphaProcessingSSE2(); -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitAlphaProcessingSSE41(); } @@ -455,7 +471,7 @@ WEBP_DSP_INIT_FUNC(WebPInitAlphaProcessing) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { WebPInitAlphaProcessingNEON(); diff --git a/3rdparty/libwebp/src/dsp/alpha_processing_neon.c b/3rdparty/libwebp/src/dsp/alpha_processing_neon.c index 9d55421704..6716fb77f0 100644 --- a/3rdparty/libwebp/src/dsp/alpha_processing_neon.c +++ b/3rdparty/libwebp/src/dsp/alpha_processing_neon.c @@ -80,10 +80,10 @@ static void ApplyAlphaMultiply_NEON(uint8_t* rgba, int alpha_first, //------------------------------------------------------------------------------ -static int DispatchAlpha_NEON(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint8_t* dst, int dst_stride) { - uint32_t alpha_mask = 0xffffffffu; +static int DispatchAlpha_NEON(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint8_t* WEBP_RESTRICT dst, int dst_stride) { + uint32_t alpha_mask = 0xffu; uint8x8_t mask8 = vdup_n_u8(0xff); uint32_t tmp[2]; int i, j; @@ -107,14 +107,16 @@ static int DispatchAlpha_NEON(const uint8_t* alpha, int alpha_stride, dst += dst_stride; } vst1_u8((uint8_t*)tmp, mask8); + alpha_mask *= 0x01010101; alpha_mask &= tmp[0]; alpha_mask &= tmp[1]; return (alpha_mask != 0xffffffffu); } -static void DispatchAlphaToGreen_NEON(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride) { +static void DispatchAlphaToGreen_NEON(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint32_t* WEBP_RESTRICT dst, + int dst_stride) { int i, j; uint8x8x4_t greens; // leave A/R/B channels zero'd. greens.val[0] = vdup_n_u8(0); @@ -131,10 +133,10 @@ static void DispatchAlphaToGreen_NEON(const uint8_t* alpha, int alpha_stride, } } -static int ExtractAlpha_NEON(const uint8_t* argb, int argb_stride, +static int ExtractAlpha_NEON(const uint8_t* WEBP_RESTRICT argb, int argb_stride, int width, int height, - uint8_t* alpha, int alpha_stride) { - uint32_t alpha_mask = 0xffffffffu; + uint8_t* WEBP_RESTRICT alpha, int alpha_stride) { + uint32_t alpha_mask = 0xffu; uint8x8_t mask8 = vdup_n_u8(0xff); uint32_t tmp[2]; int i, j; @@ -156,13 +158,14 @@ static int ExtractAlpha_NEON(const uint8_t* argb, int argb_stride, alpha += alpha_stride; } vst1_u8((uint8_t*)tmp, mask8); + alpha_mask *= 0x01010101; alpha_mask &= tmp[0]; alpha_mask &= tmp[1]; return (alpha_mask == 0xffffffffu); } -static void ExtractGreen_NEON(const uint32_t* argb, - uint8_t* alpha, int size) { +static void ExtractGreen_NEON(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size) { int i; for (i = 0; i + 16 <= size; i += 16) { const uint8x16x4_t rgbX = vld4q_u8((const uint8_t*)(argb + i)); diff --git a/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c b/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c index f6c6e0fb1a..aa0cc2848a 100644 --- a/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c +++ b/3rdparty/libwebp/src/dsp/alpha_processing_sse2.c @@ -18,16 +18,16 @@ //------------------------------------------------------------------------------ -static int DispatchAlpha_SSE2(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint8_t* dst, int dst_stride) { +static int DispatchAlpha_SSE2(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint8_t* WEBP_RESTRICT dst, int dst_stride) { // alpha_and stores an 'and' operation of all the alpha[] values. The final // value is not 0xff if any of the alpha[] is not equal to 0xff. uint32_t alpha_and = 0xff; int i, j; const __m128i zero = _mm_setzero_si128(); - const __m128i rgb_mask = _mm_set1_epi32(0xffffff00u); // to preserve RGB - const __m128i all_0xff = _mm_set_epi32(0, 0, ~0u, ~0u); + const __m128i rgb_mask = _mm_set1_epi32((int)0xffffff00); // to preserve RGB + const __m128i all_0xff = _mm_set_epi32(0, 0, ~0, ~0); __m128i all_alphas = all_0xff; // We must be able to access 3 extra bytes after the last written byte @@ -72,9 +72,10 @@ static int DispatchAlpha_SSE2(const uint8_t* alpha, int alpha_stride, return (alpha_and != 0xff); } -static void DispatchAlphaToGreen_SSE2(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride) { +static void DispatchAlphaToGreen_SSE2(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint32_t* WEBP_RESTRICT dst, + int dst_stride) { int i, j; const __m128i zero = _mm_setzero_si128(); const int limit = width & ~15; @@ -98,15 +99,15 @@ static void DispatchAlphaToGreen_SSE2(const uint8_t* alpha, int alpha_stride, } } -static int ExtractAlpha_SSE2(const uint8_t* argb, int argb_stride, +static int ExtractAlpha_SSE2(const uint8_t* WEBP_RESTRICT argb, int argb_stride, int width, int height, - uint8_t* alpha, int alpha_stride) { + uint8_t* WEBP_RESTRICT alpha, int alpha_stride) { // alpha_and stores an 'and' operation of all the alpha[] values. The final // value is not 0xff if any of the alpha[] is not equal to 0xff. uint32_t alpha_and = 0xff; int i, j; - const __m128i a_mask = _mm_set1_epi32(0xffu); // to preserve alpha - const __m128i all_0xff = _mm_set_epi32(0, 0, ~0u, ~0u); + const __m128i a_mask = _mm_set1_epi32(0xff); // to preserve alpha + const __m128i all_0xff = _mm_set_epi32(0, 0, ~0, ~0); __m128i all_alphas = all_0xff; // We must be able to access 3 extra bytes after the last written byte @@ -143,6 +144,46 @@ static int ExtractAlpha_SSE2(const uint8_t* argb, int argb_stride, return (alpha_and == 0xff); } +static void ExtractGreen_SSE2(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size) { + int i; + const __m128i mask = _mm_set1_epi32(0xff); + const __m128i* src = (const __m128i*)argb; + + for (i = 0; i + 16 <= size; i += 16, src += 4) { + const __m128i a0 = _mm_loadu_si128(src + 0); + const __m128i a1 = _mm_loadu_si128(src + 1); + const __m128i a2 = _mm_loadu_si128(src + 2); + const __m128i a3 = _mm_loadu_si128(src + 3); + const __m128i b0 = _mm_srli_epi32(a0, 8); + const __m128i b1 = _mm_srli_epi32(a1, 8); + const __m128i b2 = _mm_srli_epi32(a2, 8); + const __m128i b3 = _mm_srli_epi32(a3, 8); + const __m128i c0 = _mm_and_si128(b0, mask); + const __m128i c1 = _mm_and_si128(b1, mask); + const __m128i c2 = _mm_and_si128(b2, mask); + const __m128i c3 = _mm_and_si128(b3, mask); + const __m128i d0 = _mm_packs_epi32(c0, c1); + const __m128i d1 = _mm_packs_epi32(c2, c3); + const __m128i e = _mm_packus_epi16(d0, d1); + // store + _mm_storeu_si128((__m128i*)&alpha[i], e); + } + if (i + 8 <= size) { + const __m128i a0 = _mm_loadu_si128(src + 0); + const __m128i a1 = _mm_loadu_si128(src + 1); + const __m128i b0 = _mm_srli_epi32(a0, 8); + const __m128i b1 = _mm_srli_epi32(a1, 8); + const __m128i c0 = _mm_and_si128(b0, mask); + const __m128i c1 = _mm_and_si128(b1, mask); + const __m128i d = _mm_packs_epi32(c0, c1); + const __m128i e = _mm_packus_epi16(d, d); + _mm_storel_epi64((__m128i*)&alpha[i], e); + i += 8; + } + for (; i < size; ++i) alpha[i] = argb[i] >> 8; +} + //------------------------------------------------------------------------------ // Non-dither premultiplied modes @@ -177,7 +218,7 @@ static int ExtractAlpha_SSE2(const uint8_t* argb, int argb_stride, static void ApplyAlphaMultiply_SSE2(uint8_t* rgba, int alpha_first, int w, int h, int stride) { const __m128i zero = _mm_setzero_si128(); - const __m128i kMult = _mm_set1_epi16(0x8081u); + const __m128i kMult = _mm_set1_epi16((short)0x8081); const __m128i kMask = _mm_set_epi16(0, 0xff, 0xff, 0, 0, 0xff, 0xff, 0); const int kSpan = 4; while (h-- > 0) { @@ -266,7 +307,7 @@ static int HasAlpha32b_SSE2(const uint8_t* src, int length) { } static void AlphaReplace_SSE2(uint32_t* src, int length, uint32_t color) { - const __m128i m_color = _mm_set1_epi32(color); + const __m128i m_color = _mm_set1_epi32((int)color); const __m128i zero = _mm_setzero_si128(); int i = 0; for (; i + 8 <= length; i += 8) { @@ -317,7 +358,8 @@ static void MultARGBRow_SSE2(uint32_t* const ptr, int width, int inverse) { if (width > 0) WebPMultARGBRow_C(ptr + x, width, inverse); } -static void MultRow_SSE2(uint8_t* const ptr, const uint8_t* const alpha, +static void MultRow_SSE2(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse) { int x = 0; if (!inverse) { @@ -352,6 +394,7 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitAlphaProcessingSSE2(void) { WebPDispatchAlpha = DispatchAlpha_SSE2; WebPDispatchAlphaToGreen = DispatchAlphaToGreen_SSE2; WebPExtractAlpha = ExtractAlpha_SSE2; + WebPExtractGreen = ExtractGreen_SSE2; WebPHasAlpha8b = HasAlpha8b_SSE2; WebPHasAlpha32b = HasAlpha32b_SSE2; diff --git a/3rdparty/libwebp/src/dsp/alpha_processing_sse41.c b/3rdparty/libwebp/src/dsp/alpha_processing_sse41.c index 56040f9c88..1156ac3417 100644 --- a/3rdparty/libwebp/src/dsp/alpha_processing_sse41.c +++ b/3rdparty/libwebp/src/dsp/alpha_processing_sse41.c @@ -19,14 +19,14 @@ //------------------------------------------------------------------------------ -static int ExtractAlpha_SSE41(const uint8_t* argb, int argb_stride, - int width, int height, - uint8_t* alpha, int alpha_stride) { +static int ExtractAlpha_SSE41(const uint8_t* WEBP_RESTRICT argb, + int argb_stride, int width, int height, + uint8_t* WEBP_RESTRICT alpha, int alpha_stride) { // alpha_and stores an 'and' operation of all the alpha[] values. The final // value is not 0xff if any of the alpha[] is not equal to 0xff. uint32_t alpha_and = 0xff; int i, j; - const __m128i all_0xff = _mm_set1_epi32(~0u); + const __m128i all_0xff = _mm_set1_epi32(~0); __m128i all_alphas = all_0xff; // We must be able to access 3 extra bytes after the last written byte diff --git a/3rdparty/libwebp/src/dsp/cost.c b/3rdparty/libwebp/src/dsp/cost.c index cc681cdd4b..73d2140177 100644 --- a/3rdparty/libwebp/src/dsp/cost.c +++ b/3rdparty/libwebp/src/dsp/cost.c @@ -374,6 +374,7 @@ static void SetResidualCoeffs_C(const int16_t* const coeffs, VP8GetResidualCostFunc VP8GetResidualCost; VP8SetResidualCoeffsFunc VP8SetResidualCoeffs; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8EncDspCostInitMIPS32(void); extern void VP8EncDspCostInitMIPSdspR2(void); extern void VP8EncDspCostInitSSE2(void); @@ -395,12 +396,12 @@ WEBP_DSP_INIT_FUNC(VP8EncDspCostInit) { VP8EncDspCostInitMIPSdspR2(); } #endif -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8EncDspCostInitSSE2(); } #endif -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (VP8GetCPUInfo(kNEON)) { VP8EncDspCostInitNEON(); } diff --git a/3rdparty/libwebp/src/dsp/cost_neon.c b/3rdparty/libwebp/src/dsp/cost_neon.c index 8cc8ce58aa..6582669cb3 100644 --- a/3rdparty/libwebp/src/dsp/cost_neon.c +++ b/3rdparty/libwebp/src/dsp/cost_neon.c @@ -29,7 +29,7 @@ static void SetResidualCoeffs_NEON(const int16_t* const coeffs, const uint8x16_t eob = vcombine_u8(vqmovn_u16(eob_0), vqmovn_u16(eob_1)); const uint8x16_t masked = vandq_u8(eob, vld1q_u8(position)); -#ifdef __aarch64__ +#if WEBP_AARCH64 res->last = vmaxvq_u8(masked) - 1; #else const uint8x8_t eob_8x8 = vmax_u8(vget_low_u8(masked), vget_high_u8(masked)); @@ -43,7 +43,7 @@ static void SetResidualCoeffs_NEON(const int16_t* const coeffs, vst1_lane_s32(&res->last, vreinterpret_s32_u32(eob_32x2), 0); --res->last; -#endif // __aarch64__ +#endif // WEBP_AARCH64 res->coeffs = coeffs; } diff --git a/3rdparty/libwebp/src/dsp/cpu.c b/3rdparty/libwebp/src/dsp/cpu.c index 4ca90d88bf..2234c77b35 100644 --- a/3rdparty/libwebp/src/dsp/cpu.c +++ b/3rdparty/libwebp/src/dsp/cpu.c @@ -11,7 +11,7 @@ // // Author: Christian Duvivier (cduvivier@google.com) -#include "src/dsp/dsp.h" +#include "src/dsp/cpu.h" #if defined(WEBP_HAVE_NEON_RTCD) #include @@ -173,6 +173,7 @@ static int x86CPUInfo(CPUFeature feature) { } return 0; } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = x86CPUInfo; #elif defined(WEBP_ANDROID_NEON) // NB: needs to be before generic NEON test. static int AndroidCPUInfo(CPUFeature feature) { @@ -184,22 +185,23 @@ static int AndroidCPUInfo(CPUFeature feature) { } return 0; } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = AndroidCPUInfo; #elif defined(EMSCRIPTEN) // also needs to be before generic NEON test // Use compile flags as an indicator of SIMD support instead of a runtime check. static int wasmCPUInfo(CPUFeature feature) { switch (feature) { -#ifdef WEBP_USE_SSE2 +#ifdef WEBP_HAVE_SSE2 case kSSE2: return 1; #endif -#ifdef WEBP_USE_SSE41 +#ifdef WEBP_HAVE_SSE41 case kSSE3: case kSlowSSSE3: case kSSE4_1: return 1; #endif -#ifdef WEBP_USE_NEON +#ifdef WEBP_HAVE_NEON case kNEON: return 1; #endif @@ -208,10 +210,12 @@ static int wasmCPUInfo(CPUFeature feature) { } return 0; } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = wasmCPUInfo; -#elif defined(WEBP_USE_NEON) -// define a dummy function to enable turning off NEON at runtime by setting -// VP8DecGetCPUInfo = NULL +#elif defined(WEBP_HAVE_NEON) +// In most cases this function doesn't check for NEON support (it's assumed by +// the configuration), but enables turning off NEON at runtime, for testing +// purposes, by setting VP8GetCPUInfo = NULL. static int armCPUInfo(CPUFeature feature) { if (feature != kNEON) return 0; #if defined(__linux__) && defined(WEBP_HAVE_NEON_RTCD) @@ -235,6 +239,7 @@ static int armCPUInfo(CPUFeature feature) { return 1; #endif } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = armCPUInfo; #elif defined(WEBP_USE_MIPS32) || defined(WEBP_USE_MIPS_DSP_R2) || \ defined(WEBP_USE_MSA) @@ -246,7 +251,9 @@ static int mipsCPUInfo(CPUFeature feature) { } } +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = mipsCPUInfo; #else +WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; VP8CPUInfo VP8GetCPUInfo = NULL; #endif diff --git a/3rdparty/libwebp/src/dsp/cpu.h b/3rdparty/libwebp/src/dsp/cpu.h new file mode 100644 index 0000000000..c86540f280 --- /dev/null +++ b/3rdparty/libwebp/src/dsp/cpu.h @@ -0,0 +1,266 @@ +// Copyright 2022 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// CPU detection functions and macros. +// +// Author: Skal (pascal.massimino@gmail.com) + +#ifndef WEBP_DSP_CPU_H_ +#define WEBP_DSP_CPU_H_ + +#include + +#ifdef HAVE_CONFIG_H +#include "src/webp/config.h" +#endif + +#include "src/webp/types.h" + +#if defined(__GNUC__) +#define LOCAL_GCC_VERSION ((__GNUC__ << 8) | __GNUC_MINOR__) +#define LOCAL_GCC_PREREQ(maj, min) (LOCAL_GCC_VERSION >= (((maj) << 8) | (min))) +#else +#define LOCAL_GCC_VERSION 0 +#define LOCAL_GCC_PREREQ(maj, min) 0 +#endif + +#if defined(__clang__) +#define LOCAL_CLANG_VERSION ((__clang_major__ << 8) | __clang_minor__) +#define LOCAL_CLANG_PREREQ(maj, min) \ + (LOCAL_CLANG_VERSION >= (((maj) << 8) | (min))) +#else +#define LOCAL_CLANG_VERSION 0 +#define LOCAL_CLANG_PREREQ(maj, min) 0 +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +//------------------------------------------------------------------------------ +// x86 defines. + +#if !defined(HAVE_CONFIG_H) +#if defined(_MSC_VER) && _MSC_VER > 1310 && \ + (defined(_M_X64) || defined(_M_IX86)) +#define WEBP_MSC_SSE2 // Visual C++ SSE2 targets +#endif + +#if defined(_MSC_VER) && _MSC_VER >= 1500 && \ + (defined(_M_X64) || defined(_M_IX86)) +#define WEBP_MSC_SSE41 // Visual C++ SSE4.1 targets +#endif +#endif + +// WEBP_HAVE_* are used to indicate the presence of the instruction set in dsp +// files without intrinsics, allowing the corresponding Init() to be called. +// Files containing intrinsics will need to be built targeting the instruction +// set so should succeed on one of the earlier tests. +#if (defined(__SSE2__) || defined(WEBP_MSC_SSE2)) && \ + (!defined(HAVE_CONFIG_H) || defined(WEBP_HAVE_SSE2)) +#define WEBP_USE_SSE2 +#endif + +#if defined(WEBP_USE_SSE2) && !defined(WEBP_HAVE_SSE2) +#define WEBP_HAVE_SSE2 +#endif + +#if (defined(__SSE4_1__) || defined(WEBP_MSC_SSE41)) && \ + (!defined(HAVE_CONFIG_H) || defined(WEBP_HAVE_SSE41)) +#define WEBP_USE_SSE41 +#endif + +#if defined(WEBP_USE_SSE41) && !defined(WEBP_HAVE_SSE41) +#define WEBP_HAVE_SSE41 +#endif + +#undef WEBP_MSC_SSE41 +#undef WEBP_MSC_SSE2 + +//------------------------------------------------------------------------------ +// Arm defines. + +// The intrinsics currently cause compiler errors with arm-nacl-gcc and the +// inline assembly would need to be modified for use with Native Client. +#if ((defined(__ARM_NEON__) || defined(__aarch64__)) && \ + (!defined(HAVE_CONFIG_H) || defined(WEBP_HAVE_NEON))) && \ + !defined(__native_client__) +#define WEBP_USE_NEON +#endif + +#if !defined(WEBP_USE_NEON) && defined(__ANDROID__) && \ + defined(__ARM_ARCH_7A__) && defined(HAVE_CPU_FEATURES_H) +#define WEBP_ANDROID_NEON // Android targets that may have NEON +#define WEBP_USE_NEON +#endif + +// Note: ARM64 is supported in Visual Studio 2017, but requires the direct +// inclusion of arm64_neon.h; Visual Studio 2019 includes this file in +// arm_neon.h. Compile errors were seen with Visual Studio 2019 16.4 with +// vtbl4_u8(); a fix was made in 16.6. +#if defined(_MSC_VER) && \ + ((_MSC_VER >= 1700 && defined(_M_ARM)) || \ + (_MSC_VER >= 1926 && (defined(_M_ARM64) || defined(_M_ARM64EC)))) +#define WEBP_USE_NEON +#define WEBP_USE_INTRINSICS +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +#define WEBP_AARCH64 1 +#else +#define WEBP_AARCH64 0 +#endif + +#if defined(WEBP_USE_NEON) && !defined(WEBP_HAVE_NEON) +#define WEBP_HAVE_NEON +#endif + +//------------------------------------------------------------------------------ +// MIPS defines. + +#if defined(__mips__) && !defined(__mips64) && defined(__mips_isa_rev) && \ + (__mips_isa_rev >= 1) && (__mips_isa_rev < 6) +#define WEBP_USE_MIPS32 +#if (__mips_isa_rev >= 2) +#define WEBP_USE_MIPS32_R2 +#if defined(__mips_dspr2) || (defined(__mips_dsp_rev) && __mips_dsp_rev >= 2) +#define WEBP_USE_MIPS_DSP_R2 +#endif +#endif +#endif + +#if defined(__mips_msa) && defined(__mips_isa_rev) && (__mips_isa_rev >= 5) +#define WEBP_USE_MSA +#endif + +//------------------------------------------------------------------------------ + +#ifndef WEBP_DSP_OMIT_C_CODE +#define WEBP_DSP_OMIT_C_CODE 1 +#endif + +#if defined(WEBP_USE_NEON) && WEBP_DSP_OMIT_C_CODE +#define WEBP_NEON_OMIT_C_CODE 1 +#else +#define WEBP_NEON_OMIT_C_CODE 0 +#endif + +#if !(LOCAL_CLANG_PREREQ(3, 8) || LOCAL_GCC_PREREQ(4, 8) || WEBP_AARCH64) +#define WEBP_NEON_WORK_AROUND_GCC 1 +#else +#define WEBP_NEON_WORK_AROUND_GCC 0 +#endif + +//------------------------------------------------------------------------------ + +// This macro prevents thread_sanitizer from reporting known concurrent writes. +#define WEBP_TSAN_IGNORE_FUNCTION +#if defined(__has_feature) +#if __has_feature(thread_sanitizer) +#undef WEBP_TSAN_IGNORE_FUNCTION +#define WEBP_TSAN_IGNORE_FUNCTION __attribute__((no_sanitize_thread)) +#endif +#endif + +#if defined(__has_feature) +#if __has_feature(memory_sanitizer) +#define WEBP_MSAN +#endif +#endif + +#if defined(WEBP_USE_THREAD) && !defined(_WIN32) +#include // NOLINT + +#define WEBP_DSP_INIT(func) \ + do { \ + static volatile VP8CPUInfo func##_last_cpuinfo_used = \ + (VP8CPUInfo)&func##_last_cpuinfo_used; \ + static pthread_mutex_t func##_lock = PTHREAD_MUTEX_INITIALIZER; \ + if (pthread_mutex_lock(&func##_lock)) break; \ + if (func##_last_cpuinfo_used != VP8GetCPUInfo) func(); \ + func##_last_cpuinfo_used = VP8GetCPUInfo; \ + (void)pthread_mutex_unlock(&func##_lock); \ + } while (0) +#else // !(defined(WEBP_USE_THREAD) && !defined(_WIN32)) +#define WEBP_DSP_INIT(func) \ + do { \ + static volatile VP8CPUInfo func##_last_cpuinfo_used = \ + (VP8CPUInfo)&func##_last_cpuinfo_used; \ + if (func##_last_cpuinfo_used == VP8GetCPUInfo) break; \ + func(); \ + func##_last_cpuinfo_used = VP8GetCPUInfo; \ + } while (0) +#endif // defined(WEBP_USE_THREAD) && !defined(_WIN32) + +// Defines an Init + helper function that control multiple initialization of +// function pointers / tables. +/* Usage: + WEBP_DSP_INIT_FUNC(InitFunc) { + ...function body + } +*/ +#define WEBP_DSP_INIT_FUNC(name) \ + static WEBP_TSAN_IGNORE_FUNCTION void name##_body(void); \ + WEBP_TSAN_IGNORE_FUNCTION void name(void) { WEBP_DSP_INIT(name##_body); } \ + static WEBP_TSAN_IGNORE_FUNCTION void name##_body(void) + +#define WEBP_UBSAN_IGNORE_UNDEF +#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW +#if defined(__clang__) && defined(__has_attribute) +#if __has_attribute(no_sanitize) +// This macro prevents the undefined behavior sanitizer from reporting +// failures. This is only meant to silence unaligned loads on platforms that +// are known to support them. +#undef WEBP_UBSAN_IGNORE_UNDEF +#define WEBP_UBSAN_IGNORE_UNDEF __attribute__((no_sanitize("undefined"))) + +// This macro prevents the undefined behavior sanitizer from reporting +// failures related to unsigned integer overflows. This is only meant to +// silence cases where this well defined behavior is expected. +#undef WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW +#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW \ + __attribute__((no_sanitize("unsigned-integer-overflow"))) +#endif +#endif + +// If 'ptr' is NULL, returns NULL. Otherwise returns 'ptr + off'. +// Prevents undefined behavior sanitizer nullptr-with-nonzero-offset warning. +#if !defined(WEBP_OFFSET_PTR) +#define WEBP_OFFSET_PTR(ptr, off) (((ptr) == NULL) ? NULL : ((ptr) + (off))) +#endif + +// Regularize the definition of WEBP_SWAP_16BIT_CSP (backward compatibility) +#if !defined(WEBP_SWAP_16BIT_CSP) +#define WEBP_SWAP_16BIT_CSP 0 +#endif + +// some endian fix (e.g.: mips-gcc doesn't define __BIG_ENDIAN__) +#if !defined(WORDS_BIGENDIAN) && \ + (defined(__BIG_ENDIAN__) || defined(_M_PPC) || \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))) +#define WORDS_BIGENDIAN +#endif + +typedef enum { + kSSE2, + kSSE3, + kSlowSSSE3, // special feature for slow SSSE3 architectures + kSSE4_1, + kAVX, + kAVX2, + kNEON, + kMIPS32, + kMIPSdspR2, + kMSA +} CPUFeature; + +// returns true if the CPU supports the feature. +typedef int (*VP8CPUInfo)(CPUFeature feature); + +#endif // WEBP_DSP_CPU_H_ diff --git a/3rdparty/libwebp/src/dsp/dec.c b/3rdparty/libwebp/src/dsp/dec.c index 1119842dd3..33d8df8a62 100644 --- a/3rdparty/libwebp/src/dsp/dec.c +++ b/3rdparty/libwebp/src/dsp/dec.c @@ -734,6 +734,7 @@ VP8SimpleFilterFunc VP8SimpleHFilter16i; void (*VP8DitherCombine8x8)(const uint8_t* dither, uint8_t* dst, int dst_stride); +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8DspInitSSE2(void); extern void VP8DspInitSSE41(void); extern void VP8DspInitNEON(void); @@ -807,10 +808,10 @@ WEBP_DSP_INIT_FUNC(VP8DspInit) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8DspInitSSE2(); -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { VP8DspInitSSE41(); } @@ -834,7 +835,7 @@ WEBP_DSP_INIT_FUNC(VP8DspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8DspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/dec_neon.c b/3rdparty/libwebp/src/dsp/dec_neon.c index fa851707e2..22784cf15a 100644 --- a/3rdparty/libwebp/src/dsp/dec_neon.c +++ b/3rdparty/libwebp/src/dsp/dec_neon.c @@ -1428,7 +1428,7 @@ static WEBP_INLINE void DC8_NEON(uint8_t* dst, int do_top, int do_left) { if (do_top) { const uint8x8_t A = vld1_u8(dst - BPS); // top row -#if defined(__aarch64__) +#if WEBP_AARCH64 const uint16_t p2 = vaddlv_u8(A); sum_top = vdupq_n_u16(p2); #else @@ -1511,7 +1511,7 @@ static WEBP_INLINE void DC16_NEON(uint8_t* dst, int do_top, int do_left) { if (do_top) { const uint8x16_t A = vld1q_u8(dst - BPS); // top row -#if defined(__aarch64__) +#if WEBP_AARCH64 const uint16_t p3 = vaddlvq_u8(A); sum_top = vdupq_n_u16(p3); #else diff --git a/3rdparty/libwebp/src/dsp/dec_sse2.c b/3rdparty/libwebp/src/dsp/dec_sse2.c index 873aa59e8a..01e6bcb636 100644 --- a/3rdparty/libwebp/src/dsp/dec_sse2.c +++ b/3rdparty/libwebp/src/dsp/dec_sse2.c @@ -158,10 +158,10 @@ static void Transform_SSE2(const int16_t* in, uint8_t* dst, int do_two) { dst3 = _mm_loadl_epi64((__m128i*)(dst + 3 * BPS)); } else { // Load four bytes/pixels per line. - dst0 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 0 * BPS)); - dst1 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 1 * BPS)); - dst2 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 2 * BPS)); - dst3 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 3 * BPS)); + dst0 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 0 * BPS)); + dst1 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 1 * BPS)); + dst2 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 2 * BPS)); + dst3 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 3 * BPS)); } // Convert to 16b. dst0 = _mm_unpacklo_epi8(dst0, zero); @@ -187,10 +187,10 @@ static void Transform_SSE2(const int16_t* in, uint8_t* dst, int do_two) { _mm_storel_epi64((__m128i*)(dst + 3 * BPS), dst3); } else { // Store four bytes/pixels per line. - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); } } } @@ -213,10 +213,10 @@ static void TransformAC3(const int16_t* in, uint8_t* dst) { const __m128i m3 = _mm_subs_epi16(B, d4); const __m128i zero = _mm_setzero_si128(); // Load the source pixels. - __m128i dst0 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 0 * BPS)); - __m128i dst1 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 1 * BPS)); - __m128i dst2 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 2 * BPS)); - __m128i dst3 = _mm_cvtsi32_si128(WebPMemToUint32(dst + 3 * BPS)); + __m128i dst0 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 0 * BPS)); + __m128i dst1 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 1 * BPS)); + __m128i dst2 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 2 * BPS)); + __m128i dst3 = _mm_cvtsi32_si128(WebPMemToInt32(dst + 3 * BPS)); // Convert to 16b. dst0 = _mm_unpacklo_epi8(dst0, zero); dst1 = _mm_unpacklo_epi8(dst1, zero); @@ -233,10 +233,10 @@ static void TransformAC3(const int16_t* in, uint8_t* dst) { dst2 = _mm_packus_epi16(dst2, dst2); dst3 = _mm_packus_epi16(dst3, dst3); // Store the results. - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(dst0)); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(dst1)); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(dst2)); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(dst3)); } #undef MUL #endif // USE_TRANSFORM_AC3 @@ -477,11 +477,11 @@ static WEBP_INLINE void Load8x4_SSE2(const uint8_t* const b, int stride, // A0 = 63 62 61 60 23 22 21 20 43 42 41 40 03 02 01 00 // A1 = 73 72 71 70 33 32 31 30 53 52 51 50 13 12 11 10 const __m128i A0 = _mm_set_epi32( - WebPMemToUint32(&b[6 * stride]), WebPMemToUint32(&b[2 * stride]), - WebPMemToUint32(&b[4 * stride]), WebPMemToUint32(&b[0 * stride])); + WebPMemToInt32(&b[6 * stride]), WebPMemToInt32(&b[2 * stride]), + WebPMemToInt32(&b[4 * stride]), WebPMemToInt32(&b[0 * stride])); const __m128i A1 = _mm_set_epi32( - WebPMemToUint32(&b[7 * stride]), WebPMemToUint32(&b[3 * stride]), - WebPMemToUint32(&b[5 * stride]), WebPMemToUint32(&b[1 * stride])); + WebPMemToInt32(&b[7 * stride]), WebPMemToInt32(&b[3 * stride]), + WebPMemToInt32(&b[5 * stride]), WebPMemToInt32(&b[1 * stride])); // B0 = 53 43 52 42 51 41 50 40 13 03 12 02 11 01 10 00 // B1 = 73 63 72 62 71 61 70 60 33 23 32 22 31 21 30 20 @@ -540,7 +540,7 @@ static WEBP_INLINE void Store4x4_SSE2(__m128i* const x, uint8_t* dst, int stride) { int i; for (i = 0; i < 4; ++i, dst += stride) { - WebPUint32ToMem(dst, _mm_cvtsi128_si32(*x)); + WebPInt32ToMem(dst, _mm_cvtsi128_si32(*x)); *x = _mm_srli_si128(*x, 4); } } @@ -908,10 +908,10 @@ static void VE4_SSE2(uint8_t* dst) { // vertical const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGH00), one); const __m128i b = _mm_subs_epu8(a, lsb); const __m128i avg = _mm_avg_epu8(b, BCDEFGH0); - const uint32_t vals = _mm_cvtsi128_si32(avg); + const int vals = _mm_cvtsi128_si32(avg); int i; for (i = 0; i < 4; ++i) { - WebPUint32ToMem(dst + i * BPS, vals); + WebPInt32ToMem(dst + i * BPS, vals); } } @@ -925,10 +925,10 @@ static void LD4_SSE2(uint8_t* dst) { // Down-Left const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGHH0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i abcdefg = _mm_avg_epu8(avg2, BCDEFGH0); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); } static void VR4_SSE2(uint8_t* dst) { // Vertical-Right @@ -946,10 +946,10 @@ static void VR4_SSE2(uint8_t* dst) { // Vertical-Right const __m128i lsb = _mm_and_si128(_mm_xor_si128(IXABCD, ABCD0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i efgh = _mm_avg_epu8(avg2, XABCD); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); // these two are hard to implement in SSE2, so we keep the C-version: DST(0, 2) = AVG3(J, I, X); @@ -970,11 +970,12 @@ static void VL4_SSE2(uint8_t* dst) { // Vertical-Left const __m128i abbc = _mm_or_si128(ab, bc); const __m128i lsb2 = _mm_and_si128(abbc, lsb1); const __m128i avg4 = _mm_subs_epu8(avg3, lsb2); - const uint32_t extra_out = _mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); + const uint32_t extra_out = + (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); // these two are hard to get and irregular DST(3, 2) = (extra_out >> 0) & 0xff; @@ -990,7 +991,7 @@ static void RD4_SSE2(uint8_t* dst) { // Down-right const uint32_t K = dst[-1 + 2 * BPS]; const uint32_t L = dst[-1 + 3 * BPS]; const __m128i LKJI_____ = - _mm_cvtsi32_si128(L | (K << 8) | (J << 16) | (I << 24)); + _mm_cvtsi32_si128((int)(L | (K << 8) | (J << 16) | (I << 24))); const __m128i LKJIXABCD = _mm_or_si128(LKJI_____, ____XABCD); const __m128i KJIXABCD_ = _mm_srli_si128(LKJIXABCD, 1); const __m128i JIXABCD__ = _mm_srli_si128(LKJIXABCD, 2); @@ -998,10 +999,10 @@ static void RD4_SSE2(uint8_t* dst) { // Down-right const __m128i lsb = _mm_and_si128(_mm_xor_si128(JIXABCD__, LKJIXABCD), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i abcdefg = _mm_avg_epu8(avg2, KJIXABCD_); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); } #undef DST @@ -1015,13 +1016,13 @@ static WEBP_INLINE void TrueMotion_SSE2(uint8_t* dst, int size) { const __m128i zero = _mm_setzero_si128(); int y; if (size == 4) { - const __m128i top_values = _mm_cvtsi32_si128(WebPMemToUint32(top)); + const __m128i top_values = _mm_cvtsi32_si128(WebPMemToInt32(top)); const __m128i top_base = _mm_unpacklo_epi8(top_values, zero); for (y = 0; y < 4; ++y, dst += BPS) { const int val = dst[-1] - top[-1]; const __m128i base = _mm_set1_epi16(val); const __m128i out = _mm_packus_epi16(_mm_add_epi16(base, top_base), zero); - WebPUint32ToMem(dst, _mm_cvtsi128_si32(out)); + WebPInt32ToMem(dst, _mm_cvtsi128_si32(out)); } } else if (size == 8) { const __m128i top_values = _mm_loadl_epi64((const __m128i*)top); @@ -1062,7 +1063,7 @@ static void VE16_SSE2(uint8_t* dst) { static void HE16_SSE2(uint8_t* dst) { // horizontal int j; for (j = 16; j > 0; --j) { - const __m128i values = _mm_set1_epi8(dst[-1]); + const __m128i values = _mm_set1_epi8((char)dst[-1]); _mm_storeu_si128((__m128i*)dst, values); dst += BPS; } @@ -1070,7 +1071,7 @@ static void HE16_SSE2(uint8_t* dst) { // horizontal static WEBP_INLINE void Put16_SSE2(uint8_t v, uint8_t* dst) { int j; - const __m128i values = _mm_set1_epi8(v); + const __m128i values = _mm_set1_epi8((char)v); for (j = 0; j < 16; ++j) { _mm_storeu_si128((__m128i*)(dst + j * BPS), values); } @@ -1130,7 +1131,7 @@ static void VE8uv_SSE2(uint8_t* dst) { // vertical // helper for chroma-DC predictions static WEBP_INLINE void Put8x8uv_SSE2(uint8_t v, uint8_t* dst) { int j; - const __m128i values = _mm_set1_epi8(v); + const __m128i values = _mm_set1_epi8((char)v); for (j = 0; j < 8; ++j) { _mm_storel_epi64((__m128i*)(dst + j * BPS), values); } diff --git a/3rdparty/libwebp/src/dsp/dec_sse41.c b/3rdparty/libwebp/src/dsp/dec_sse41.c index 8f18506d54..08a3630272 100644 --- a/3rdparty/libwebp/src/dsp/dec_sse41.c +++ b/3rdparty/libwebp/src/dsp/dec_sse41.c @@ -23,7 +23,7 @@ static void HE16_SSE41(uint8_t* dst) { // horizontal int j; const __m128i kShuffle3 = _mm_set1_epi8(3); for (j = 16; j > 0; --j) { - const __m128i in = _mm_cvtsi32_si128(WebPMemToUint32(dst - 4)); + const __m128i in = _mm_cvtsi32_si128(WebPMemToInt32(dst - 4)); const __m128i values = _mm_shuffle_epi8(in, kShuffle3); _mm_storeu_si128((__m128i*)dst, values); dst += BPS; diff --git a/3rdparty/libwebp/src/dsp/dsp.h b/3rdparty/libwebp/src/dsp/dsp.h index 298c721ae2..d2000b8efc 100644 --- a/3rdparty/libwebp/src/dsp/dsp.h +++ b/3rdparty/libwebp/src/dsp/dsp.h @@ -18,6 +18,7 @@ #include "src/webp/config.h" #endif +#include "src/dsp/cpu.h" #include "src/webp/types.h" #ifdef __cplusplus @@ -27,205 +28,22 @@ extern "C" { #define BPS 32 // this is the common stride for enc/dec //------------------------------------------------------------------------------ -// CPU detection +// WEBP_RESTRICT +// Declares a pointer with the restrict type qualifier if available. +// This allows code to hint to the compiler that only this pointer references a +// particular object or memory region within the scope of the block in which it +// is declared. This may allow for improved optimizations due to the lack of +// pointer aliasing. See also: +// https://en.cppreference.com/w/c/language/restrict #if defined(__GNUC__) -# define LOCAL_GCC_VERSION ((__GNUC__ << 8) | __GNUC_MINOR__) -# define LOCAL_GCC_PREREQ(maj, min) \ - (LOCAL_GCC_VERSION >= (((maj) << 8) | (min))) +#define WEBP_RESTRICT __restrict__ +#elif defined(_MSC_VER) +#define WEBP_RESTRICT __restrict #else -# define LOCAL_GCC_VERSION 0 -# define LOCAL_GCC_PREREQ(maj, min) 0 +#define WEBP_RESTRICT #endif -#if defined(__clang__) -# define LOCAL_CLANG_VERSION ((__clang_major__ << 8) | __clang_minor__) -# define LOCAL_CLANG_PREREQ(maj, min) \ - (LOCAL_CLANG_VERSION >= (((maj) << 8) | (min))) -#else -# define LOCAL_CLANG_VERSION 0 -# define LOCAL_CLANG_PREREQ(maj, min) 0 -#endif - -#ifndef __has_builtin -# define __has_builtin(x) 0 -#endif - -#if !defined(HAVE_CONFIG_H) -#if defined(_MSC_VER) && _MSC_VER > 1310 && \ - (defined(_M_X64) || defined(_M_IX86)) -#define WEBP_MSC_SSE2 // Visual C++ SSE2 targets -#endif - -#if defined(_MSC_VER) && _MSC_VER >= 1500 && \ - (defined(_M_X64) || defined(_M_IX86)) -#define WEBP_MSC_SSE41 // Visual C++ SSE4.1 targets -#endif -#endif - -// WEBP_HAVE_* are used to indicate the presence of the instruction set in dsp -// files without intrinsics, allowing the corresponding Init() to be called. -// Files containing intrinsics will need to be built targeting the instruction -// set so should succeed on one of the earlier tests. -#if defined(__SSE2__) || defined(WEBP_MSC_SSE2) || defined(WEBP_HAVE_SSE2) -#define WEBP_USE_SSE2 -#endif - -#if defined(__SSE4_1__) || defined(WEBP_MSC_SSE41) || defined(WEBP_HAVE_SSE41) -#define WEBP_USE_SSE41 -#endif - -#undef WEBP_MSC_SSE41 -#undef WEBP_MSC_SSE2 - -// The intrinsics currently cause compiler errors with arm-nacl-gcc and the -// inline assembly would need to be modified for use with Native Client. -#if (defined(__ARM_NEON__) || \ - defined(__aarch64__) || defined(WEBP_HAVE_NEON)) && \ - !defined(__native_client__) -#define WEBP_USE_NEON -#endif - -#if !defined(WEBP_USE_NEON) && defined(__ANDROID__) && \ - defined(__ARM_ARCH_7A__) && defined(HAVE_CPU_FEATURES_H) -#define WEBP_ANDROID_NEON // Android targets that may have NEON -#define WEBP_USE_NEON -#endif - -#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_M_ARM) -#define WEBP_USE_NEON -#define WEBP_USE_INTRINSICS -#endif - -#if defined(__mips__) && !defined(__mips64) && \ - defined(__mips_isa_rev) && (__mips_isa_rev >= 1) && (__mips_isa_rev < 6) -#define WEBP_USE_MIPS32 -#if (__mips_isa_rev >= 2) -#define WEBP_USE_MIPS32_R2 -#if defined(__mips_dspr2) || (defined(__mips_dsp_rev) && __mips_dsp_rev >= 2) -#define WEBP_USE_MIPS_DSP_R2 -#endif -#endif -#endif - -#if defined(__mips_msa) && defined(__mips_isa_rev) && (__mips_isa_rev >= 5) -#define WEBP_USE_MSA -#endif - -#ifndef WEBP_DSP_OMIT_C_CODE -#define WEBP_DSP_OMIT_C_CODE 1 -#endif - -#if (defined(__aarch64__) || defined(__ARM_NEON__)) && WEBP_DSP_OMIT_C_CODE -#define WEBP_NEON_OMIT_C_CODE 1 -#else -#define WEBP_NEON_OMIT_C_CODE 0 -#endif - -#if !(LOCAL_CLANG_PREREQ(3,8) || LOCAL_GCC_PREREQ(4,8) || defined(__aarch64__)) -#define WEBP_NEON_WORK_AROUND_GCC 1 -#else -#define WEBP_NEON_WORK_AROUND_GCC 0 -#endif - -// This macro prevents thread_sanitizer from reporting known concurrent writes. -#define WEBP_TSAN_IGNORE_FUNCTION -#if defined(__has_feature) -#if __has_feature(thread_sanitizer) -#undef WEBP_TSAN_IGNORE_FUNCTION -#define WEBP_TSAN_IGNORE_FUNCTION __attribute__((no_sanitize_thread)) -#endif -#endif - -#if defined(WEBP_USE_THREAD) && !defined(_WIN32) -#include // NOLINT - -#define WEBP_DSP_INIT(func) do { \ - static volatile VP8CPUInfo func ## _last_cpuinfo_used = \ - (VP8CPUInfo)&func ## _last_cpuinfo_used; \ - static pthread_mutex_t func ## _lock = PTHREAD_MUTEX_INITIALIZER; \ - if (pthread_mutex_lock(&func ## _lock)) break; \ - if (func ## _last_cpuinfo_used != VP8GetCPUInfo) func(); \ - func ## _last_cpuinfo_used = VP8GetCPUInfo; \ - (void)pthread_mutex_unlock(&func ## _lock); \ -} while (0) -#else // !(defined(WEBP_USE_THREAD) && !defined(_WIN32)) -#define WEBP_DSP_INIT(func) do { \ - static volatile VP8CPUInfo func ## _last_cpuinfo_used = \ - (VP8CPUInfo)&func ## _last_cpuinfo_used; \ - if (func ## _last_cpuinfo_used == VP8GetCPUInfo) break; \ - func(); \ - func ## _last_cpuinfo_used = VP8GetCPUInfo; \ -} while (0) -#endif // defined(WEBP_USE_THREAD) && !defined(_WIN32) - -// Defines an Init + helper function that control multiple initialization of -// function pointers / tables. -/* Usage: - WEBP_DSP_INIT_FUNC(InitFunc) { - ...function body - } -*/ -#define WEBP_DSP_INIT_FUNC(name) \ - static WEBP_TSAN_IGNORE_FUNCTION void name ## _body(void); \ - WEBP_TSAN_IGNORE_FUNCTION void name(void) { \ - WEBP_DSP_INIT(name ## _body); \ - } \ - static WEBP_TSAN_IGNORE_FUNCTION void name ## _body(void) - -#define WEBP_UBSAN_IGNORE_UNDEF -#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW -#if defined(__clang__) && defined(__has_attribute) -#if __has_attribute(no_sanitize) -// This macro prevents the undefined behavior sanitizer from reporting -// failures. This is only meant to silence unaligned loads on platforms that -// are known to support them. -#undef WEBP_UBSAN_IGNORE_UNDEF -#define WEBP_UBSAN_IGNORE_UNDEF \ - __attribute__((no_sanitize("undefined"))) - -// This macro prevents the undefined behavior sanitizer from reporting -// failures related to unsigned integer overflows. This is only meant to -// silence cases where this well defined behavior is expected. -#undef WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW -#define WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW \ - __attribute__((no_sanitize("unsigned-integer-overflow"))) -#endif -#endif - -// If 'ptr' is NULL, returns NULL. Otherwise returns 'ptr + off'. -// Prevents undefined behavior sanitizer nullptr-with-nonzero-offset warning. -#if !defined(WEBP_OFFSET_PTR) -#define WEBP_OFFSET_PTR(ptr, off) (((ptr) == NULL) ? NULL : ((ptr) + (off))) -#endif - -// Regularize the definition of WEBP_SWAP_16BIT_CSP (backward compatibility) -#if !defined(WEBP_SWAP_16BIT_CSP) -#define WEBP_SWAP_16BIT_CSP 0 -#endif - -// some endian fix (e.g.: mips-gcc doesn't define __BIG_ENDIAN__) -#if !defined(WORDS_BIGENDIAN) && \ - (defined(__BIG_ENDIAN__) || defined(_M_PPC) || \ - (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))) -#define WORDS_BIGENDIAN -#endif - -typedef enum { - kSSE2, - kSSE3, - kSlowSSSE3, // special feature for slow SSSE3 architectures - kSSE4_1, - kAVX, - kAVX2, - kNEON, - kMIPS32, - kMIPSdspR2, - kMSA -} CPUFeature; -// returns true if the CPU supports the feature. -typedef int (*VP8CPUInfo)(CPUFeature feature); -WEBP_EXTERN VP8CPUInfo VP8GetCPUInfo; //------------------------------------------------------------------------------ // Init stub generator @@ -514,15 +332,6 @@ extern void WebPConvertARGBToUV_C(const uint32_t* argb, uint8_t* u, uint8_t* v, extern void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, uint8_t* u, uint8_t* v, int width); -// utilities for accurate RGB->YUV conversion -extern uint64_t (*WebPSharpYUVUpdateY)(const uint16_t* src, const uint16_t* ref, - uint16_t* dst, int len); -extern void (*WebPSharpYUVUpdateRGB)(const int16_t* src, const int16_t* ref, - int16_t* dst, int len); -extern void (*WebPSharpYUVFilterRow)(const int16_t* A, const int16_t* B, - int len, - const uint16_t* best_y, uint16_t* out); - // Must be called before using the above. void WebPInitConvertARGBToYUV(void); @@ -578,26 +387,29 @@ extern void (*WebPApplyAlphaMultiply4444)( // Dispatch the values from alpha[] plane to the ARGB destination 'dst'. // Returns true if alpha[] plane has non-trivial values different from 0xff. -extern int (*WebPDispatchAlpha)(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint8_t* dst, int dst_stride); +extern int (*WebPDispatchAlpha)(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint8_t* WEBP_RESTRICT dst, int dst_stride); // Transfer packed 8b alpha[] values to green channel in dst[], zero'ing the // A/R/B values. 'dst_stride' is the stride for dst[] in uint32_t units. -extern void (*WebPDispatchAlphaToGreen)(const uint8_t* alpha, int alpha_stride, - int width, int height, - uint32_t* dst, int dst_stride); +extern void (*WebPDispatchAlphaToGreen)(const uint8_t* WEBP_RESTRICT alpha, + int alpha_stride, int width, int height, + uint32_t* WEBP_RESTRICT dst, + int dst_stride); // Extract the alpha values from 32b values in argb[] and pack them into alpha[] // (this is the opposite of WebPDispatchAlpha). // Returns true if there's only trivial 0xff alpha values. -extern int (*WebPExtractAlpha)(const uint8_t* argb, int argb_stride, - int width, int height, - uint8_t* alpha, int alpha_stride); +extern int (*WebPExtractAlpha)(const uint8_t* WEBP_RESTRICT argb, + int argb_stride, int width, int height, + uint8_t* WEBP_RESTRICT alpha, + int alpha_stride); // Extract the green values from 32b values in argb[] and pack them into alpha[] // (this is the opposite of WebPDispatchAlphaToGreen). -extern void (*WebPExtractGreen)(const uint32_t* argb, uint8_t* alpha, int size); +extern void (*WebPExtractGreen)(const uint32_t* WEBP_RESTRICT argb, + uint8_t* WEBP_RESTRICT alpha, int size); // Pre-Multiply operation transforms x into x * A / 255 (where x=Y,R,G or B). // Un-Multiply operation transforms x into x * 255 / A. @@ -610,29 +422,35 @@ void WebPMultARGBRows(uint8_t* ptr, int stride, int width, int num_rows, int inverse); // Same for a row of single values, with side alpha values. -extern void (*WebPMultRow)(uint8_t* const ptr, const uint8_t* const alpha, +extern void (*WebPMultRow)(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse); // Same a WebPMultRow(), but for several 'num_rows' rows. -void WebPMultRows(uint8_t* ptr, int stride, - const uint8_t* alpha, int alpha_stride, +void WebPMultRows(uint8_t* WEBP_RESTRICT ptr, int stride, + const uint8_t* WEBP_RESTRICT alpha, int alpha_stride, int width, int num_rows, int inverse); // Plain-C versions, used as fallback by some implementations. -void WebPMultRow_C(uint8_t* const ptr, const uint8_t* const alpha, +void WebPMultRow_C(uint8_t* WEBP_RESTRICT const ptr, + const uint8_t* WEBP_RESTRICT const alpha, int width, int inverse); void WebPMultARGBRow_C(uint32_t* const ptr, int width, int inverse); #ifdef WORDS_BIGENDIAN // ARGB packing function: a/r/g/b input is rgba or bgra order. -extern void (*WebPPackARGB)(const uint8_t* a, const uint8_t* r, - const uint8_t* g, const uint8_t* b, int len, - uint32_t* out); +extern void (*WebPPackARGB)(const uint8_t* WEBP_RESTRICT a, + const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, uint32_t* WEBP_RESTRICT out); #endif // RGB packing function. 'step' can be 3 or 4. r/g/b input is rgb or bgr order. -extern void (*WebPPackRGB)(const uint8_t* r, const uint8_t* g, const uint8_t* b, - int len, int step, uint32_t* out); +extern void (*WebPPackRGB)(const uint8_t* WEBP_RESTRICT r, + const uint8_t* WEBP_RESTRICT g, + const uint8_t* WEBP_RESTRICT b, + int len, int step, uint32_t* WEBP_RESTRICT out); // This function returns true if src[i] contains a value different from 0xff. extern int (*WebPHasAlpha8b)(const uint8_t* src, int length); diff --git a/3rdparty/libwebp/src/dsp/enc.c b/3rdparty/libwebp/src/dsp/enc.c index 2fddbc4c52..2ba97ba8d6 100644 --- a/3rdparty/libwebp/src/dsp/enc.c +++ b/3rdparty/libwebp/src/dsp/enc.c @@ -732,6 +732,7 @@ VP8QuantizeBlockWHT VP8EncQuantizeBlockWHT; VP8BlockCopy VP8Copy4x4; VP8BlockCopy VP8Copy16x8; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8EncDspInitSSE2(void); extern void VP8EncDspInitSSE41(void); extern void VP8EncDspInitNEON(void); @@ -773,10 +774,10 @@ WEBP_DSP_INIT_FUNC(VP8EncDspInit) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8EncDspInitSSE2(); -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { VP8EncDspInitSSE41(); } @@ -800,7 +801,7 @@ WEBP_DSP_INIT_FUNC(VP8EncDspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8EncDspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/enc_neon.c b/3rdparty/libwebp/src/dsp/enc_neon.c index 43bf1245c5..714800367b 100644 --- a/3rdparty/libwebp/src/dsp/enc_neon.c +++ b/3rdparty/libwebp/src/dsp/enc_neon.c @@ -9,7 +9,7 @@ // // ARM NEON version of speed-critical encoding functions. // -// adapted from libvpx (http://www.webmproject.org/code/) +// adapted from libvpx (https://www.webmproject.org/code/) #include "src/dsp/dsp.h" @@ -764,9 +764,14 @@ static WEBP_INLINE void AccumulateSSE16_NEON(const uint8_t* const a, // Horizontal sum of all four uint32_t values in 'sum'. static int SumToInt_NEON(uint32x4_t sum) { +#if WEBP_AARCH64 + return (int)vaddvq_u32(sum); +#else const uint64x2_t sum2 = vpaddlq_u32(sum); - const uint64_t sum3 = vgetq_lane_u64(sum2, 0) + vgetq_lane_u64(sum2, 1); - return (int)sum3; + const uint32x2_t sum3 = vadd_u32(vreinterpret_u32_u64(vget_low_u64(sum2)), + vreinterpret_u32_u64(vget_high_u64(sum2))); + return (int)vget_lane_u32(sum3, 0); +#endif } static int SSE16x16_NEON(const uint8_t* a, const uint8_t* b) { @@ -860,7 +865,7 @@ static int QuantizeBlock_NEON(int16_t in[16], int16_t out[16], uint8x8x4_t shuffles; // vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use // non-standard versions there. -#if defined(__APPLE__) && defined(__aarch64__) && \ +#if defined(__APPLE__) && WEBP_AARCH64 && \ defined(__apple_build_version__) && (__apple_build_version__< 6020037) uint8x16x2_t all_out; INIT_VECTOR2(all_out, vreinterpretq_u8_s16(out0), vreinterpretq_u8_s16(out1)); diff --git a/3rdparty/libwebp/src/dsp/enc_sse2.c b/3rdparty/libwebp/src/dsp/enc_sse2.c index b2e78ed941..010624a2f7 100644 --- a/3rdparty/libwebp/src/dsp/enc_sse2.c +++ b/3rdparty/libwebp/src/dsp/enc_sse2.c @@ -25,9 +25,160 @@ //------------------------------------------------------------------------------ // Transforms (Paragraph 14.4) -// Does one or two inverse transforms. -static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, - int do_two) { +// Does one inverse transform. +static void ITransform_One_SSE2(const uint8_t* ref, const int16_t* in, + uint8_t* dst) { + // This implementation makes use of 16-bit fixed point versions of two + // multiply constants: + // K1 = sqrt(2) * cos (pi/8) ~= 85627 / 2^16 + // K2 = sqrt(2) * sin (pi/8) ~= 35468 / 2^16 + // + // To be able to use signed 16-bit integers, we use the following trick to + // have constants within range: + // - Associated constants are obtained by subtracting the 16-bit fixed point + // version of one: + // k = K - (1 << 16) => K = k + (1 << 16) + // K1 = 85267 => k1 = 20091 + // K2 = 35468 => k2 = -30068 + // - The multiplication of a variable by a constant become the sum of the + // variable and the multiplication of that variable by the associated + // constant: + // (x * K) >> 16 = (x * (k + (1 << 16))) >> 16 = ((x * k ) >> 16) + x + const __m128i k1k2 = _mm_set_epi16(-30068, -30068, -30068, -30068, + 20091, 20091, 20091, 20091); + const __m128i k2k1 = _mm_set_epi16(20091, 20091, 20091, 20091, + -30068, -30068, -30068, -30068); + const __m128i zero = _mm_setzero_si128(); + const __m128i zero_four = _mm_set_epi16(0, 0, 0, 0, 4, 4, 4, 4); + __m128i T01, T23; + + // Load and concatenate the transform coefficients. + const __m128i in01 = _mm_loadu_si128((const __m128i*)&in[0]); + const __m128i in23 = _mm_loadu_si128((const __m128i*)&in[8]); + // a00 a10 a20 a30 a01 a11 a21 a31 + // a02 a12 a22 a32 a03 a13 a23 a33 + + // Vertical pass and subsequent transpose. + { + const __m128i in1 = _mm_unpackhi_epi64(in01, in01); + const __m128i in3 = _mm_unpackhi_epi64(in23, in23); + + // First pass, c and d calculations are longer because of the "trick" + // multiplications. + // c = MUL(in1, K2) - MUL(in3, K1) = MUL(in1, k2) - MUL(in3, k1) + in1 - in3 + // d = MUL(in1, K1) + MUL(in3, K2) = MUL(in1, k1) + MUL(in3, k2) + in1 + in3 + const __m128i a_d3 = _mm_add_epi16(in01, in23); + const __m128i b_c3 = _mm_sub_epi16(in01, in23); + const __m128i c1d1 = _mm_mulhi_epi16(in1, k2k1); + const __m128i c2d2 = _mm_mulhi_epi16(in3, k1k2); + const __m128i c3 = _mm_unpackhi_epi64(b_c3, b_c3); + const __m128i c4 = _mm_sub_epi16(c1d1, c2d2); + const __m128i c = _mm_add_epi16(c3, c4); + const __m128i d4u = _mm_add_epi16(c1d1, c2d2); + const __m128i du = _mm_add_epi16(a_d3, d4u); + const __m128i d = _mm_unpackhi_epi64(du, du); + + // Second pass. + const __m128i comb_ab = _mm_unpacklo_epi64(a_d3, b_c3); + const __m128i comb_dc = _mm_unpacklo_epi64(d, c); + + const __m128i tmp01 = _mm_add_epi16(comb_ab, comb_dc); + const __m128i tmp32 = _mm_sub_epi16(comb_ab, comb_dc); + const __m128i tmp23 = _mm_shuffle_epi32(tmp32, _MM_SHUFFLE(1, 0, 3, 2)); + + const __m128i transpose_0 = _mm_unpacklo_epi16(tmp01, tmp23); + const __m128i transpose_1 = _mm_unpackhi_epi16(tmp01, tmp23); + // a00 a20 a01 a21 a02 a22 a03 a23 + // a10 a30 a11 a31 a12 a32 a13 a33 + + T01 = _mm_unpacklo_epi16(transpose_0, transpose_1); + T23 = _mm_unpackhi_epi16(transpose_0, transpose_1); + // a00 a10 a20 a30 a01 a11 a21 a31 + // a02 a12 a22 a32 a03 a13 a23 a33 + } + + // Horizontal pass and subsequent transpose. + { + const __m128i T1 = _mm_unpackhi_epi64(T01, T01); + const __m128i T3 = _mm_unpackhi_epi64(T23, T23); + + // First pass, c and d calculations are longer because of the "trick" + // multiplications. + const __m128i dc = _mm_add_epi16(T01, zero_four); + + // c = MUL(T1, K2) - MUL(T3, K1) = MUL(T1, k2) - MUL(T3, k1) + T1 - T3 + // d = MUL(T1, K1) + MUL(T3, K2) = MUL(T1, k1) + MUL(T3, k2) + T1 + T3 + const __m128i a_d3 = _mm_add_epi16(dc, T23); + const __m128i b_c3 = _mm_sub_epi16(dc, T23); + const __m128i c1d1 = _mm_mulhi_epi16(T1, k2k1); + const __m128i c2d2 = _mm_mulhi_epi16(T3, k1k2); + const __m128i c3 = _mm_unpackhi_epi64(b_c3, b_c3); + const __m128i c4 = _mm_sub_epi16(c1d1, c2d2); + const __m128i c = _mm_add_epi16(c3, c4); + const __m128i d4u = _mm_add_epi16(c1d1, c2d2); + const __m128i du = _mm_add_epi16(a_d3, d4u); + const __m128i d = _mm_unpackhi_epi64(du, du); + + // Second pass. + const __m128i comb_ab = _mm_unpacklo_epi64(a_d3, b_c3); + const __m128i comb_dc = _mm_unpacklo_epi64(d, c); + + const __m128i tmp01 = _mm_add_epi16(comb_ab, comb_dc); + const __m128i tmp32 = _mm_sub_epi16(comb_ab, comb_dc); + const __m128i tmp23 = _mm_shuffle_epi32(tmp32, _MM_SHUFFLE(1, 0, 3, 2)); + + const __m128i shifted01 = _mm_srai_epi16(tmp01, 3); + const __m128i shifted23 = _mm_srai_epi16(tmp23, 3); + // a00 a01 a02 a03 a10 a11 a12 a13 + // a20 a21 a22 a23 a30 a31 a32 a33 + + const __m128i transpose_0 = _mm_unpacklo_epi16(shifted01, shifted23); + const __m128i transpose_1 = _mm_unpackhi_epi16(shifted01, shifted23); + // a00 a20 a01 a21 a02 a22 a03 a23 + // a10 a30 a11 a31 a12 a32 a13 a33 + + T01 = _mm_unpacklo_epi16(transpose_0, transpose_1); + T23 = _mm_unpackhi_epi16(transpose_0, transpose_1); + // a00 a10 a20 a30 a01 a11 a21 a31 + // a02 a12 a22 a32 a03 a13 a23 a33 + } + + // Add inverse transform to 'ref' and store. + { + // Load the reference(s). + __m128i ref01, ref23, ref0123; + int32_t buf[4]; + + // Load four bytes/pixels per line. + const __m128i ref0 = _mm_cvtsi32_si128(WebPMemToInt32(&ref[0 * BPS])); + const __m128i ref1 = _mm_cvtsi32_si128(WebPMemToInt32(&ref[1 * BPS])); + const __m128i ref2 = _mm_cvtsi32_si128(WebPMemToInt32(&ref[2 * BPS])); + const __m128i ref3 = _mm_cvtsi32_si128(WebPMemToInt32(&ref[3 * BPS])); + ref01 = _mm_unpacklo_epi32(ref0, ref1); + ref23 = _mm_unpacklo_epi32(ref2, ref3); + + // Convert to 16b. + ref01 = _mm_unpacklo_epi8(ref01, zero); + ref23 = _mm_unpacklo_epi8(ref23, zero); + // Add the inverse transform(s). + ref01 = _mm_add_epi16(ref01, T01); + ref23 = _mm_add_epi16(ref23, T23); + // Unsigned saturate to 8b. + ref0123 = _mm_packus_epi16(ref01, ref23); + + _mm_storeu_si128((__m128i *)buf, ref0123); + + // Store four bytes/pixels per line. + WebPInt32ToMem(&dst[0 * BPS], buf[0]); + WebPInt32ToMem(&dst[1 * BPS], buf[1]); + WebPInt32ToMem(&dst[2 * BPS], buf[2]); + WebPInt32ToMem(&dst[3 * BPS], buf[3]); + } +} + +// Does two inverse transforms. +static void ITransform_Two_SSE2(const uint8_t* ref, const int16_t* in, + uint8_t* dst) { // This implementation makes use of 16-bit fixed point versions of two // multiply constants: // K1 = sqrt(2) * cos (pi/8) ~= 85627 / 2^16 @@ -49,33 +200,21 @@ static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, __m128i T0, T1, T2, T3; // Load and concatenate the transform coefficients (we'll do two inverse - // transforms in parallel). In the case of only one inverse transform, the - // second half of the vectors will just contain random value we'll never - // use nor store. + // transforms in parallel). __m128i in0, in1, in2, in3; { - in0 = _mm_loadl_epi64((const __m128i*)&in[0]); - in1 = _mm_loadl_epi64((const __m128i*)&in[4]); - in2 = _mm_loadl_epi64((const __m128i*)&in[8]); - in3 = _mm_loadl_epi64((const __m128i*)&in[12]); - // a00 a10 a20 a30 x x x x - // a01 a11 a21 a31 x x x x - // a02 a12 a22 a32 x x x x - // a03 a13 a23 a33 x x x x - if (do_two) { - const __m128i inB0 = _mm_loadl_epi64((const __m128i*)&in[16]); - const __m128i inB1 = _mm_loadl_epi64((const __m128i*)&in[20]); - const __m128i inB2 = _mm_loadl_epi64((const __m128i*)&in[24]); - const __m128i inB3 = _mm_loadl_epi64((const __m128i*)&in[28]); - in0 = _mm_unpacklo_epi64(in0, inB0); - in1 = _mm_unpacklo_epi64(in1, inB1); - in2 = _mm_unpacklo_epi64(in2, inB2); - in3 = _mm_unpacklo_epi64(in3, inB3); - // a00 a10 a20 a30 b00 b10 b20 b30 - // a01 a11 a21 a31 b01 b11 b21 b31 - // a02 a12 a22 a32 b02 b12 b22 b32 - // a03 a13 a23 a33 b03 b13 b23 b33 - } + const __m128i tmp0 = _mm_loadu_si128((const __m128i*)&in[0]); + const __m128i tmp1 = _mm_loadu_si128((const __m128i*)&in[8]); + const __m128i tmp2 = _mm_loadu_si128((const __m128i*)&in[16]); + const __m128i tmp3 = _mm_loadu_si128((const __m128i*)&in[24]); + in0 = _mm_unpacklo_epi64(tmp0, tmp2); + in1 = _mm_unpackhi_epi64(tmp0, tmp2); + in2 = _mm_unpacklo_epi64(tmp1, tmp3); + in3 = _mm_unpackhi_epi64(tmp1, tmp3); + // a00 a10 a20 a30 b00 b10 b20 b30 + // a01 a11 a21 a31 b01 b11 b21 b31 + // a02 a12 a22 a32 b02 b12 b22 b32 + // a03 a13 a23 a33 b03 b13 b23 b33 } // Vertical pass and subsequent transpose. @@ -148,19 +287,11 @@ static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, const __m128i zero = _mm_setzero_si128(); // Load the reference(s). __m128i ref0, ref1, ref2, ref3; - if (do_two) { - // Load eight bytes/pixels per line. - ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]); - ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]); - ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]); - ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]); - } else { - // Load four bytes/pixels per line. - ref0 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[0 * BPS])); - ref1 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[1 * BPS])); - ref2 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[2 * BPS])); - ref3 = _mm_cvtsi32_si128(WebPMemToUint32(&ref[3 * BPS])); - } + // Load eight bytes/pixels per line. + ref0 = _mm_loadl_epi64((const __m128i*)&ref[0 * BPS]); + ref1 = _mm_loadl_epi64((const __m128i*)&ref[1 * BPS]); + ref2 = _mm_loadl_epi64((const __m128i*)&ref[2 * BPS]); + ref3 = _mm_loadl_epi64((const __m128i*)&ref[3 * BPS]); // Convert to 16b. ref0 = _mm_unpacklo_epi8(ref0, zero); ref1 = _mm_unpacklo_epi8(ref1, zero); @@ -176,20 +307,21 @@ static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, ref1 = _mm_packus_epi16(ref1, ref1); ref2 = _mm_packus_epi16(ref2, ref2); ref3 = _mm_packus_epi16(ref3, ref3); - // Store the results. - if (do_two) { - // Store eight bytes/pixels per line. - _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0); - _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1); - _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2); - _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3); - } else { - // Store four bytes/pixels per line. - WebPUint32ToMem(&dst[0 * BPS], _mm_cvtsi128_si32(ref0)); - WebPUint32ToMem(&dst[1 * BPS], _mm_cvtsi128_si32(ref1)); - WebPUint32ToMem(&dst[2 * BPS], _mm_cvtsi128_si32(ref2)); - WebPUint32ToMem(&dst[3 * BPS], _mm_cvtsi128_si32(ref3)); - } + // Store eight bytes/pixels per line. + _mm_storel_epi64((__m128i*)&dst[0 * BPS], ref0); + _mm_storel_epi64((__m128i*)&dst[1 * BPS], ref1); + _mm_storel_epi64((__m128i*)&dst[2 * BPS], ref2); + _mm_storel_epi64((__m128i*)&dst[3 * BPS], ref3); + } +} + +// Does one or two inverse transforms. +static void ITransform_SSE2(const uint8_t* ref, const int16_t* in, uint8_t* dst, + int do_two) { + if (do_two) { + ITransform_Two_SSE2(ref, in, dst); + } else { + ITransform_One_SSE2(ref, in, dst); } } @@ -481,7 +613,7 @@ static void CollectHistogram_SSE2(const uint8_t* ref, const uint8_t* pred, // helper for chroma-DC predictions static WEBP_INLINE void Put8x8uv_SSE2(uint8_t v, uint8_t* dst) { int j; - const __m128i values = _mm_set1_epi8(v); + const __m128i values = _mm_set1_epi8((char)v); for (j = 0; j < 8; ++j) { _mm_storel_epi64((__m128i*)(dst + j * BPS), values); } @@ -489,7 +621,7 @@ static WEBP_INLINE void Put8x8uv_SSE2(uint8_t v, uint8_t* dst) { static WEBP_INLINE void Put16_SSE2(uint8_t v, uint8_t* dst) { int j; - const __m128i values = _mm_set1_epi8(v); + const __m128i values = _mm_set1_epi8((char)v); for (j = 0; j < 16; ++j) { _mm_store_si128((__m128i*)(dst + j * BPS), values); } @@ -540,7 +672,7 @@ static WEBP_INLINE void VerticalPred_SSE2(uint8_t* dst, static WEBP_INLINE void HE8uv_SSE2(uint8_t* dst, const uint8_t* left) { int j; for (j = 0; j < 8; ++j) { - const __m128i values = _mm_set1_epi8(left[j]); + const __m128i values = _mm_set1_epi8((char)left[j]); _mm_storel_epi64((__m128i*)dst, values); dst += BPS; } @@ -549,7 +681,7 @@ static WEBP_INLINE void HE8uv_SSE2(uint8_t* dst, const uint8_t* left) { static WEBP_INLINE void HE16_SSE2(uint8_t* dst, const uint8_t* left) { int j; for (j = 0; j < 16; ++j) { - const __m128i values = _mm_set1_epi8(left[j]); + const __m128i values = _mm_set1_epi8((char)left[j]); _mm_store_si128((__m128i*)dst, values); dst += BPS; } @@ -722,10 +854,10 @@ static WEBP_INLINE void VE4_SSE2(uint8_t* dst, const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGH00), one); const __m128i b = _mm_subs_epu8(a, lsb); const __m128i avg = _mm_avg_epu8(b, BCDEFGH0); - const uint32_t vals = _mm_cvtsi128_si32(avg); + const int vals = _mm_cvtsi128_si32(avg); int i; for (i = 0; i < 4; ++i) { - WebPUint32ToMem(dst + i * BPS, vals); + WebPInt32ToMem(dst + i * BPS, vals); } } @@ -760,10 +892,10 @@ static WEBP_INLINE void LD4_SSE2(uint8_t* dst, const __m128i lsb = _mm_and_si128(_mm_xor_si128(ABCDEFGH, CDEFGHH0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i abcdefg = _mm_avg_epu8(avg2, BCDEFGH0); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); } static WEBP_INLINE void VR4_SSE2(uint8_t* dst, @@ -782,10 +914,10 @@ static WEBP_INLINE void VR4_SSE2(uint8_t* dst, const __m128i lsb = _mm_and_si128(_mm_xor_si128(IXABCD, ABCD0), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i efgh = _mm_avg_epu8(avg2, XABCD); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( abcd )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( efgh )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(abcd, 1))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_slli_si128(efgh, 1))); // these two are hard to implement in SSE2, so we keep the C-version: DST(0, 2) = AVG3(J, I, X); @@ -807,11 +939,12 @@ static WEBP_INLINE void VL4_SSE2(uint8_t* dst, const __m128i abbc = _mm_or_si128(ab, bc); const __m128i lsb2 = _mm_and_si128(abbc, lsb1); const __m128i avg4 = _mm_subs_epu8(avg3, lsb2); - const uint32_t extra_out = _mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); + const uint32_t extra_out = + (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(avg4, 4)); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32( avg1 )); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32( avg4 )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg1, 1))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(avg4, 1))); // these two are hard to get and irregular DST(3, 2) = (extra_out >> 0) & 0xff; @@ -829,10 +962,10 @@ static WEBP_INLINE void RD4_SSE2(uint8_t* dst, const __m128i lsb = _mm_and_si128(_mm_xor_si128(JIXABCD__, LKJIXABCD), one); const __m128i avg2 = _mm_subs_epu8(avg1, lsb); const __m128i abcdefg = _mm_avg_epu8(avg2, KJIXABCD_); - WebPUint32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); - WebPUint32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); - WebPUint32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); - WebPUint32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); + WebPInt32ToMem(dst + 3 * BPS, _mm_cvtsi128_si32( abcdefg )); + WebPInt32ToMem(dst + 2 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 1))); + WebPInt32ToMem(dst + 1 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 2))); + WebPInt32ToMem(dst + 0 * BPS, _mm_cvtsi128_si32(_mm_srli_si128(abcdefg, 3))); } static WEBP_INLINE void HU4_SSE2(uint8_t* dst, const uint8_t* top) { @@ -875,14 +1008,14 @@ static WEBP_INLINE void HD4_SSE2(uint8_t* dst, const uint8_t* top) { static WEBP_INLINE void TM4_SSE2(uint8_t* dst, const uint8_t* top) { const __m128i zero = _mm_setzero_si128(); - const __m128i top_values = _mm_cvtsi32_si128(WebPMemToUint32(top)); + const __m128i top_values = _mm_cvtsi32_si128(WebPMemToInt32(top)); const __m128i top_base = _mm_unpacklo_epi8(top_values, zero); int y; for (y = 0; y < 4; ++y, dst += BPS) { const int val = top[-2 - y] - top[-1]; const __m128i base = _mm_set1_epi16(val); const __m128i out = _mm_packus_epi16(_mm_add_epi16(base, top_base), zero); - WebPUint32ToMem(dst, _mm_cvtsi128_si32(out)); + WebPInt32ToMem(dst, _mm_cvtsi128_si32(out)); } } diff --git a/3rdparty/libwebp/src/dsp/filters.c b/3rdparty/libwebp/src/dsp/filters.c index 9e910d99c9..c1350d5c9d 100644 --- a/3rdparty/libwebp/src/dsp/filters.c +++ b/3rdparty/libwebp/src/dsp/filters.c @@ -189,6 +189,12 @@ static void GradientFilter_C(const uint8_t* data, int width, int height, //------------------------------------------------------------------------------ +static void NoneUnfilter_C(const uint8_t* prev, const uint8_t* in, + uint8_t* out, int width) { + (void)prev; + if (out != in) memcpy(out, in, width * sizeof(*out)); +} + static void HorizontalUnfilter_C(const uint8_t* prev, const uint8_t* in, uint8_t* out, int width) { uint8_t pred = (prev == NULL) ? 0 : prev[0]; @@ -233,13 +239,14 @@ static void GradientUnfilter_C(const uint8_t* prev, const uint8_t* in, WebPFilterFunc WebPFilters[WEBP_FILTER_LAST]; WebPUnfilterFunc WebPUnfilters[WEBP_FILTER_LAST]; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8FiltersInitMIPSdspR2(void); extern void VP8FiltersInitMSA(void); extern void VP8FiltersInitNEON(void); extern void VP8FiltersInitSSE2(void); WEBP_DSP_INIT_FUNC(VP8FiltersInit) { - WebPUnfilters[WEBP_FILTER_NONE] = NULL; + WebPUnfilters[WEBP_FILTER_NONE] = NoneUnfilter_C; #if !WEBP_NEON_OMIT_C_CODE WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_C; WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_C; @@ -254,7 +261,7 @@ WEBP_DSP_INIT_FUNC(VP8FiltersInit) { #endif if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8FiltersInitSSE2(); } @@ -271,13 +278,14 @@ WEBP_DSP_INIT_FUNC(VP8FiltersInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8FiltersInitNEON(); } #endif + assert(WebPUnfilters[WEBP_FILTER_NONE] != NULL); assert(WebPUnfilters[WEBP_FILTER_HORIZONTAL] != NULL); assert(WebPUnfilters[WEBP_FILTER_VERTICAL] != NULL); assert(WebPUnfilters[WEBP_FILTER_GRADIENT] != NULL); diff --git a/3rdparty/libwebp/src/dsp/filters_sse2.c b/3rdparty/libwebp/src/dsp/filters_sse2.c index 4b3f2d020f..5c33ec15e2 100644 --- a/3rdparty/libwebp/src/dsp/filters_sse2.c +++ b/3rdparty/libwebp/src/dsp/filters_sse2.c @@ -320,7 +320,12 @@ extern void VP8FiltersInitSSE2(void); WEBP_TSAN_IGNORE_FUNCTION void VP8FiltersInitSSE2(void) { WebPUnfilters[WEBP_FILTER_HORIZONTAL] = HorizontalUnfilter_SSE2; +#if defined(CHROMIUM) + // TODO(crbug.com/654974) + (void)VerticalUnfilter_SSE2; +#else WebPUnfilters[WEBP_FILTER_VERTICAL] = VerticalUnfilter_SSE2; +#endif WebPUnfilters[WEBP_FILTER_GRADIENT] = GradientUnfilter_SSE2; WebPFilters[WEBP_FILTER_HORIZONTAL] = HorizontalFilter_SSE2; diff --git a/3rdparty/libwebp/src/dsp/lossless.c b/3rdparty/libwebp/src/dsp/lossless.c index 46b220e2ed..9f81209453 100644 --- a/3rdparty/libwebp/src/dsp/lossless.c +++ b/3rdparty/libwebp/src/dsp/lossless.c @@ -49,7 +49,7 @@ static WEBP_INLINE uint32_t Clip255(uint32_t a) { } static WEBP_INLINE int AddSubtractComponentFull(int a, int b, int c) { - return Clip255(a + b - c); + return Clip255((uint32_t)(a + b - c)); } static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, @@ -66,7 +66,7 @@ static WEBP_INLINE uint32_t ClampedAddSubtractFull(uint32_t c0, uint32_t c1, } static WEBP_INLINE int AddSubtractComponentHalf(int a, int b) { - return Clip255(a + (a - b) / 2); + return Clip255((uint32_t)(a + (a - b) / 2)); } static WEBP_INLINE uint32_t ClampedAddSubtractHalf(uint32_t c0, uint32_t c1, @@ -107,63 +107,77 @@ static WEBP_INLINE uint32_t Select(uint32_t a, uint32_t b, uint32_t c) { //------------------------------------------------------------------------------ // Predictors -uint32_t VP8LPredictor0_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor0_C(const uint32_t* const left, + const uint32_t* const top) { (void)top; (void)left; return ARGB_BLACK; } -uint32_t VP8LPredictor1_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor1_C(const uint32_t* const left, + const uint32_t* const top) { (void)top; - return left; + return *left; } -uint32_t VP8LPredictor2_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor2_C(const uint32_t* const left, + const uint32_t* const top) { (void)left; return top[0]; } -uint32_t VP8LPredictor3_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor3_C(const uint32_t* const left, + const uint32_t* const top) { (void)left; return top[1]; } -uint32_t VP8LPredictor4_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor4_C(const uint32_t* const left, + const uint32_t* const top) { (void)left; return top[-1]; } -uint32_t VP8LPredictor5_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average3(left, top[0], top[1]); +uint32_t VP8LPredictor5_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average3(*left, top[0], top[1]); return pred; } -uint32_t VP8LPredictor6_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average2(left, top[-1]); +uint32_t VP8LPredictor6_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average2(*left, top[-1]); return pred; } -uint32_t VP8LPredictor7_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average2(left, top[0]); +uint32_t VP8LPredictor7_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average2(*left, top[0]); return pred; } -uint32_t VP8LPredictor8_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor8_C(const uint32_t* const left, + const uint32_t* const top) { const uint32_t pred = Average2(top[-1], top[0]); (void)left; return pred; } -uint32_t VP8LPredictor9_C(uint32_t left, const uint32_t* const top) { +uint32_t VP8LPredictor9_C(const uint32_t* const left, + const uint32_t* const top) { const uint32_t pred = Average2(top[0], top[1]); (void)left; return pred; } -uint32_t VP8LPredictor10_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average4(left, top[-1], top[0], top[1]); +uint32_t VP8LPredictor10_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average4(*left, top[-1], top[0], top[1]); return pred; } -uint32_t VP8LPredictor11_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Select(top[0], left, top[-1]); +uint32_t VP8LPredictor11_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Select(top[0], *left, top[-1]); return pred; } -uint32_t VP8LPredictor12_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = ClampedAddSubtractFull(left, top[0], top[-1]); +uint32_t VP8LPredictor12_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractFull(*left, top[0], top[-1]); return pred; } -uint32_t VP8LPredictor13_C(uint32_t left, const uint32_t* const top) { - const uint32_t pred = ClampedAddSubtractHalf(left, top[0], top[-1]); +uint32_t VP8LPredictor13_C(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractHalf(*left, top[0], top[-1]); return pred; } @@ -279,10 +293,10 @@ void VP8LTransformColorInverse_C(const VP8LMultipliers* const m, const uint32_t red = argb >> 16; int new_red = red & 0xff; int new_blue = argb & 0xff; - new_red += ColorTransformDelta(m->green_to_red_, green); + new_red += ColorTransformDelta((int8_t)m->green_to_red_, green); new_red &= 0xff; - new_blue += ColorTransformDelta(m->green_to_blue_, green); - new_blue += ColorTransformDelta(m->red_to_blue_, (int8_t)new_red); + new_blue += ColorTransformDelta((int8_t)m->green_to_blue_, green); + new_blue += ColorTransformDelta((int8_t)m->red_to_blue_, (int8_t)new_red); new_blue &= 0xff; dst[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); } @@ -381,7 +395,7 @@ void VP8LInverseTransform(const VP8LTransform* const transform, assert(row_start < row_end); assert(row_end <= transform->ysize_); switch (transform->type_) { - case SUBTRACT_GREEN: + case SUBTRACT_GREEN_TRANSFORM: VP8LAddGreenToBlueAndRed(in, (row_end - row_start) * width, out); break; case PREDICTOR_TRANSFORM: @@ -574,7 +588,9 @@ VP8LConvertFunc VP8LConvertBGRAToBGR; VP8LMapARGBFunc VP8LMapColor32b; VP8LMapAlphaFunc VP8LMapColor8b; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8LDspInitSSE2(void); +extern void VP8LDspInitSSE41(void); extern void VP8LDspInitNEON(void); extern void VP8LDspInitMIPSdspR2(void); extern void VP8LDspInitMSA(void); @@ -621,9 +637,14 @@ WEBP_DSP_INIT_FUNC(VP8LDspInit) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8LDspInitSSE2(); +#if defined(WEBP_HAVE_SSE41) + if (VP8GetCPUInfo(kSSE4_1)) { + VP8LDspInitSSE41(); + } +#endif } #endif #if defined(WEBP_USE_MIPS_DSP_R2) @@ -638,7 +659,7 @@ WEBP_DSP_INIT_FUNC(VP8LDspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8LDspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/lossless.h b/3rdparty/libwebp/src/dsp/lossless.h index ebd316d1ed..0bf10a1a3d 100644 --- a/3rdparty/libwebp/src/dsp/lossless.h +++ b/3rdparty/libwebp/src/dsp/lossless.h @@ -28,23 +28,38 @@ extern "C" { //------------------------------------------------------------------------------ // Decoding -typedef uint32_t (*VP8LPredictorFunc)(uint32_t left, const uint32_t* const top); +typedef uint32_t (*VP8LPredictorFunc)(const uint32_t* const left, + const uint32_t* const top); extern VP8LPredictorFunc VP8LPredictors[16]; -uint32_t VP8LPredictor0_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor1_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor2_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor3_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor4_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor5_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor6_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor7_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor8_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor9_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor10_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor11_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor12_C(uint32_t left, const uint32_t* const top); -uint32_t VP8LPredictor13_C(uint32_t left, const uint32_t* const top); +uint32_t VP8LPredictor0_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor1_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor2_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor3_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor4_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor5_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor6_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor7_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor8_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor9_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor10_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor11_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor12_C(const uint32_t* const left, + const uint32_t* const top); +uint32_t VP8LPredictor13_C(const uint32_t* const left, + const uint32_t* const top); // These Add/Sub function expects upper[-1] and out[-1] to be readable. typedef void (*VP8LPredictorAddSubFunc)(const uint32_t* in, @@ -167,9 +182,9 @@ extern VP8LPredictorAddSubFunc VP8LPredictorsSub_C[16]; // ----------------------------------------------------------------------------- // Huffman-cost related functions. -typedef double (*VP8LCostFunc)(const uint32_t* population, int length); -typedef double (*VP8LCostCombinedFunc)(const uint32_t* X, const uint32_t* Y, - int length); +typedef uint32_t (*VP8LCostFunc)(const uint32_t* population, int length); +typedef uint32_t (*VP8LCostCombinedFunc)(const uint32_t* X, const uint32_t* Y, + int length); typedef float (*VP8LCombinedShannonEntropyFunc)(const int X[256], const int Y[256]); @@ -183,7 +198,7 @@ typedef struct { // small struct to hold counters } VP8LStreaks; typedef struct { // small struct to hold bit entropy results - double entropy; // entropy + float entropy; // entropy uint32_t sum; // sum of the population int nonzeros; // number of non-zero elements in the population uint32_t max_val; // maximum value in the population diff --git a/3rdparty/libwebp/src/dsp/lossless_common.h b/3rdparty/libwebp/src/dsp/lossless_common.h index 96a106f9ee..d6139b2b57 100644 --- a/3rdparty/libwebp/src/dsp/lossless_common.h +++ b/3rdparty/libwebp/src/dsp/lossless_common.h @@ -16,9 +16,9 @@ #ifndef WEBP_DSP_LOSSLESS_COMMON_H_ #define WEBP_DSP_LOSSLESS_COMMON_H_ -#include "src/webp/types.h" - +#include "src/dsp/cpu.h" #include "src/utils/utils.h" +#include "src/webp/types.h" #ifdef __cplusplus extern "C" { @@ -166,7 +166,7 @@ uint32_t VP8LSubPixels(uint32_t a, uint32_t b) { } //------------------------------------------------------------------------------ -// Transform-related functions use din both encoding and decoding. +// Transform-related functions used in both encoding and decoding. // Macros used to create a batch predictor that iteratively uses a // one-pixel predictor. @@ -179,7 +179,7 @@ static void PREDICTOR_ADD(const uint32_t* in, const uint32_t* upper, \ int x; \ assert(upper != NULL); \ for (x = 0; x < num_pixels; ++x) { \ - const uint32_t pred = (PREDICTOR)(out[x - 1], upper + x); \ + const uint32_t pred = (PREDICTOR)(&out[x - 1], upper + x); \ out[x] = VP8LAddPixels(in[x], pred); \ } \ } diff --git a/3rdparty/libwebp/src/dsp/lossless_enc.c b/3rdparty/libwebp/src/dsp/lossless_enc.c index a0c7ab9117..997d56c2ad 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc.c @@ -329,6 +329,15 @@ const uint8_t kPrefixEncodeExtraBitsValue[PREFIX_LOOKUP_IDX_MAX] = { static float FastSLog2Slow_C(uint32_t v) { assert(v >= LOG_LOOKUP_IDX_MAX); if (v < APPROX_LOG_WITH_CORRECTION_MAX) { +#if !defined(WEBP_HAVE_SLOW_CLZ_CTZ) + // use clz if available + const int log_cnt = BitsLog2Floor(v) - 7; + const uint32_t y = 1 << log_cnt; + int correction = 0; + const float v_f = (float)v; + const uint32_t orig_v = v; + v >>= log_cnt; +#else int log_cnt = 0; uint32_t y = 1; int correction = 0; @@ -339,6 +348,7 @@ static float FastSLog2Slow_C(uint32_t v) { v = v >> 1; y = y << 1; } while (v >= LOG_LOOKUP_IDX_MAX); +#endif // vf = (2^log_cnt) * Xf; where y = 2^log_cnt and Xf < 256 // Xf = floor(Xf) * (1 + (v % y) / v) // log2(Xf) = log2(floor(Xf)) + log2(1 + (v % y) / v) @@ -355,6 +365,14 @@ static float FastSLog2Slow_C(uint32_t v) { static float FastLog2Slow_C(uint32_t v) { assert(v >= LOG_LOOKUP_IDX_MAX); if (v < APPROX_LOG_WITH_CORRECTION_MAX) { +#if !defined(WEBP_HAVE_SLOW_CLZ_CTZ) + // use clz if available + const int log_cnt = BitsLog2Floor(v) - 7; + const uint32_t y = 1 << log_cnt; + const uint32_t orig_v = v; + double log_2; + v >>= log_cnt; +#else int log_cnt = 0; uint32_t y = 1; const uint32_t orig_v = v; @@ -364,6 +382,7 @@ static float FastLog2Slow_C(uint32_t v) { v = v >> 1; y = y << 1; } while (v >= LOG_LOOKUP_IDX_MAX); +#endif log_2 = kLog2Table[v] + log_cnt; if (orig_v >= APPROX_LOG_MAX) { // Since the division is still expensive, add this correction factor only @@ -383,7 +402,7 @@ static float FastLog2Slow_C(uint32_t v) { // Compute the combined Shanon's entropy for distribution {X} and {X+Y} static float CombinedShannonEntropy_C(const int X[256], const int Y[256]) { int i; - double retval = 0.; + float retval = 0.f; int sumX = 0, sumXY = 0; for (i = 0; i < 256; ++i) { const int x = X[i]; @@ -399,7 +418,7 @@ static float CombinedShannonEntropy_C(const int X[256], const int Y[256]) { } } retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); - return (float)retval; + return retval; } void VP8LBitEntropyInit(VP8LBitEntropy* const entropy) { @@ -503,11 +522,11 @@ static void GetCombinedEntropyUnrefined_C(const uint32_t X[], void VP8LSubtractGreenFromBlueAndRed_C(uint32_t* argb_data, int num_pixels) { int i; for (i = 0; i < num_pixels; ++i) { - const int argb = argb_data[i]; + const int argb = (int)argb_data[i]; const int green = (argb >> 8) & 0xff; const uint32_t new_r = (((argb >> 16) & 0xff) - green) & 0xff; const uint32_t new_b = (((argb >> 0) & 0xff) - green) & 0xff; - argb_data[i] = (argb & 0xff00ff00u) | (new_r << 16) | new_b; + argb_data[i] = ((uint32_t)argb & 0xff00ff00u) | (new_r << 16) | new_b; } } @@ -528,10 +547,10 @@ void VP8LTransformColor_C(const VP8LMultipliers* const m, uint32_t* data, const int8_t red = U32ToS8(argb >> 16); int new_red = red & 0xff; int new_blue = argb & 0xff; - new_red -= ColorTransformDelta(m->green_to_red_, green); + new_red -= ColorTransformDelta((int8_t)m->green_to_red_, green); new_red &= 0xff; - new_blue -= ColorTransformDelta(m->green_to_blue_, green); - new_blue -= ColorTransformDelta(m->red_to_blue_, red); + new_blue -= ColorTransformDelta((int8_t)m->green_to_blue_, green); + new_blue -= ColorTransformDelta((int8_t)m->red_to_blue_, red); new_blue &= 0xff; data[i] = (argb & 0xff00ff00u) | (new_red << 16) | (new_blue); } @@ -541,7 +560,7 @@ static WEBP_INLINE uint8_t TransformColorRed(uint8_t green_to_red, uint32_t argb) { const int8_t green = U32ToS8(argb >> 8); int new_red = argb >> 16; - new_red -= ColorTransformDelta(green_to_red, green); + new_red -= ColorTransformDelta((int8_t)green_to_red, green); return (new_red & 0xff); } @@ -550,9 +569,9 @@ static WEBP_INLINE uint8_t TransformColorBlue(uint8_t green_to_blue, uint32_t argb) { const int8_t green = U32ToS8(argb >> 8); const int8_t red = U32ToS8(argb >> 16); - uint8_t new_blue = argb & 0xff; - new_blue -= ColorTransformDelta(green_to_blue, green); - new_blue -= ColorTransformDelta(red_to_blue, red); + int new_blue = argb & 0xff; + new_blue -= ColorTransformDelta((int8_t)green_to_blue, green); + new_blue -= ColorTransformDelta((int8_t)red_to_blue, red); return (new_blue & 0xff); } @@ -617,20 +636,25 @@ void VP8LBundleColorMap_C(const uint8_t* const row, int width, int xbits, //------------------------------------------------------------------------------ -static double ExtraCost_C(const uint32_t* population, int length) { +static uint32_t ExtraCost_C(const uint32_t* population, int length) { int i; - double cost = 0.; - for (i = 2; i < length - 2; ++i) cost += (i >> 1) * population[i + 2]; + uint32_t cost = population[4] + population[5]; + assert(length % 2 == 0); + for (i = 2; i < length / 2 - 1; ++i) { + cost += i * (population[2 * i + 2] + population[2 * i + 3]); + } return cost; } -static double ExtraCostCombined_C(const uint32_t* X, const uint32_t* Y, - int length) { +static uint32_t ExtraCostCombined_C(const uint32_t* X, const uint32_t* Y, + int length) { int i; - double cost = 0.; - for (i = 2; i < length - 2; ++i) { - const int xy = X[i + 2] + Y[i + 2]; - cost += (i >> 1) * xy; + uint32_t cost = X[4] + Y[4] + X[5] + Y[5]; + assert(length % 2 == 0); + for (i = 2; i < length / 2 - 1; ++i) { + const int xy0 = X[2 * i + 2] + Y[2 * i + 2]; + const int xy1 = X[2 * i + 3] + Y[2 * i + 3]; + cost += i * (xy0 + xy1); } return cost; } @@ -726,7 +750,7 @@ static void PredictorSub##PREDICTOR_I##_C(const uint32_t* in, \ assert(upper != NULL); \ for (x = 0; x < num_pixels; ++x) { \ const uint32_t pred = \ - VP8LPredictor##PREDICTOR_I##_C(in[x - 1], upper + x); \ + VP8LPredictor##PREDICTOR_I##_C(&in[x - 1], upper + x); \ out[x] = VP8LSubPixels(in[x], pred); \ } \ } @@ -772,6 +796,7 @@ VP8LBundleColorMapFunc VP8LBundleColorMap; VP8LPredictorAddSubFunc VP8LPredictorsSub[16]; VP8LPredictorAddSubFunc VP8LPredictorsSub_C[16]; +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8LEncDspInitSSE2(void); extern void VP8LEncDspInitSSE41(void); extern void VP8LEncDspInitNEON(void); @@ -843,10 +868,10 @@ WEBP_DSP_INIT_FUNC(VP8LEncDspInit) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8LEncDspInitSSE2(); -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { VP8LEncDspInitSSE41(); } @@ -870,7 +895,7 @@ WEBP_DSP_INIT_FUNC(VP8LEncDspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { VP8LEncDspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c b/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c index 0412a093cf..e10f12da9d 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc_mips32.c @@ -103,8 +103,8 @@ static float FastLog2Slow_MIPS32(uint32_t v) { // cost += i * *(pop + 1); // pop += 2; // } -// return (double)cost; -static double ExtraCost_MIPS32(const uint32_t* const population, int length) { +// return cost; +static uint32_t ExtraCost_MIPS32(const uint32_t* const population, int length) { int i, temp0, temp1; const uint32_t* pop = &population[4]; const uint32_t* const LoopEnd = &population[length]; @@ -130,7 +130,7 @@ static double ExtraCost_MIPS32(const uint32_t* const population, int length) { : "memory", "hi", "lo" ); - return (double)((int64_t)temp0 << 32 | temp1); + return ((int64_t)temp0 << 32 | temp1); } // C version of this function: @@ -148,9 +148,9 @@ static double ExtraCost_MIPS32(const uint32_t* const population, int length) { // pX += 2; // pY += 2; // } -// return (double)cost; -static double ExtraCostCombined_MIPS32(const uint32_t* const X, - const uint32_t* const Y, int length) { +// return cost; +static uint32_t ExtraCostCombined_MIPS32(const uint32_t* const X, + const uint32_t* const Y, int length) { int i, temp0, temp1, temp2, temp3; const uint32_t* pX = &X[4]; const uint32_t* pY = &Y[4]; @@ -183,7 +183,7 @@ static double ExtraCostCombined_MIPS32(const uint32_t* const X, : "memory", "hi", "lo" ); - return (double)((int64_t)temp0 << 32 | temp1); + return ((int64_t)temp0 << 32 | temp1); } #define HUFFMAN_COST_PASS \ @@ -347,24 +347,24 @@ static void GetCombinedEntropyUnrefined_MIPS32(const uint32_t X[], static void AddVector_MIPS32(const uint32_t* pa, const uint32_t* pb, uint32_t* pout, int size) { uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; - const uint32_t end = ((size) / 4) * 4; + const int end = ((size) / 4) * 4; const uint32_t* const LoopEnd = pa + end; int i; ASM_START ADD_TO_OUT(0, 4, 8, 12, 1, pa, pb, pout) ASM_END_0 - for (i = end; i < size; ++i) pout[i] = pa[i] + pb[i]; + for (i = 0; i < size - end; ++i) pout[i] = pa[i] + pb[i]; } static void AddVectorEq_MIPS32(const uint32_t* pa, uint32_t* pout, int size) { uint32_t temp0, temp1, temp2, temp3, temp4, temp5, temp6, temp7; - const uint32_t end = ((size) / 4) * 4; + const int end = ((size) / 4) * 4; const uint32_t* const LoopEnd = pa + end; int i; ASM_START ADD_TO_OUT(0, 4, 8, 12, 0, pa, pout, pout) ASM_END_1 - for (i = end; i < size; ++i) pout[i] += pa[i]; + for (i = 0; i < size - end; ++i) pout[i] += pa[i]; } #undef ASM_END_1 diff --git a/3rdparty/libwebp/src/dsp/lossless_enc_neon.c b/3rdparty/libwebp/src/dsp/lossless_enc_neon.c index 7c7b73f8b6..e32c7961a2 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc_neon.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc_neon.c @@ -25,7 +25,7 @@ // vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use // non-standard versions there. -#if defined(__APPLE__) && defined(__aarch64__) && \ +#if defined(__APPLE__) && WEBP_AARCH64 && \ defined(__apple_build_version__) && (__apple_build_version__< 6020037) #define USE_VTBLQ #endif diff --git a/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c b/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c index 90c263735f..66cbaab772 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc_sse2.c @@ -54,8 +54,8 @@ static void TransformColor_SSE2(const VP8LMultipliers* const m, const __m128i mults_rb = MK_CST_16(CST_5b(m->green_to_red_), CST_5b(m->green_to_blue_)); const __m128i mults_b2 = MK_CST_16(CST_5b(m->red_to_blue_), 0); - const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks - const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks + const __m128i mask_ag = _mm_set1_epi32((int)0xff00ff00); // alpha-green masks + const __m128i mask_rb = _mm_set1_epi32(0x00ff00ff); // red-blue masks int i; for (i = 0; i + 4 <= num_pixels; i += 4) { const __m128i in = _mm_loadu_si128((__m128i*)&argb_data[i]); // argb @@ -232,79 +232,55 @@ static void AddVectorEq_SSE2(const uint32_t* a, uint32_t* out, int size) { //------------------------------------------------------------------------------ // Entropy -// Checks whether the X or Y contribution is worth computing and adding. -// Used in loop unrolling. -#define ANALYZE_X_OR_Y(x_or_y, j) \ - do { \ - if ((x_or_y)[i + (j)] != 0) retval -= VP8LFastSLog2((x_or_y)[i + (j)]); \ - } while (0) +// TODO(https://crbug.com/webp/499): this function produces different results +// from the C code due to use of double/float resulting in output differences +// when compared to -noasm. +#if !(defined(WEBP_HAVE_SLOW_CLZ_CTZ) || defined(__i386__) || defined(_M_IX86)) -// Checks whether the X + Y contribution is worth computing and adding. -// Used in loop unrolling. -#define ANALYZE_XY(j) \ - do { \ - if (tmp[j] != 0) { \ - retval -= VP8LFastSLog2(tmp[j]); \ - ANALYZE_X_OR_Y(X, j); \ - } \ - } while (0) - -#if !(defined(__i386__) || defined(_M_IX86)) static float CombinedShannonEntropy_SSE2(const int X[256], const int Y[256]) { int i; - double retval = 0.; - int sumX, sumXY; - int32_t tmp[4]; - __m128i zero = _mm_setzero_si128(); - // Sums up X + Y, 4 ints at a time (and will merge it at the end for sumXY). - __m128i sumXY_128 = zero; - __m128i sumX_128 = zero; + float retval = 0.f; + int sumX = 0, sumXY = 0; + const __m128i zero = _mm_setzero_si128(); - for (i = 0; i < 256; i += 4) { - const __m128i x = _mm_loadu_si128((const __m128i*)(X + i)); - const __m128i y = _mm_loadu_si128((const __m128i*)(Y + i)); - - // Check if any X is non-zero: this actually provides a speedup as X is - // usually sparse. - if (_mm_movemask_epi8(_mm_cmpeq_epi32(x, zero)) != 0xFFFF) { - const __m128i xy_128 = _mm_add_epi32(x, y); - sumXY_128 = _mm_add_epi32(sumXY_128, xy_128); - - sumX_128 = _mm_add_epi32(sumX_128, x); - - // Analyze the different X + Y. - _mm_storeu_si128((__m128i*)tmp, xy_128); - - ANALYZE_XY(0); - ANALYZE_XY(1); - ANALYZE_XY(2); - ANALYZE_XY(3); - } else { - // X is fully 0, so only deal with Y. - sumXY_128 = _mm_add_epi32(sumXY_128, y); - - ANALYZE_X_OR_Y(Y, 0); - ANALYZE_X_OR_Y(Y, 1); - ANALYZE_X_OR_Y(Y, 2); - ANALYZE_X_OR_Y(Y, 3); + for (i = 0; i < 256; i += 16) { + const __m128i x0 = _mm_loadu_si128((const __m128i*)(X + i + 0)); + const __m128i y0 = _mm_loadu_si128((const __m128i*)(Y + i + 0)); + const __m128i x1 = _mm_loadu_si128((const __m128i*)(X + i + 4)); + const __m128i y1 = _mm_loadu_si128((const __m128i*)(Y + i + 4)); + const __m128i x2 = _mm_loadu_si128((const __m128i*)(X + i + 8)); + const __m128i y2 = _mm_loadu_si128((const __m128i*)(Y + i + 8)); + const __m128i x3 = _mm_loadu_si128((const __m128i*)(X + i + 12)); + const __m128i y3 = _mm_loadu_si128((const __m128i*)(Y + i + 12)); + const __m128i x4 = _mm_packs_epi16(_mm_packs_epi32(x0, x1), + _mm_packs_epi32(x2, x3)); + const __m128i y4 = _mm_packs_epi16(_mm_packs_epi32(y0, y1), + _mm_packs_epi32(y2, y3)); + const int32_t mx = _mm_movemask_epi8(_mm_cmpgt_epi8(x4, zero)); + int32_t my = _mm_movemask_epi8(_mm_cmpgt_epi8(y4, zero)) | mx; + while (my) { + const int32_t j = BitsCtz(my); + int xy; + if ((mx >> j) & 1) { + const int x = X[i + j]; + sumXY += x; + retval -= VP8LFastSLog2(x); + } + xy = X[i + j] + Y[i + j]; + sumX += xy; + retval -= VP8LFastSLog2(xy); + my &= my - 1; } } - - // Sum up sumX_128 to get sumX. - _mm_storeu_si128((__m128i*)tmp, sumX_128); - sumX = tmp[3] + tmp[2] + tmp[1] + tmp[0]; - - // Sum up sumXY_128 to get sumXY. - _mm_storeu_si128((__m128i*)tmp, sumXY_128); - sumXY = tmp[3] + tmp[2] + tmp[1] + tmp[0]; - retval += VP8LFastSLog2(sumX) + VP8LFastSLog2(sumXY); - return (float)retval; + return retval; } -#endif // !(defined(__i386__) || defined(_M_IX86)) -#undef ANALYZE_X_OR_Y -#undef ANALYZE_XY +#else + +#define DONT_USE_COMBINED_SHANNON_ENTROPY_SSE2_FUNC // won't be faster + +#endif //------------------------------------------------------------------------------ @@ -400,7 +376,7 @@ static void BundleColorMap_SSE2(const uint8_t* const row, int width, int xbits, break; } case 2: { - const __m128i mask_or = _mm_set1_epi32(0xff000000); + const __m128i mask_or = _mm_set1_epi32((int)0xff000000); const __m128i mul_cst = _mm_set1_epi16(0x0104); const __m128i mask_mul = _mm_set1_epi16(0x0f00); for (x = 0; x + 16 <= width; x += 16, dst += 4) { @@ -451,7 +427,7 @@ static WEBP_INLINE void Average2_m128i(const __m128i* const a0, static void PredictorSub0_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; - const __m128i black = _mm_set1_epi32(ARGB_BLACK); + const __m128i black = _mm_set1_epi32((int)ARGB_BLACK); for (i = 0; i + 4 <= num_pixels; i += 4) { const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); const __m128i res = _mm_sub_epi8(src, black); @@ -662,10 +638,7 @@ WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE2(void) { VP8LCollectColorRedTransforms = CollectColorRedTransforms_SSE2; VP8LAddVector = AddVector_SSE2; VP8LAddVectorEq = AddVectorEq_SSE2; - // TODO(https://crbug.com/webp/499): this function produces different results - // from the C code due to use of double/float resulting in output differences - // when compared to -noasm. -#if !(defined(__i386__) || defined(_M_IX86)) +#if !defined(DONT_USE_COMBINED_SHANNON_ENTROPY_SSE2_FUNC) VP8LCombinedShannonEntropy = CombinedShannonEntropy_SSE2; #endif VP8LVectorMismatch = VectorMismatch_SSE2; diff --git a/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c b/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c index 719d8ed25e..7ab83c2604 100644 --- a/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c +++ b/3rdparty/libwebp/src/dsp/lossless_enc_sse41.c @@ -18,8 +18,53 @@ #include #include "src/dsp/lossless.h" -// For sign-extended multiplying constants, pre-shifted by 5: -#define CST_5b(X) (((int16_t)((uint16_t)(X) << 8)) >> 5) +//------------------------------------------------------------------------------ +// Cost operations. + +static WEBP_INLINE uint32_t HorizontalSum_SSE41(__m128i cost) { + cost = _mm_add_epi32(cost, _mm_srli_si128(cost, 8)); + cost = _mm_add_epi32(cost, _mm_srli_si128(cost, 4)); + return _mm_cvtsi128_si32(cost); +} + +static uint32_t ExtraCost_SSE41(const uint32_t* const a, int length) { + int i; + __m128i cost = _mm_set_epi32(2 * a[7], 2 * a[6], a[5], a[4]); + assert(length % 8 == 0); + + for (i = 8; i + 8 <= length; i += 8) { + const int j = (i - 2) >> 1; + const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i]); + const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); + const __m128i w = _mm_set_epi32(j + 3, j + 2, j + 1, j); + const __m128i a2 = _mm_hadd_epi32(a0, a1); + const __m128i mul = _mm_mullo_epi32(a2, w); + cost = _mm_add_epi32(mul, cost); + } + return HorizontalSum_SSE41(cost); +} + +static uint32_t ExtraCostCombined_SSE41(const uint32_t* const a, + const uint32_t* const b, int length) { + int i; + __m128i cost = _mm_add_epi32(_mm_set_epi32(2 * a[7], 2 * a[6], a[5], a[4]), + _mm_set_epi32(2 * b[7], 2 * b[6], b[5], b[4])); + assert(length % 8 == 0); + + for (i = 8; i + 8 <= length; i += 8) { + const int j = (i - 2) >> 1; + const __m128i a0 = _mm_loadu_si128((const __m128i*)&a[i]); + const __m128i a1 = _mm_loadu_si128((const __m128i*)&a[i + 4]); + const __m128i b0 = _mm_loadu_si128((const __m128i*)&b[i]); + const __m128i b1 = _mm_loadu_si128((const __m128i*)&b[i + 4]); + const __m128i w = _mm_set_epi32(j + 3, j + 2, j + 1, j); + const __m128i a2 = _mm_hadd_epi32(a0, a1); + const __m128i b2 = _mm_hadd_epi32(b0, b1); + const __m128i mul = _mm_mullo_epi32(_mm_add_epi32(a2, b2), w); + cost = _mm_add_epi32(mul, cost); + } + return HorizontalSum_SSE41(cost); +} //------------------------------------------------------------------------------ // Subtract-Green Transform @@ -44,46 +89,50 @@ static void SubtractGreenFromBlueAndRed_SSE41(uint32_t* argb_data, //------------------------------------------------------------------------------ // Color Transform -#define SPAN 8 +// For sign-extended multiplying constants, pre-shifted by 5: +#define CST_5b(X) (((int16_t)((uint16_t)(X) << 8)) >> 5) + +#define MK_CST_16(HI, LO) \ + _mm_set1_epi32((int)(((uint32_t)(HI) << 16) | ((LO) & 0xffff))) + static void CollectColorBlueTransforms_SSE41(const uint32_t* argb, int stride, int tile_width, int tile_height, int green_to_blue, int red_to_blue, int histo[]) { - const __m128i mults_r = _mm_set1_epi16(CST_5b(red_to_blue)); - const __m128i mults_g = _mm_set1_epi16(CST_5b(green_to_blue)); - const __m128i mask_g = _mm_set1_epi16((short)0xff00); // green mask - const __m128i mask_gb = _mm_set1_epi32(0xffff); // green/blue mask - const __m128i mask_b = _mm_set1_epi16(0x00ff); // blue mask - const __m128i shuffler_lo = _mm_setr_epi8(-1, 2, -1, 6, -1, 10, -1, 14, -1, - -1, -1, -1, -1, -1, -1, -1); - const __m128i shuffler_hi = _mm_setr_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, - 2, -1, 6, -1, 10, -1, 14); - int y; - for (y = 0; y < tile_height; ++y) { - const uint32_t* const src = argb + y * stride; - int i, x; - for (x = 0; x + SPAN <= tile_width; x += SPAN) { - uint16_t values[SPAN]; - const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); - const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); - const __m128i r0 = _mm_shuffle_epi8(in0, shuffler_lo); - const __m128i r1 = _mm_shuffle_epi8(in1, shuffler_hi); - const __m128i r = _mm_or_si128(r0, r1); // r 0 - const __m128i gb0 = _mm_and_si128(in0, mask_gb); - const __m128i gb1 = _mm_and_si128(in1, mask_gb); - const __m128i gb = _mm_packus_epi32(gb0, gb1); // g b - const __m128i g = _mm_and_si128(gb, mask_g); // g 0 - const __m128i A = _mm_mulhi_epi16(r, mults_r); // x dbr - const __m128i B = _mm_mulhi_epi16(g, mults_g); // x dbg - const __m128i C = _mm_sub_epi8(gb, B); // x b' - const __m128i D = _mm_sub_epi8(C, A); // x b'' - const __m128i E = _mm_and_si128(D, mask_b); // 0 b'' - _mm_storeu_si128((__m128i*)values, E); - for (i = 0; i < SPAN; ++i) ++histo[values[i]]; + const __m128i mult = + MK_CST_16(CST_5b(red_to_blue) + 256,CST_5b(green_to_blue)); + const __m128i perm = + _mm_setr_epi8(-1, 1, -1, 2, -1, 5, -1, 6, -1, 9, -1, 10, -1, 13, -1, 14); + if (tile_width >= 4) { + int y; + for (y = 0; y < tile_height; ++y) { + const uint32_t* const src = argb + y * stride; + const __m128i A1 = _mm_loadu_si128((const __m128i*)src); + const __m128i B1 = _mm_shuffle_epi8(A1, perm); + const __m128i C1 = _mm_mulhi_epi16(B1, mult); + const __m128i D1 = _mm_sub_epi16(A1, C1); + __m128i E = _mm_add_epi16(_mm_srli_epi32(D1, 16), D1); + int x; + for (x = 4; x + 4 <= tile_width; x += 4) { + const __m128i A2 = _mm_loadu_si128((const __m128i*)(src + x)); + __m128i B2, C2, D2; + ++histo[_mm_extract_epi8(E, 0)]; + B2 = _mm_shuffle_epi8(A2, perm); + ++histo[_mm_extract_epi8(E, 4)]; + C2 = _mm_mulhi_epi16(B2, mult); + ++histo[_mm_extract_epi8(E, 8)]; + D2 = _mm_sub_epi16(A2, C2); + ++histo[_mm_extract_epi8(E, 12)]; + E = _mm_add_epi16(_mm_srli_epi32(D2, 16), D2); + } + ++histo[_mm_extract_epi8(E, 0)]; + ++histo[_mm_extract_epi8(E, 4)]; + ++histo[_mm_extract_epi8(E, 8)]; + ++histo[_mm_extract_epi8(E, 12)]; } } { - const int left_over = tile_width & (SPAN - 1); + const int left_over = tile_width & 3; if (left_over > 0) { VP8LCollectColorBlueTransforms_C(argb + tile_width - left_over, stride, left_over, tile_height, @@ -95,33 +144,37 @@ static void CollectColorBlueTransforms_SSE41(const uint32_t* argb, int stride, static void CollectColorRedTransforms_SSE41(const uint32_t* argb, int stride, int tile_width, int tile_height, int green_to_red, int histo[]) { - const __m128i mults_g = _mm_set1_epi16(CST_5b(green_to_red)); - const __m128i mask_g = _mm_set1_epi32(0x00ff00); // green mask - const __m128i mask = _mm_set1_epi16(0xff); - int y; - for (y = 0; y < tile_height; ++y) { - const uint32_t* const src = argb + y * stride; - int i, x; - for (x = 0; x + SPAN <= tile_width; x += SPAN) { - uint16_t values[SPAN]; - const __m128i in0 = _mm_loadu_si128((__m128i*)&src[x + 0]); - const __m128i in1 = _mm_loadu_si128((__m128i*)&src[x + SPAN / 2]); - const __m128i g0 = _mm_and_si128(in0, mask_g); // 0 0 | g 0 - const __m128i g1 = _mm_and_si128(in1, mask_g); - const __m128i g = _mm_packus_epi32(g0, g1); // g 0 - const __m128i A0 = _mm_srli_epi32(in0, 16); // 0 0 | x r - const __m128i A1 = _mm_srli_epi32(in1, 16); - const __m128i A = _mm_packus_epi32(A0, A1); // x r - const __m128i B = _mm_mulhi_epi16(g, mults_g); // x dr - const __m128i C = _mm_sub_epi8(A, B); // x r' - const __m128i D = _mm_and_si128(C, mask); // 0 r' - _mm_storeu_si128((__m128i*)values, D); - for (i = 0; i < SPAN; ++i) ++histo[values[i]]; + const __m128i mult = MK_CST_16(0, CST_5b(green_to_red)); + const __m128i mask_g = _mm_set1_epi32(0x0000ff00); + if (tile_width >= 4) { + int y; + for (y = 0; y < tile_height; ++y) { + const uint32_t* const src = argb + y * stride; + const __m128i A1 = _mm_loadu_si128((const __m128i*)src); + const __m128i B1 = _mm_and_si128(A1, mask_g); + const __m128i C1 = _mm_madd_epi16(B1, mult); + __m128i D = _mm_sub_epi16(A1, C1); + int x; + for (x = 4; x + 4 <= tile_width; x += 4) { + const __m128i A2 = _mm_loadu_si128((const __m128i*)(src + x)); + __m128i B2, C2; + ++histo[_mm_extract_epi8(D, 2)]; + B2 = _mm_and_si128(A2, mask_g); + ++histo[_mm_extract_epi8(D, 6)]; + C2 = _mm_madd_epi16(B2, mult); + ++histo[_mm_extract_epi8(D, 10)]; + ++histo[_mm_extract_epi8(D, 14)]; + D = _mm_sub_epi16(A2, C2); + } + ++histo[_mm_extract_epi8(D, 2)]; + ++histo[_mm_extract_epi8(D, 6)]; + ++histo[_mm_extract_epi8(D, 10)]; + ++histo[_mm_extract_epi8(D, 14)]; } } { - const int left_over = tile_width & (SPAN - 1); + const int left_over = tile_width & 3; if (left_over > 0) { VP8LCollectColorRedTransforms_C(argb + tile_width - left_over, stride, left_over, tile_height, green_to_red, @@ -130,12 +183,16 @@ static void CollectColorRedTransforms_SSE41(const uint32_t* argb, int stride, } } +#undef MK_CST_16 + //------------------------------------------------------------------------------ // Entry point extern void VP8LEncDspInitSSE41(void); WEBP_TSAN_IGNORE_FUNCTION void VP8LEncDspInitSSE41(void) { + VP8LExtraCost = ExtraCost_SSE41; + VP8LExtraCostCombined = ExtraCostCombined_SSE41; VP8LSubtractGreenFromBlueAndRed = SubtractGreenFromBlueAndRed_SSE41; VP8LCollectColorBlueTransforms = CollectColorBlueTransforms_SSE41; VP8LCollectColorRedTransforms = CollectColorRedTransforms_SSE41; diff --git a/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c b/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c index 9888854d57..bfe5ea6b38 100644 --- a/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c +++ b/3rdparty/libwebp/src/dsp/lossless_mips_dsp_r2.c @@ -188,46 +188,51 @@ static WEBP_INLINE uint32_t Average4(uint32_t a0, uint32_t a1, return Average2(Average2(a0, a1), Average2(a2, a3)); } -static uint32_t Predictor5_MIPSdspR2(uint32_t left, const uint32_t* const top) { - return Average3(left, top[0], top[1]); +static uint32_t Predictor5_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { + return Average3(*left, top[0], top[1]); } -static uint32_t Predictor6_MIPSdspR2(uint32_t left, const uint32_t* const top) { - return Average2(left, top[-1]); +static uint32_t Predictor6_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { + return Average2(*left, top[-1]); } -static uint32_t Predictor7_MIPSdspR2(uint32_t left, const uint32_t* const top) { - return Average2(left, top[0]); +static uint32_t Predictor7_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { + return Average2(*left, top[0]); } -static uint32_t Predictor8_MIPSdspR2(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor8_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { (void)left; return Average2(top[-1], top[0]); } -static uint32_t Predictor9_MIPSdspR2(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor9_MIPSdspR2(const uint32_t* const left, + const uint32_t* const top) { (void)left; return Average2(top[0], top[1]); } -static uint32_t Predictor10_MIPSdspR2(uint32_t left, +static uint32_t Predictor10_MIPSdspR2(const uint32_t* const left, const uint32_t* const top) { - return Average4(left, top[-1], top[0], top[1]); + return Average4(*left, top[-1], top[0], top[1]); } -static uint32_t Predictor11_MIPSdspR2(uint32_t left, +static uint32_t Predictor11_MIPSdspR2(const uint32_t* const left, const uint32_t* const top) { - return Select(top[0], left, top[-1]); + return Select(top[0], *left, top[-1]); } -static uint32_t Predictor12_MIPSdspR2(uint32_t left, +static uint32_t Predictor12_MIPSdspR2(const uint32_t* const left, const uint32_t* const top) { - return ClampedAddSubtractFull(left, top[0], top[-1]); + return ClampedAddSubtractFull(*left, top[0], top[-1]); } -static uint32_t Predictor13_MIPSdspR2(uint32_t left, +static uint32_t Predictor13_MIPSdspR2(const uint32_t* const left, const uint32_t* const top) { - return ClampedAddSubtractHalf(left, top[0], top[-1]); + return ClampedAddSubtractHalf(*left, top[0], top[-1]); } // Add green to blue and red channels (i.e. perform the inverse transform of diff --git a/3rdparty/libwebp/src/dsp/lossless_neon.c b/3rdparty/libwebp/src/dsp/lossless_neon.c index 76a1b6f873..ddc9b61711 100644 --- a/3rdparty/libwebp/src/dsp/lossless_neon.c +++ b/3rdparty/libwebp/src/dsp/lossless_neon.c @@ -188,17 +188,21 @@ static WEBP_INLINE uint32_t Average3_NEON(uint32_t a0, uint32_t a1, return avg; } -static uint32_t Predictor5_NEON(uint32_t left, const uint32_t* const top) { - return Average3_NEON(left, top[0], top[1]); +static uint32_t Predictor5_NEON(const uint32_t* const left, + const uint32_t* const top) { + return Average3_NEON(*left, top[0], top[1]); } -static uint32_t Predictor6_NEON(uint32_t left, const uint32_t* const top) { - return Average2_NEON(left, top[-1]); +static uint32_t Predictor6_NEON(const uint32_t* const left, + const uint32_t* const top) { + return Average2_NEON(*left, top[-1]); } -static uint32_t Predictor7_NEON(uint32_t left, const uint32_t* const top) { - return Average2_NEON(left, top[0]); +static uint32_t Predictor7_NEON(const uint32_t* const left, + const uint32_t* const top) { + return Average2_NEON(*left, top[0]); } -static uint32_t Predictor13_NEON(uint32_t left, const uint32_t* const top) { - return ClampedAddSubtractHalf_NEON(left, top[0], top[-1]); +static uint32_t Predictor13_NEON(const uint32_t* const left, + const uint32_t* const top) { + return ClampedAddSubtractHalf_NEON(*left, top[0], top[-1]); } // Batch versions of those functions. @@ -494,7 +498,7 @@ static void PredictorAdd13_NEON(const uint32_t* in, const uint32_t* upper, // vtbl?_u8 are marked unavailable for iOS arm64 with Xcode < 6.3, use // non-standard versions there. -#if defined(__APPLE__) && defined(__aarch64__) && \ +#if defined(__APPLE__) && WEBP_AARCH64 && \ defined(__apple_build_version__) && (__apple_build_version__< 6020037) #define USE_VTBLQ #endif diff --git a/3rdparty/libwebp/src/dsp/lossless_sse2.c b/3rdparty/libwebp/src/dsp/lossless_sse2.c index aef0cee1b3..4b6a532c23 100644 --- a/3rdparty/libwebp/src/dsp/lossless_sse2.c +++ b/3rdparty/libwebp/src/dsp/lossless_sse2.c @@ -18,7 +18,6 @@ #include "src/dsp/common_sse2.h" #include "src/dsp/lossless.h" #include "src/dsp/lossless_common.h" -#include #include //------------------------------------------------------------------------------ @@ -28,23 +27,22 @@ static WEBP_INLINE uint32_t ClampedAddSubtractFull_SSE2(uint32_t c0, uint32_t c1, uint32_t c2) { const __m128i zero = _mm_setzero_si128(); - const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c0), zero); - const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c1), zero); - const __m128i C2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c2), zero); + const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c0), zero); + const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c1), zero); + const __m128i C2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c2), zero); const __m128i V1 = _mm_add_epi16(C0, C1); const __m128i V2 = _mm_sub_epi16(V1, C2); const __m128i b = _mm_packus_epi16(V2, V2); - const uint32_t output = _mm_cvtsi128_si32(b); - return output; + return (uint32_t)_mm_cvtsi128_si32(b); } static WEBP_INLINE uint32_t ClampedAddSubtractHalf_SSE2(uint32_t c0, uint32_t c1, uint32_t c2) { const __m128i zero = _mm_setzero_si128(); - const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c0), zero); - const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c1), zero); - const __m128i B0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(c2), zero); + const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c0), zero); + const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c1), zero); + const __m128i B0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c2), zero); const __m128i avg = _mm_add_epi16(C1, C0); const __m128i A0 = _mm_srli_epi16(avg, 1); const __m128i A1 = _mm_sub_epi16(A0, B0); @@ -53,16 +51,15 @@ static WEBP_INLINE uint32_t ClampedAddSubtractHalf_SSE2(uint32_t c0, const __m128i A3 = _mm_srai_epi16(A2, 1); const __m128i A4 = _mm_add_epi16(A0, A3); const __m128i A5 = _mm_packus_epi16(A4, A4); - const uint32_t output = _mm_cvtsi128_si32(A5); - return output; + return (uint32_t)_mm_cvtsi128_si32(A5); } static WEBP_INLINE uint32_t Select_SSE2(uint32_t a, uint32_t b, uint32_t c) { int pa_minus_pb; const __m128i zero = _mm_setzero_si128(); - const __m128i A0 = _mm_cvtsi32_si128(a); - const __m128i B0 = _mm_cvtsi32_si128(b); - const __m128i C0 = _mm_cvtsi32_si128(c); + const __m128i A0 = _mm_cvtsi32_si128((int)a); + const __m128i B0 = _mm_cvtsi32_si128((int)b); + const __m128i C0 = _mm_cvtsi32_si128((int)c); const __m128i AC0 = _mm_subs_epu8(A0, C0); const __m128i CA0 = _mm_subs_epu8(C0, A0); const __m128i BC0 = _mm_subs_epu8(B0, C0); @@ -95,8 +92,8 @@ static WEBP_INLINE void Average2_uint32_SSE2(const uint32_t a0, __m128i* const avg) { // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1) const __m128i ones = _mm_set1_epi8(1); - const __m128i A0 = _mm_cvtsi32_si128(a0); - const __m128i A1 = _mm_cvtsi32_si128(a1); + const __m128i A0 = _mm_cvtsi32_si128((int)a0); + const __m128i A1 = _mm_cvtsi32_si128((int)a1); const __m128i avg1 = _mm_avg_epu8(A0, A1); const __m128i one = _mm_and_si128(_mm_xor_si128(A0, A1), ones); *avg = _mm_sub_epi8(avg1, one); @@ -104,8 +101,8 @@ static WEBP_INLINE void Average2_uint32_SSE2(const uint32_t a0, static WEBP_INLINE __m128i Average2_uint32_16_SSE2(uint32_t a0, uint32_t a1) { const __m128i zero = _mm_setzero_si128(); - const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a0), zero); - const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero); + const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a0), zero); + const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a1), zero); const __m128i sum = _mm_add_epi16(A1, A0); return _mm_srli_epi16(sum, 1); } @@ -113,19 +110,18 @@ static WEBP_INLINE __m128i Average2_uint32_16_SSE2(uint32_t a0, uint32_t a1) { static WEBP_INLINE uint32_t Average2_SSE2(uint32_t a0, uint32_t a1) { __m128i output; Average2_uint32_SSE2(a0, a1, &output); - return _mm_cvtsi128_si32(output); + return (uint32_t)_mm_cvtsi128_si32(output); } static WEBP_INLINE uint32_t Average3_SSE2(uint32_t a0, uint32_t a1, uint32_t a2) { const __m128i zero = _mm_setzero_si128(); const __m128i avg1 = Average2_uint32_16_SSE2(a0, a2); - const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128(a1), zero); + const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a1), zero); const __m128i sum = _mm_add_epi16(avg1, A1); const __m128i avg2 = _mm_srli_epi16(sum, 1); const __m128i A2 = _mm_packus_epi16(avg2, avg2); - const uint32_t output = _mm_cvtsi128_si32(A2); - return output; + return (uint32_t)_mm_cvtsi128_si32(A2); } static WEBP_INLINE uint32_t Average4_SSE2(uint32_t a0, uint32_t a1, @@ -135,46 +131,54 @@ static WEBP_INLINE uint32_t Average4_SSE2(uint32_t a0, uint32_t a1, const __m128i sum = _mm_add_epi16(avg2, avg1); const __m128i avg3 = _mm_srli_epi16(sum, 1); const __m128i A0 = _mm_packus_epi16(avg3, avg3); - const uint32_t output = _mm_cvtsi128_si32(A0); - return output; + return (uint32_t)_mm_cvtsi128_si32(A0); } -static uint32_t Predictor5_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average3_SSE2(left, top[0], top[1]); +static uint32_t Predictor5_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average3_SSE2(*left, top[0], top[1]); return pred; } -static uint32_t Predictor6_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average2_SSE2(left, top[-1]); +static uint32_t Predictor6_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average2_SSE2(*left, top[-1]); return pred; } -static uint32_t Predictor7_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average2_SSE2(left, top[0]); +static uint32_t Predictor7_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average2_SSE2(*left, top[0]); return pred; } -static uint32_t Predictor8_SSE2(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor8_SSE2(const uint32_t* const left, + const uint32_t* const top) { const uint32_t pred = Average2_SSE2(top[-1], top[0]); (void)left; return pred; } -static uint32_t Predictor9_SSE2(uint32_t left, const uint32_t* const top) { +static uint32_t Predictor9_SSE2(const uint32_t* const left, + const uint32_t* const top) { const uint32_t pred = Average2_SSE2(top[0], top[1]); (void)left; return pred; } -static uint32_t Predictor10_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Average4_SSE2(left, top[-1], top[0], top[1]); +static uint32_t Predictor10_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Average4_SSE2(*left, top[-1], top[0], top[1]); return pred; } -static uint32_t Predictor11_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = Select_SSE2(top[0], left, top[-1]); +static uint32_t Predictor11_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = Select_SSE2(top[0], *left, top[-1]); return pred; } -static uint32_t Predictor12_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = ClampedAddSubtractFull_SSE2(left, top[0], top[-1]); +static uint32_t Predictor12_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractFull_SSE2(*left, top[0], top[-1]); return pred; } -static uint32_t Predictor13_SSE2(uint32_t left, const uint32_t* const top) { - const uint32_t pred = ClampedAddSubtractHalf_SSE2(left, top[0], top[-1]); +static uint32_t Predictor13_SSE2(const uint32_t* const left, + const uint32_t* const top) { + const uint32_t pred = ClampedAddSubtractHalf_SSE2(*left, top[0], top[-1]); return pred; } @@ -184,7 +188,7 @@ static uint32_t Predictor13_SSE2(uint32_t left, const uint32_t* const top) { static void PredictorAdd0_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; - const __m128i black = _mm_set1_epi32(ARGB_BLACK); + const __m128i black = _mm_set1_epi32((int)ARGB_BLACK); for (i = 0; i + 4 <= num_pixels; i += 4) { const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); const __m128i res = _mm_add_epi8(src, black); @@ -200,7 +204,7 @@ static void PredictorAdd0_SSE2(const uint32_t* in, const uint32_t* upper, static void PredictorAdd1_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; - __m128i prev = _mm_set1_epi32(out[-1]); + __m128i prev = _mm_set1_epi32((int)out[-1]); for (i = 0; i + 4 <= num_pixels; i += 4) { // a | b | c | d const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); @@ -277,12 +281,12 @@ GENERATE_PREDICTOR_2(9, upper[i + 1]) #undef GENERATE_PREDICTOR_2 // Predictor10: average of (average of (L,TL), average of (T, TR)). -#define DO_PRED10(OUT) do { \ - __m128i avgLTL, avg; \ - Average2_m128i(&L, &TL, &avgLTL); \ - Average2_m128i(&avgTTR, &avgLTL, &avg); \ - L = _mm_add_epi8(avg, src); \ - out[i + (OUT)] = _mm_cvtsi128_si32(L); \ +#define DO_PRED10(OUT) do { \ + __m128i avgLTL, avg; \ + Average2_m128i(&L, &TL, &avgLTL); \ + Average2_m128i(&avgTTR, &avgLTL, &avg); \ + L = _mm_add_epi8(avg, src); \ + out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(L); \ } while (0) #define DO_PRED10_SHIFT do { \ @@ -295,7 +299,7 @@ GENERATE_PREDICTOR_2(9, upper[i + 1]) static void PredictorAdd10_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; - __m128i L = _mm_cvtsi32_si128(out[-1]); + __m128i L = _mm_cvtsi32_si128((int)out[-1]); for (i = 0; i + 4 <= num_pixels; i += 4) { __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); @@ -328,7 +332,7 @@ static void PredictorAdd10_SSE2(const uint32_t* in, const uint32_t* upper, const __m128i B = _mm_andnot_si128(mask, T); \ const __m128i pred = _mm_or_si128(A, B); /* pred = (pa > b)? L : T*/ \ L = _mm_add_epi8(src, pred); \ - out[i + (OUT)] = _mm_cvtsi128_si32(L); \ + out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(L); \ } while (0) #define DO_PRED11_SHIFT do { \ @@ -343,7 +347,7 @@ static void PredictorAdd11_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; __m128i pa; - __m128i L = _mm_cvtsi32_si128(out[-1]); + __m128i L = _mm_cvtsi32_si128((int)out[-1]); for (i = 0; i + 4 <= num_pixels; i += 4) { __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); @@ -376,12 +380,12 @@ static void PredictorAdd11_SSE2(const uint32_t* in, const uint32_t* upper, #undef DO_PRED11_SHIFT // Predictor12: ClampedAddSubtractFull. -#define DO_PRED12(DIFF, LANE, OUT) do { \ - const __m128i all = _mm_add_epi16(L, (DIFF)); \ - const __m128i alls = _mm_packus_epi16(all, all); \ - const __m128i res = _mm_add_epi8(src, alls); \ - out[i + (OUT)] = _mm_cvtsi128_si32(res); \ - L = _mm_unpacklo_epi8(res, zero); \ +#define DO_PRED12(DIFF, LANE, OUT) do { \ + const __m128i all = _mm_add_epi16(L, (DIFF)); \ + const __m128i alls = _mm_packus_epi16(all, all); \ + const __m128i res = _mm_add_epi8(src, alls); \ + out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(res); \ + L = _mm_unpacklo_epi8(res, zero); \ } while (0) #define DO_PRED12_SHIFT(DIFF, LANE) do { \ @@ -394,7 +398,7 @@ static void PredictorAdd12_SSE2(const uint32_t* in, const uint32_t* upper, int num_pixels, uint32_t* out) { int i; const __m128i zero = _mm_setzero_si128(); - const __m128i L8 = _mm_cvtsi32_si128(out[-1]); + const __m128i L8 = _mm_cvtsi32_si128((int)out[-1]); __m128i L = _mm_unpacklo_epi8(L8, zero); for (i = 0; i + 4 <= num_pixels; i += 4) { // Load 4 pixels at a time. @@ -460,7 +464,7 @@ static void TransformColorInverse_SSE2(const VP8LMultipliers* const m, const __m128i mults_b2 = MK_CST_16(CST(red_to_blue_), 0); #undef MK_CST_16 #undef CST - const __m128i mask_ag = _mm_set1_epi32(0xff00ff00); // alpha-green masks + const __m128i mask_ag = _mm_set1_epi32((int)0xff00ff00); // alpha-green masks int i; for (i = 0; i + 4 <= num_pixels; i += 4) { const __m128i in = _mm_loadu_si128((const __m128i*)&src[i]); // argb @@ -524,7 +528,7 @@ static void ConvertBGRAToRGB_SSE2(const uint32_t* src, int num_pixels, static void ConvertBGRAToRGBA_SSE2(const uint32_t* src, int num_pixels, uint8_t* dst) { - const __m128i red_blue_mask = _mm_set1_epi32(0x00ff00ffu); + const __m128i red_blue_mask = _mm_set1_epi32(0x00ff00ff); const __m128i* in = (const __m128i*)src; __m128i* out = (__m128i*)dst; while (num_pixels >= 8) { @@ -553,7 +557,7 @@ static void ConvertBGRAToRGBA_SSE2(const uint32_t* src, static void ConvertBGRAToRGBA4444_SSE2(const uint32_t* src, int num_pixels, uint8_t* dst) { const __m128i mask_0x0f = _mm_set1_epi8(0x0f); - const __m128i mask_0xf0 = _mm_set1_epi8(0xf0); + const __m128i mask_0xf0 = _mm_set1_epi8((char)0xf0); const __m128i* in = (const __m128i*)src; __m128i* out = (__m128i*)dst; while (num_pixels >= 8) { @@ -588,8 +592,8 @@ static void ConvertBGRAToRGBA4444_SSE2(const uint32_t* src, static void ConvertBGRAToRGB565_SSE2(const uint32_t* src, int num_pixels, uint8_t* dst) { - const __m128i mask_0xe0 = _mm_set1_epi8(0xe0); - const __m128i mask_0xf8 = _mm_set1_epi8(0xf8); + const __m128i mask_0xe0 = _mm_set1_epi8((char)0xe0); + const __m128i mask_0xf8 = _mm_set1_epi8((char)0xf8); const __m128i mask_0x07 = _mm_set1_epi8(0x07); const __m128i* in = (const __m128i*)src; __m128i* out = (__m128i*)dst; diff --git a/3rdparty/libwebp/src/dsp/lossless_sse41.c b/3rdparty/libwebp/src/dsp/lossless_sse41.c new file mode 100644 index 0000000000..bb7ce7611f --- /dev/null +++ b/3rdparty/libwebp/src/dsp/lossless_sse41.c @@ -0,0 +1,133 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// SSE41 variant of methods for lossless decoder + +#include "src/dsp/dsp.h" + +#if defined(WEBP_USE_SSE41) + +#include "src/dsp/common_sse41.h" +#include "src/dsp/lossless.h" +#include "src/dsp/lossless_common.h" + +//------------------------------------------------------------------------------ +// Color-space conversion functions + +static void TransformColorInverse_SSE41(const VP8LMultipliers* const m, + const uint32_t* const src, + int num_pixels, uint32_t* dst) { +// sign-extended multiplying constants, pre-shifted by 5. +#define CST(X) (((int16_t)(m->X << 8)) >> 5) // sign-extend + const __m128i mults_rb = + _mm_set1_epi32((int)((uint32_t)CST(green_to_red_) << 16 | + (CST(green_to_blue_) & 0xffff))); + const __m128i mults_b2 = _mm_set1_epi32(CST(red_to_blue_)); +#undef CST + const __m128i mask_ag = _mm_set1_epi32((int)0xff00ff00); + const __m128i perm1 = _mm_setr_epi8(-1, 1, -1, 1, -1, 5, -1, 5, + -1, 9, -1, 9, -1, 13, -1, 13); + const __m128i perm2 = _mm_setr_epi8(-1, 2, -1, -1, -1, 6, -1, -1, + -1, 10, -1, -1, -1, 14, -1, -1); + int i; + for (i = 0; i + 4 <= num_pixels; i += 4) { + const __m128i A = _mm_loadu_si128((const __m128i*)(src + i)); + const __m128i B = _mm_shuffle_epi8(A, perm1); // argb -> g0g0 + const __m128i C = _mm_mulhi_epi16(B, mults_rb); + const __m128i D = _mm_add_epi8(A, C); + const __m128i E = _mm_shuffle_epi8(D, perm2); + const __m128i F = _mm_mulhi_epi16(E, mults_b2); + const __m128i G = _mm_add_epi8(D, F); + const __m128i out = _mm_blendv_epi8(G, A, mask_ag); + _mm_storeu_si128((__m128i*)&dst[i], out); + } + // Fall-back to C-version for left-overs. + if (i != num_pixels) { + VP8LTransformColorInverse_C(m, src + i, num_pixels - i, dst + i); + } +} + +//------------------------------------------------------------------------------ + +#define ARGB_TO_RGB_SSE41 do { \ + while (num_pixels >= 16) { \ + const __m128i in0 = _mm_loadu_si128(in + 0); \ + const __m128i in1 = _mm_loadu_si128(in + 1); \ + const __m128i in2 = _mm_loadu_si128(in + 2); \ + const __m128i in3 = _mm_loadu_si128(in + 3); \ + const __m128i a0 = _mm_shuffle_epi8(in0, perm0); \ + const __m128i a1 = _mm_shuffle_epi8(in1, perm1); \ + const __m128i a2 = _mm_shuffle_epi8(in2, perm2); \ + const __m128i a3 = _mm_shuffle_epi8(in3, perm3); \ + const __m128i b0 = _mm_blend_epi16(a0, a1, 0xc0); \ + const __m128i b1 = _mm_blend_epi16(a1, a2, 0xf0); \ + const __m128i b2 = _mm_blend_epi16(a2, a3, 0xfc); \ + _mm_storeu_si128(out + 0, b0); \ + _mm_storeu_si128(out + 1, b1); \ + _mm_storeu_si128(out + 2, b2); \ + in += 4; \ + out += 3; \ + num_pixels -= 16; \ + } \ +} while (0) + +static void ConvertBGRAToRGB_SSE41(const uint32_t* src, int num_pixels, + uint8_t* dst) { + const __m128i* in = (const __m128i*)src; + __m128i* out = (__m128i*)dst; + const __m128i perm0 = _mm_setr_epi8(2, 1, 0, 6, 5, 4, 10, 9, + 8, 14, 13, 12, -1, -1, -1, -1); + const __m128i perm1 = _mm_shuffle_epi32(perm0, 0x39); + const __m128i perm2 = _mm_shuffle_epi32(perm0, 0x4e); + const __m128i perm3 = _mm_shuffle_epi32(perm0, 0x93); + + ARGB_TO_RGB_SSE41; + + // left-overs + if (num_pixels > 0) { + VP8LConvertBGRAToRGB_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + } +} + +static void ConvertBGRAToBGR_SSE41(const uint32_t* src, + int num_pixels, uint8_t* dst) { + const __m128i* in = (const __m128i*)src; + __m128i* out = (__m128i*)dst; + const __m128i perm0 = _mm_setr_epi8(0, 1, 2, 4, 5, 6, 8, 9, 10, + 12, 13, 14, -1, -1, -1, -1); + const __m128i perm1 = _mm_shuffle_epi32(perm0, 0x39); + const __m128i perm2 = _mm_shuffle_epi32(perm0, 0x4e); + const __m128i perm3 = _mm_shuffle_epi32(perm0, 0x93); + + ARGB_TO_RGB_SSE41; + + // left-overs + if (num_pixels > 0) { + VP8LConvertBGRAToBGR_C((const uint32_t*)in, num_pixels, (uint8_t*)out); + } +} + +#undef ARGB_TO_RGB_SSE41 + +//------------------------------------------------------------------------------ +// Entry point + +extern void VP8LDspInitSSE41(void); + +WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitSSE41(void) { + VP8LTransformColorInverse = TransformColorInverse_SSE41; + VP8LConvertBGRAToRGB = ConvertBGRAToRGB_SSE41; + VP8LConvertBGRAToBGR = ConvertBGRAToBGR_SSE41; +} + +#else // !WEBP_USE_SSE41 + +WEBP_DSP_INIT_STUB(VP8LDspInitSSE41) + +#endif // WEBP_USE_SSE41 diff --git a/3rdparty/libwebp/src/dsp/msa_macro.h b/3rdparty/libwebp/src/dsp/msa_macro.h index a16c0bb300..90adbbc319 100644 --- a/3rdparty/libwebp/src/dsp/msa_macro.h +++ b/3rdparty/libwebp/src/dsp/msa_macro.h @@ -14,6 +14,10 @@ #ifndef WEBP_DSP_MSA_MACRO_H_ #define WEBP_DSP_MSA_MACRO_H_ +#include "src/dsp/dsp.h" + +#if defined(WEBP_USE_MSA) + #include #include @@ -69,27 +73,25 @@ #define ST_UW(...) ST_W(v4u32, __VA_ARGS__) #define ST_SW(...) ST_W(v4i32, __VA_ARGS__) -#define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME) \ - static inline TYPE FUNC_NAME(const void* const psrc) { \ - const uint8_t* const psrc_m = (const uint8_t*)psrc; \ - TYPE val_m; \ - __asm__ volatile ( \ - "" #INSTR " %[val_m], %[psrc_m] \n\t" \ - : [val_m] "=r" (val_m) \ - : [psrc_m] "m" (*psrc_m)); \ - return val_m; \ +#define MSA_LOAD_FUNC(TYPE, INSTR, FUNC_NAME) \ + static inline TYPE FUNC_NAME(const void* const psrc) { \ + const uint8_t* const psrc_m = (const uint8_t*)psrc; \ + TYPE val_m; \ + __asm__ volatile("" #INSTR " %[val_m], %[psrc_m] \n\t" \ + : [val_m] "=r"(val_m) \ + : [psrc_m] "m"(*psrc_m)); \ + return val_m; \ } #define MSA_LOAD(psrc, FUNC_NAME) FUNC_NAME(psrc) -#define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME) \ - static inline void FUNC_NAME(TYPE val, void* const pdst) { \ - uint8_t* const pdst_m = (uint8_t*)pdst; \ - TYPE val_m = val; \ - __asm__ volatile ( \ - " " #INSTR " %[val_m], %[pdst_m] \n\t" \ - : [pdst_m] "=m" (*pdst_m) \ - : [val_m] "r" (val_m)); \ +#define MSA_STORE_FUNC(TYPE, INSTR, FUNC_NAME) \ + static inline void FUNC_NAME(TYPE val, void* const pdst) { \ + uint8_t* const pdst_m = (uint8_t*)pdst; \ + TYPE val_m = val; \ + __asm__ volatile(" " #INSTR " %[val_m], %[pdst_m] \n\t" \ + : [pdst_m] "=m"(*pdst_m) \ + : [val_m] "r"(val_m)); \ } #define MSA_STORE(val, pdst, FUNC_NAME) FUNC_NAME(val, pdst) @@ -1389,4 +1391,5 @@ static WEBP_INLINE uint32_t func_hadd_uh_u32(v8u16 in) { } while (0) #define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__) +#endif // WEBP_USE_MSA #endif // WEBP_DSP_MSA_MACRO_H_ diff --git a/3rdparty/libwebp/src/dsp/neon.h b/3rdparty/libwebp/src/dsp/neon.h index aa1dea1301..14acb4044b 100644 --- a/3rdparty/libwebp/src/dsp/neon.h +++ b/3rdparty/libwebp/src/dsp/neon.h @@ -12,14 +12,16 @@ #ifndef WEBP_DSP_NEON_H_ #define WEBP_DSP_NEON_H_ -#include - #include "src/dsp/dsp.h" +#if defined(WEBP_USE_NEON) + +#include + // Right now, some intrinsics functions seem slower, so we disable them // everywhere except newer clang/gcc or aarch64 where the inline assembly is // incompatible. -#if LOCAL_CLANG_PREREQ(3,8) || LOCAL_GCC_PREREQ(4,9) || defined(__aarch64__) +#if LOCAL_CLANG_PREREQ(3, 8) || LOCAL_GCC_PREREQ(4, 9) || WEBP_AARCH64 #define WEBP_USE_INTRINSICS // use intrinsics when possible #endif @@ -44,7 +46,7 @@ // if using intrinsics, this flag avoids some functions that make gcc-4.6.3 // crash ("internal compiler error: in immed_double_const, at emit-rtl."). // (probably similar to gcc.gnu.org/bugzilla/show_bug.cgi?id=48183) -#if !(LOCAL_CLANG_PREREQ(3,8) || LOCAL_GCC_PREREQ(4,8) || defined(__aarch64__)) +#if !(LOCAL_CLANG_PREREQ(3, 8) || LOCAL_GCC_PREREQ(4, 8) || WEBP_AARCH64) #define WORK_AROUND_GCC #endif @@ -98,4 +100,5 @@ static WEBP_INLINE int32x4x4_t Transpose4x4_NEON(const int32x4x4_t rows) { } while (0) #endif +#endif // WEBP_USE_NEON #endif // WEBP_DSP_NEON_H_ diff --git a/3rdparty/libwebp/src/dsp/quant.h b/3rdparty/libwebp/src/dsp/quant.h index 5e8dba8d19..dcbc11c77c 100644 --- a/3rdparty/libwebp/src/dsp/quant.h +++ b/3rdparty/libwebp/src/dsp/quant.h @@ -21,18 +21,24 @@ #define IsFlat IsFlat_NEON -static uint32x2_t horizontal_add_uint32x4(const uint32x4_t a) { +static uint32_t horizontal_add_uint32x4(const uint32x4_t a) { +#if WEBP_AARCH64 + return vaddvq_u32(a); +#else const uint64x2_t b = vpaddlq_u32(a); - return vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), - vreinterpret_u32_u64(vget_high_u64(b))); + const uint32x2_t c = vadd_u32(vreinterpret_u32_u64(vget_low_u64(b)), + vreinterpret_u32_u64(vget_high_u64(b))); + return vget_lane_u32(c, 0); +#endif } static WEBP_INLINE int IsFlat(const int16_t* levels, int num_blocks, int thresh) { const int16x8_t tst_ones = vdupq_n_s16(-1); uint32x4_t sum = vdupq_n_u32(0); + int i; - for (int i = 0; i < num_blocks; ++i) { + for (i = 0; i < num_blocks; ++i) { // Set DC to zero. const int16x8_t a_0 = vsetq_lane_s16(0, vld1q_s16(levels), 0); const int16x8_t a_1 = vld1q_s16(levels + 8); @@ -45,7 +51,7 @@ static WEBP_INLINE int IsFlat(const int16_t* levels, int num_blocks, levels += 16; } - return thresh >= (int32_t)vget_lane_u32(horizontal_add_uint32x4(sum), 0); + return thresh >= (int)horizontal_add_uint32x4(sum); } #else diff --git a/3rdparty/libwebp/src/dsp/rescaler.c b/3rdparty/libwebp/src/dsp/rescaler.c index c5a01e82df..325d8be180 100644 --- a/3rdparty/libwebp/src/dsp/rescaler.c +++ b/3rdparty/libwebp/src/dsp/rescaler.c @@ -38,8 +38,9 @@ void WebPRescalerImportRowExpand_C(WebPRescaler* const wrk, int x_out = channel; // simple bilinear interpolation int accum = wrk->x_add; - int left = src[x_in]; - int right = (wrk->src_width > 1) ? src[x_in + x_stride] : left; + rescaler_t left = (rescaler_t)src[x_in]; + rescaler_t right = + (wrk->src_width > 1) ? (rescaler_t)src[x_in + x_stride] : left; x_in += x_stride; while (1) { wrk->frow[x_out] = right * wrk->x_add + (left - right) * accum; @@ -50,7 +51,7 @@ void WebPRescalerImportRowExpand_C(WebPRescaler* const wrk, left = right; x_in += x_stride; assert(x_in < wrk->src_width * x_stride); - right = src[x_in]; + right = (rescaler_t)src[x_in]; accum += wrk->x_add; } } @@ -196,6 +197,7 @@ WebPRescalerImportRowFunc WebPRescalerImportRowShrink; WebPRescalerExportRowFunc WebPRescalerExportRowExpand; WebPRescalerExportRowFunc WebPRescalerExportRowShrink; +extern VP8CPUInfo VP8GetCPUInfo; extern void WebPRescalerDspInitSSE2(void); extern void WebPRescalerDspInitMIPS32(void); extern void WebPRescalerDspInitMIPSdspR2(void); @@ -213,7 +215,7 @@ WEBP_DSP_INIT_FUNC(WebPRescalerDspInit) { WebPRescalerImportRowShrink = WebPRescalerImportRowShrink_C; if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPRescalerDspInitSSE2(); } @@ -235,7 +237,7 @@ WEBP_DSP_INIT_FUNC(WebPRescalerDspInit) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { WebPRescalerDspInitNEON(); diff --git a/3rdparty/libwebp/src/dsp/rescaler_sse2.c b/3rdparty/libwebp/src/dsp/rescaler_sse2.c index d7effea16e..3f18e94e93 100644 --- a/3rdparty/libwebp/src/dsp/rescaler_sse2.c +++ b/3rdparty/libwebp/src/dsp/rescaler_sse2.c @@ -85,7 +85,7 @@ static void RescalerImportRowExpand_SSE2(WebPRescaler* const wrk, const __m128i mult = _mm_cvtsi32_si128(((x_add - accum) << 16) | accum); const __m128i out = _mm_madd_epi16(cur_pixels, mult); assert(sizeof(*frow) == sizeof(uint32_t)); - WebPUint32ToMem((uint8_t*)frow, _mm_cvtsi128_si32(out)); + WebPInt32ToMem((uint8_t*)frow, _mm_cvtsi128_si32(out)); frow += 1; if (frow >= frow_end) break; accum -= wrk->x_sub; @@ -132,7 +132,7 @@ static void RescalerImportRowShrink_SSE2(WebPRescaler* const wrk, __m128i base = zero; accum += wrk->x_add; while (accum > 0) { - const __m128i A = _mm_cvtsi32_si128(WebPMemToUint32(src)); + const __m128i A = _mm_cvtsi32_si128(WebPMemToInt32(src)); src += 4; base = _mm_unpacklo_epi8(A, zero); // To avoid overflow, we need: base * x_add / x_sub < 32768 @@ -198,7 +198,7 @@ static WEBP_INLINE void ProcessRow_SSE2(const __m128i* const A0, const __m128i* const mult, uint8_t* const dst) { const __m128i rounder = _mm_set_epi32(0, ROUNDER, 0, ROUNDER); - const __m128i mask = _mm_set_epi32(0xffffffffu, 0, 0xffffffffu, 0); + const __m128i mask = _mm_set_epi32(~0, 0, ~0, 0); const __m128i B0 = _mm_mul_epu32(*A0, *mult); const __m128i B1 = _mm_mul_epu32(*A1, *mult); const __m128i B2 = _mm_mul_epu32(*A2, *mult); diff --git a/3rdparty/libwebp/src/dsp/ssim.c b/3rdparty/libwebp/src/dsp/ssim.c index 989ce8254c..9a1341ed95 100644 --- a/3rdparty/libwebp/src/dsp/ssim.c +++ b/3rdparty/libwebp/src/dsp/ssim.c @@ -137,6 +137,7 @@ VP8SSIMGetClippedFunc VP8SSIMGetClipped; VP8AccumulateSSEFunc VP8AccumulateSSE; #endif +extern VP8CPUInfo VP8GetCPUInfo; extern void VP8SSIMDspInitSSE2(void); WEBP_DSP_INIT_FUNC(VP8SSIMDspInit) { @@ -150,7 +151,7 @@ WEBP_DSP_INIT_FUNC(VP8SSIMDspInit) { #endif if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { VP8SSIMDspInitSSE2(); } diff --git a/3rdparty/libwebp/src/dsp/upsampling.c b/3rdparty/libwebp/src/dsp/upsampling.c index 9b60da5bbb..983b9c42d3 100644 --- a/3rdparty/libwebp/src/dsp/upsampling.c +++ b/3rdparty/libwebp/src/dsp/upsampling.c @@ -215,6 +215,7 @@ static void EmptyYuv444Func(const uint8_t* y, WebPYUV444Converter WebPYUV444Converters[MODE_LAST]; +extern VP8CPUInfo VP8GetCPUInfo; extern void WebPInitYUV444ConvertersMIPSdspR2(void); extern void WebPInitYUV444ConvertersSSE2(void); extern void WebPInitYUV444ConvertersSSE41(void); @@ -233,12 +234,12 @@ WEBP_DSP_INIT_FUNC(WebPInitYUV444Converters) { WebPYUV444Converters[MODE_rgbA_4444] = WebPYuv444ToRgba4444_C; if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitYUV444ConvertersSSE2(); } #endif -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitYUV444ConvertersSSE41(); } @@ -278,12 +279,12 @@ WEBP_DSP_INIT_FUNC(WebPInitUpsamplers) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitUpsamplersSSE2(); } #endif -#if defined(WEBP_USE_SSE41) +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitUpsamplersSSE41(); } @@ -300,7 +301,7 @@ WEBP_DSP_INIT_FUNC(WebPInitUpsamplers) { #endif } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { WebPInitUpsamplersNEON(); diff --git a/3rdparty/libwebp/src/dsp/upsampling_neon.c b/3rdparty/libwebp/src/dsp/upsampling_neon.c index 6ba71a7de5..bbc000ca2d 100644 --- a/3rdparty/libwebp/src/dsp/upsampling_neon.c +++ b/3rdparty/libwebp/src/dsp/upsampling_neon.c @@ -111,7 +111,7 @@ static const int16_t kCoeffs1[4] = { 19077, 26149, 6419, 13320 }; vst4_u8(out, v255_r_g_b); \ } while (0) -#if !defined(WEBP_SWAP_16BIT_CSP) +#if (WEBP_SWAP_16BIT_CSP == 0) #define ZIP_U8(lo, hi) vzip_u8((lo), (hi)) #else #define ZIP_U8(lo, hi) vzip_u8((hi), (lo)) diff --git a/3rdparty/libwebp/src/dsp/upsampling_sse2.c b/3rdparty/libwebp/src/dsp/upsampling_sse2.c index 340f1e2ac2..08b6d0b1cf 100644 --- a/3rdparty/libwebp/src/dsp/upsampling_sse2.c +++ b/3rdparty/libwebp/src/dsp/upsampling_sse2.c @@ -121,7 +121,7 @@ static void FUNC_NAME(const uint8_t* top_y, const uint8_t* bottom_y, \ int uv_pos, pos; \ /* 16byte-aligned array to cache reconstructed u and v */ \ uint8_t uv_buf[14 * 32 + 15] = { 0 }; \ - uint8_t* const r_u = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~15); \ + uint8_t* const r_u = (uint8_t*)((uintptr_t)(uv_buf + 15) & ~(uintptr_t)15); \ uint8_t* const r_v = r_u + 32; \ \ assert(top_y != NULL); \ diff --git a/3rdparty/libwebp/src/dsp/yuv.c b/3rdparty/libwebp/src/dsp/yuv.c index 14e67fc28e..8a04b85d82 100644 --- a/3rdparty/libwebp/src/dsp/yuv.c +++ b/3rdparty/libwebp/src/dsp/yuv.c @@ -70,6 +70,7 @@ void WebPSamplerProcessPlane(const uint8_t* y, int y_stride, WebPSamplerRowFunc WebPSamplers[MODE_LAST]; +extern VP8CPUInfo VP8GetCPUInfo; extern void WebPInitSamplersSSE2(void); extern void WebPInitSamplersSSE41(void); extern void WebPInitSamplersMIPS32(void); @@ -90,16 +91,16 @@ WEBP_DSP_INIT_FUNC(WebPInitSamplers) { // If defined, use CPUInfo() to overwrite some pointers with faster versions. if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitSamplersSSE2(); } -#endif // WEBP_USE_SSE2 -#if defined(WEBP_USE_SSE41) +#endif // WEBP_HAVE_SSE2 +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitSamplersSSE41(); } -#endif // WEBP_USE_SSE41 +#endif // WEBP_HAVE_SSE41 #if defined(WEBP_USE_MIPS32) if (VP8GetCPUInfo(kMIPS32)) { WebPInitSamplersMIPS32(); @@ -194,50 +195,6 @@ void WebPConvertRGBA32ToUV_C(const uint16_t* rgb, //----------------------------------------------------------------------------- -#if !WEBP_NEON_OMIT_C_CODE -#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic -static uint16_t clip_y(int v) { - return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v; -} - -static uint64_t SharpYUVUpdateY_C(const uint16_t* ref, const uint16_t* src, - uint16_t* dst, int len) { - uint64_t diff = 0; - int i; - for (i = 0; i < len; ++i) { - const int diff_y = ref[i] - src[i]; - const int new_y = (int)dst[i] + diff_y; - dst[i] = clip_y(new_y); - diff += (uint64_t)abs(diff_y); - } - return diff; -} - -static void SharpYUVUpdateRGB_C(const int16_t* ref, const int16_t* src, - int16_t* dst, int len) { - int i; - for (i = 0; i < len; ++i) { - const int diff_uv = ref[i] - src[i]; - dst[i] += diff_uv; - } -} - -static void SharpYUVFilterRow_C(const int16_t* A, const int16_t* B, int len, - const uint16_t* best_y, uint16_t* out) { - int i; - for (i = 0; i < len; ++i, ++A, ++B) { - const int v0 = (A[0] * 9 + A[1] * 3 + B[0] * 3 + B[1] + 8) >> 4; - const int v1 = (A[1] * 9 + A[0] * 3 + B[1] * 3 + B[0] + 8) >> 4; - out[2 * i + 0] = clip_y(best_y[2 * i + 0] + v0); - out[2 * i + 1] = clip_y(best_y[2 * i + 1] + v1); - } -} -#endif // !WEBP_NEON_OMIT_C_CODE - -#undef MAX_Y - -//----------------------------------------------------------------------------- - void (*WebPConvertRGB24ToY)(const uint8_t* rgb, uint8_t* y, int width); void (*WebPConvertBGR24ToY)(const uint8_t* bgr, uint8_t* y, int width); void (*WebPConvertRGBA32ToUV)(const uint16_t* rgb, @@ -247,18 +204,9 @@ void (*WebPConvertARGBToY)(const uint32_t* argb, uint8_t* y, int width); void (*WebPConvertARGBToUV)(const uint32_t* argb, uint8_t* u, uint8_t* v, int src_width, int do_store); -uint64_t (*WebPSharpYUVUpdateY)(const uint16_t* ref, const uint16_t* src, - uint16_t* dst, int len); -void (*WebPSharpYUVUpdateRGB)(const int16_t* ref, const int16_t* src, - int16_t* dst, int len); -void (*WebPSharpYUVFilterRow)(const int16_t* A, const int16_t* B, int len, - const uint16_t* best_y, uint16_t* out); - extern void WebPInitConvertARGBToYUVSSE2(void); extern void WebPInitConvertARGBToYUVSSE41(void); extern void WebPInitConvertARGBToYUVNEON(void); -extern void WebPInitSharpYUVSSE2(void); -extern void WebPInitSharpYUVNEON(void); WEBP_DSP_INIT_FUNC(WebPInitConvertARGBToYUV) { WebPConvertARGBToY = ConvertARGBToY_C; @@ -269,40 +217,29 @@ WEBP_DSP_INIT_FUNC(WebPInitConvertARGBToYUV) { WebPConvertRGBA32ToUV = WebPConvertRGBA32ToUV_C; -#if !WEBP_NEON_OMIT_C_CODE - WebPSharpYUVUpdateY = SharpYUVUpdateY_C; - WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_C; - WebPSharpYUVFilterRow = SharpYUVFilterRow_C; -#endif - if (VP8GetCPUInfo != NULL) { -#if defined(WEBP_USE_SSE2) +#if defined(WEBP_HAVE_SSE2) if (VP8GetCPUInfo(kSSE2)) { WebPInitConvertARGBToYUVSSE2(); - WebPInitSharpYUVSSE2(); } -#endif // WEBP_USE_SSE2 -#if defined(WEBP_USE_SSE41) +#endif // WEBP_HAVE_SSE2 +#if defined(WEBP_HAVE_SSE41) if (VP8GetCPUInfo(kSSE4_1)) { WebPInitConvertARGBToYUVSSE41(); } -#endif // WEBP_USE_SSE41 +#endif // WEBP_HAVE_SSE41 } -#if defined(WEBP_USE_NEON) +#if defined(WEBP_HAVE_NEON) if (WEBP_NEON_OMIT_C_CODE || (VP8GetCPUInfo != NULL && VP8GetCPUInfo(kNEON))) { WebPInitConvertARGBToYUVNEON(); - WebPInitSharpYUVNEON(); } -#endif // WEBP_USE_NEON +#endif // WEBP_HAVE_NEON assert(WebPConvertARGBToY != NULL); assert(WebPConvertARGBToUV != NULL); assert(WebPConvertRGB24ToY != NULL); assert(WebPConvertBGR24ToY != NULL); assert(WebPConvertRGBA32ToUV != NULL); - assert(WebPSharpYUVUpdateY != NULL); - assert(WebPSharpYUVUpdateRGB != NULL); - assert(WebPSharpYUVFilterRow != NULL); } diff --git a/3rdparty/libwebp/src/dsp/yuv.h b/3rdparty/libwebp/src/dsp/yuv.h index c12be1d094..66a397d117 100644 --- a/3rdparty/libwebp/src/dsp/yuv.h +++ b/3rdparty/libwebp/src/dsp/yuv.h @@ -10,7 +10,7 @@ // inline YUV<->RGB conversion function // // The exact naming is Y'CbCr, following the ITU-R BT.601 standard. -// More information at: http://en.wikipedia.org/wiki/YCbCr +// More information at: https://en.wikipedia.org/wiki/YCbCr // Y = 0.2569 * R + 0.5044 * G + 0.0979 * B + 16 // U = -0.1483 * R - 0.2911 * G + 0.4394 * B + 128 // V = 0.4394 * R - 0.3679 * G - 0.0715 * B + 128 diff --git a/3rdparty/libwebp/src/dsp/yuv_neon.c b/3rdparty/libwebp/src/dsp/yuv_neon.c index a34d60248f..ff77b00980 100644 --- a/3rdparty/libwebp/src/dsp/yuv_neon.c +++ b/3rdparty/libwebp/src/dsp/yuv_neon.c @@ -173,116 +173,8 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUVNEON(void) { WebPConvertRGBA32ToUV = ConvertRGBA32ToUV_NEON; } -//------------------------------------------------------------------------------ - -#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic -static uint16_t clip_y_NEON(int v) { - return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v; -} - -static uint64_t SharpYUVUpdateY_NEON(const uint16_t* ref, const uint16_t* src, - uint16_t* dst, int len) { - int i; - const int16x8_t zero = vdupq_n_s16(0); - const int16x8_t max = vdupq_n_s16(MAX_Y); - uint64x2_t sum = vdupq_n_u64(0); - uint64_t diff; - - for (i = 0; i + 8 <= len; i += 8) { - const int16x8_t A = vreinterpretq_s16_u16(vld1q_u16(ref + i)); - const int16x8_t B = vreinterpretq_s16_u16(vld1q_u16(src + i)); - const int16x8_t C = vreinterpretq_s16_u16(vld1q_u16(dst + i)); - const int16x8_t D = vsubq_s16(A, B); // diff_y - const int16x8_t F = vaddq_s16(C, D); // new_y - const uint16x8_t H = - vreinterpretq_u16_s16(vmaxq_s16(vminq_s16(F, max), zero)); - const int16x8_t I = vabsq_s16(D); // abs(diff_y) - vst1q_u16(dst + i, H); - sum = vpadalq_u32(sum, vpaddlq_u16(vreinterpretq_u16_s16(I))); - } - diff = vgetq_lane_u64(sum, 0) + vgetq_lane_u64(sum, 1); - for (; i < len; ++i) { - const int diff_y = ref[i] - src[i]; - const int new_y = (int)(dst[i]) + diff_y; - dst[i] = clip_y_NEON(new_y); - diff += (uint64_t)(abs(diff_y)); - } - return diff; -} - -static void SharpYUVUpdateRGB_NEON(const int16_t* ref, const int16_t* src, - int16_t* dst, int len) { - int i; - for (i = 0; i + 8 <= len; i += 8) { - const int16x8_t A = vld1q_s16(ref + i); - const int16x8_t B = vld1q_s16(src + i); - const int16x8_t C = vld1q_s16(dst + i); - const int16x8_t D = vsubq_s16(A, B); // diff_uv - const int16x8_t E = vaddq_s16(C, D); // new_uv - vst1q_s16(dst + i, E); - } - for (; i < len; ++i) { - const int diff_uv = ref[i] - src[i]; - dst[i] += diff_uv; - } -} - -static void SharpYUVFilterRow_NEON(const int16_t* A, const int16_t* B, int len, - const uint16_t* best_y, uint16_t* out) { - int i; - const int16x8_t max = vdupq_n_s16(MAX_Y); - const int16x8_t zero = vdupq_n_s16(0); - for (i = 0; i + 8 <= len; i += 8) { - const int16x8_t a0 = vld1q_s16(A + i + 0); - const int16x8_t a1 = vld1q_s16(A + i + 1); - const int16x8_t b0 = vld1q_s16(B + i + 0); - const int16x8_t b1 = vld1q_s16(B + i + 1); - const int16x8_t a0b1 = vaddq_s16(a0, b1); - const int16x8_t a1b0 = vaddq_s16(a1, b0); - const int16x8_t a0a1b0b1 = vaddq_s16(a0b1, a1b0); // A0+A1+B0+B1 - const int16x8_t a0b1_2 = vaddq_s16(a0b1, a0b1); // 2*(A0+B1) - const int16x8_t a1b0_2 = vaddq_s16(a1b0, a1b0); // 2*(A1+B0) - const int16x8_t c0 = vshrq_n_s16(vaddq_s16(a0b1_2, a0a1b0b1), 3); - const int16x8_t c1 = vshrq_n_s16(vaddq_s16(a1b0_2, a0a1b0b1), 3); - const int16x8_t d0 = vaddq_s16(c1, a0); - const int16x8_t d1 = vaddq_s16(c0, a1); - const int16x8_t e0 = vrshrq_n_s16(d0, 1); - const int16x8_t e1 = vrshrq_n_s16(d1, 1); - const int16x8x2_t f = vzipq_s16(e0, e1); - const int16x8_t g0 = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i + 0)); - const int16x8_t g1 = vreinterpretq_s16_u16(vld1q_u16(best_y + 2 * i + 8)); - const int16x8_t h0 = vaddq_s16(g0, f.val[0]); - const int16x8_t h1 = vaddq_s16(g1, f.val[1]); - const int16x8_t i0 = vmaxq_s16(vminq_s16(h0, max), zero); - const int16x8_t i1 = vmaxq_s16(vminq_s16(h1, max), zero); - vst1q_u16(out + 2 * i + 0, vreinterpretq_u16_s16(i0)); - vst1q_u16(out + 2 * i + 8, vreinterpretq_u16_s16(i1)); - } - for (; i < len; ++i) { - const int a0b1 = A[i + 0] + B[i + 1]; - const int a1b0 = A[i + 1] + B[i + 0]; - const int a0a1b0b1 = a0b1 + a1b0 + 8; - const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; - const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; - out[2 * i + 0] = clip_y_NEON(best_y[2 * i + 0] + v0); - out[2 * i + 1] = clip_y_NEON(best_y[2 * i + 1] + v1); - } -} -#undef MAX_Y - -//------------------------------------------------------------------------------ - -extern void WebPInitSharpYUVNEON(void); - -WEBP_TSAN_IGNORE_FUNCTION void WebPInitSharpYUVNEON(void) { - WebPSharpYUVUpdateY = SharpYUVUpdateY_NEON; - WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_NEON; - WebPSharpYUVFilterRow = SharpYUVFilterRow_NEON; -} - #else // !WEBP_USE_NEON WEBP_DSP_INIT_STUB(WebPInitConvertARGBToYUVNEON) -WEBP_DSP_INIT_STUB(WebPInitSharpYUVNEON) #endif // WEBP_USE_NEON diff --git a/3rdparty/libwebp/src/dsp/yuv_sse2.c b/3rdparty/libwebp/src/dsp/yuv_sse2.c index baa48d5371..01a48f9af2 100644 --- a/3rdparty/libwebp/src/dsp/yuv_sse2.c +++ b/3rdparty/libwebp/src/dsp/yuv_sse2.c @@ -15,10 +15,12 @@ #if defined(WEBP_USE_SSE2) -#include "src/dsp/common_sse2.h" #include #include +#include "src/dsp/common_sse2.h" +#include "src/utils/utils.h" + //----------------------------------------------------------------------------- // Convert spans of 32 pixels to various RGB formats for the fancy upsampler. @@ -74,7 +76,7 @@ static WEBP_INLINE __m128i Load_HI_16_SSE2(const uint8_t* src) { // Load and replicate the U/V samples static WEBP_INLINE __m128i Load_UV_HI_8_SSE2(const uint8_t* src) { const __m128i zero = _mm_setzero_si128(); - const __m128i tmp0 = _mm_cvtsi32_si128(*(const uint32_t*)src); + const __m128i tmp0 = _mm_cvtsi32_si128(WebPMemToInt32(src)); const __m128i tmp1 = _mm_unpacklo_epi8(zero, tmp0); return _mm_unpacklo_epi16(tmp1, tmp1); // replicate samples } @@ -130,7 +132,7 @@ static WEBP_INLINE void PackAndStore4444_SSE2(const __m128i* const R, const __m128i rg0 = _mm_packus_epi16(*B, *A); const __m128i ba0 = _mm_packus_epi16(*R, *G); #endif - const __m128i mask_0xf0 = _mm_set1_epi8(0xf0); + const __m128i mask_0xf0 = _mm_set1_epi8((char)0xf0); const __m128i rb1 = _mm_unpacklo_epi8(rg0, ba0); // rbrbrbrbrb... const __m128i ga1 = _mm_unpackhi_epi8(rg0, ba0); // gagagagaga... const __m128i rb2 = _mm_and_si128(rb1, mask_0xf0); @@ -147,9 +149,10 @@ static WEBP_INLINE void PackAndStore565_SSE2(const __m128i* const R, const __m128i r0 = _mm_packus_epi16(*R, *R); const __m128i g0 = _mm_packus_epi16(*G, *G); const __m128i b0 = _mm_packus_epi16(*B, *B); - const __m128i r1 = _mm_and_si128(r0, _mm_set1_epi8(0xf8)); + const __m128i r1 = _mm_and_si128(r0, _mm_set1_epi8((char)0xf8)); const __m128i b1 = _mm_and_si128(_mm_srli_epi16(b0, 3), _mm_set1_epi8(0x1f)); - const __m128i g1 = _mm_srli_epi16(_mm_and_si128(g0, _mm_set1_epi8(0xe0)), 5); + const __m128i g1 = + _mm_srli_epi16(_mm_and_si128(g0, _mm_set1_epi8((char)0xe0)), 5); const __m128i g2 = _mm_slli_epi16(_mm_and_si128(g0, _mm_set1_epi8(0x1c)), 3); const __m128i rg = _mm_or_si128(r1, g1); const __m128i gb = _mm_or_si128(g2, b1); @@ -747,128 +750,9 @@ WEBP_TSAN_IGNORE_FUNCTION void WebPInitConvertARGBToYUVSSE2(void) { WebPConvertRGBA32ToUV = ConvertRGBA32ToUV_SSE2; } -//------------------------------------------------------------------------------ - -#define MAX_Y ((1 << 10) - 1) // 10b precision over 16b-arithmetic -static uint16_t clip_y(int v) { - return (v < 0) ? 0 : (v > MAX_Y) ? MAX_Y : (uint16_t)v; -} - -static uint64_t SharpYUVUpdateY_SSE2(const uint16_t* ref, const uint16_t* src, - uint16_t* dst, int len) { - uint64_t diff = 0; - uint32_t tmp[4]; - int i; - const __m128i zero = _mm_setzero_si128(); - const __m128i max = _mm_set1_epi16(MAX_Y); - const __m128i one = _mm_set1_epi16(1); - __m128i sum = zero; - - for (i = 0; i + 8 <= len; i += 8) { - const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); - const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); - const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); - const __m128i D = _mm_sub_epi16(A, B); // diff_y - const __m128i E = _mm_cmpgt_epi16(zero, D); // sign (-1 or 0) - const __m128i F = _mm_add_epi16(C, D); // new_y - const __m128i G = _mm_or_si128(E, one); // -1 or 1 - const __m128i H = _mm_max_epi16(_mm_min_epi16(F, max), zero); - const __m128i I = _mm_madd_epi16(D, G); // sum(abs(...)) - _mm_storeu_si128((__m128i*)(dst + i), H); - sum = _mm_add_epi32(sum, I); - } - _mm_storeu_si128((__m128i*)tmp, sum); - diff = tmp[3] + tmp[2] + tmp[1] + tmp[0]; - for (; i < len; ++i) { - const int diff_y = ref[i] - src[i]; - const int new_y = (int)dst[i] + diff_y; - dst[i] = clip_y(new_y); - diff += (uint64_t)abs(diff_y); - } - return diff; -} - -static void SharpYUVUpdateRGB_SSE2(const int16_t* ref, const int16_t* src, - int16_t* dst, int len) { - int i = 0; - for (i = 0; i + 8 <= len; i += 8) { - const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); - const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); - const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); - const __m128i D = _mm_sub_epi16(A, B); // diff_uv - const __m128i E = _mm_add_epi16(C, D); // new_uv - _mm_storeu_si128((__m128i*)(dst + i), E); - } - for (; i < len; ++i) { - const int diff_uv = ref[i] - src[i]; - dst[i] += diff_uv; - } -} - -static void SharpYUVFilterRow_SSE2(const int16_t* A, const int16_t* B, int len, - const uint16_t* best_y, uint16_t* out) { - int i; - const __m128i kCst8 = _mm_set1_epi16(8); - const __m128i max = _mm_set1_epi16(MAX_Y); - const __m128i zero = _mm_setzero_si128(); - for (i = 0; i + 8 <= len; i += 8) { - const __m128i a0 = _mm_loadu_si128((const __m128i*)(A + i + 0)); - const __m128i a1 = _mm_loadu_si128((const __m128i*)(A + i + 1)); - const __m128i b0 = _mm_loadu_si128((const __m128i*)(B + i + 0)); - const __m128i b1 = _mm_loadu_si128((const __m128i*)(B + i + 1)); - const __m128i a0b1 = _mm_add_epi16(a0, b1); - const __m128i a1b0 = _mm_add_epi16(a1, b0); - const __m128i a0a1b0b1 = _mm_add_epi16(a0b1, a1b0); // A0+A1+B0+B1 - const __m128i a0a1b0b1_8 = _mm_add_epi16(a0a1b0b1, kCst8); - const __m128i a0b1_2 = _mm_add_epi16(a0b1, a0b1); // 2*(A0+B1) - const __m128i a1b0_2 = _mm_add_epi16(a1b0, a1b0); // 2*(A1+B0) - const __m128i c0 = _mm_srai_epi16(_mm_add_epi16(a0b1_2, a0a1b0b1_8), 3); - const __m128i c1 = _mm_srai_epi16(_mm_add_epi16(a1b0_2, a0a1b0b1_8), 3); - const __m128i d0 = _mm_add_epi16(c1, a0); - const __m128i d1 = _mm_add_epi16(c0, a1); - const __m128i e0 = _mm_srai_epi16(d0, 1); - const __m128i e1 = _mm_srai_epi16(d1, 1); - const __m128i f0 = _mm_unpacklo_epi16(e0, e1); - const __m128i f1 = _mm_unpackhi_epi16(e0, e1); - const __m128i g0 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); - const __m128i g1 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 8)); - const __m128i h0 = _mm_add_epi16(g0, f0); - const __m128i h1 = _mm_add_epi16(g1, f1); - const __m128i i0 = _mm_max_epi16(_mm_min_epi16(h0, max), zero); - const __m128i i1 = _mm_max_epi16(_mm_min_epi16(h1, max), zero); - _mm_storeu_si128((__m128i*)(out + 2 * i + 0), i0); - _mm_storeu_si128((__m128i*)(out + 2 * i + 8), i1); - } - for (; i < len; ++i) { - // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = - // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 - // We reuse the common sub-expressions. - const int a0b1 = A[i + 0] + B[i + 1]; - const int a1b0 = A[i + 1] + B[i + 0]; - const int a0a1b0b1 = a0b1 + a1b0 + 8; - const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; - const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; - out[2 * i + 0] = clip_y(best_y[2 * i + 0] + v0); - out[2 * i + 1] = clip_y(best_y[2 * i + 1] + v1); - } -} - -#undef MAX_Y - -//------------------------------------------------------------------------------ - -extern void WebPInitSharpYUVSSE2(void); - -WEBP_TSAN_IGNORE_FUNCTION void WebPInitSharpYUVSSE2(void) { - WebPSharpYUVUpdateY = SharpYUVUpdateY_SSE2; - WebPSharpYUVUpdateRGB = SharpYUVUpdateRGB_SSE2; - WebPSharpYUVFilterRow = SharpYUVFilterRow_SSE2; -} - #else // !WEBP_USE_SSE2 WEBP_DSP_INIT_STUB(WebPInitSamplersSSE2) WEBP_DSP_INIT_STUB(WebPInitConvertARGBToYUVSSE2) -WEBP_DSP_INIT_STUB(WebPInitSharpYUVSSE2) #endif // WEBP_USE_SSE2 diff --git a/3rdparty/libwebp/src/dsp/yuv_sse41.c b/3rdparty/libwebp/src/dsp/yuv_sse41.c index 579d1f7402..f79b802e47 100644 --- a/3rdparty/libwebp/src/dsp/yuv_sse41.c +++ b/3rdparty/libwebp/src/dsp/yuv_sse41.c @@ -15,10 +15,12 @@ #if defined(WEBP_USE_SSE41) -#include "src/dsp/common_sse41.h" #include #include +#include "src/dsp/common_sse41.h" +#include "src/utils/utils.h" + //----------------------------------------------------------------------------- // Convert spans of 32 pixels to various RGB formats for the fancy upsampler. @@ -74,7 +76,7 @@ static WEBP_INLINE __m128i Load_HI_16_SSE41(const uint8_t* src) { // Load and replicate the U/V samples static WEBP_INLINE __m128i Load_UV_HI_8_SSE41(const uint8_t* src) { const __m128i zero = _mm_setzero_si128(); - const __m128i tmp0 = _mm_cvtsi32_si128(*(const uint32_t*)src); + const __m128i tmp0 = _mm_cvtsi32_si128(WebPMemToInt32(src)); const __m128i tmp1 = _mm_unpacklo_epi8(zero, tmp0); return _mm_unpacklo_epi16(tmp1, tmp1); // replicate samples } diff --git a/3rdparty/libwebp/src/enc/alpha_enc.c b/3rdparty/libwebp/src/enc/alpha_enc.c index dce9ca957d..4a599f88a9 100644 --- a/3rdparty/libwebp/src/enc/alpha_enc.c +++ b/3rdparty/libwebp/src/enc/alpha_enc.c @@ -13,6 +13,7 @@ #include #include +#include #include "src/enc/vp8i_enc.h" #include "src/dsp/dsp.h" @@ -54,7 +55,7 @@ static int EncodeLossless(const uint8_t* const data, int width, int height, WebPConfig config; WebPPicture picture; - WebPPictureInit(&picture); + if (!WebPPictureInit(&picture)) return 0; picture.width = width; picture.height = height; picture.use_argb = 1; @@ -86,7 +87,7 @@ static int EncodeLossless(const uint8_t* const data, int width, int height, // a decoder bug related to alpha with color cache. // See: https://code.google.com/p/webp/issues/detail?id=239 // Need to re-enable this later. - ok = (VP8LEncodeStream(&config, &picture, bw, 0 /*use_cache*/) == VP8_ENC_OK); + ok = VP8LEncodeStream(&config, &picture, bw, /*use_cache=*/0); WebPPictureFree(&picture); ok = ok && !bw->error_; if (!ok) { @@ -140,6 +141,11 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height, !reduce_levels, &tmp_bw, &result->stats); if (ok) { output = VP8LBitWriterFinish(&tmp_bw); + if (tmp_bw.error_) { + VP8LBitWriterWipeOut(&tmp_bw); + memset(&result->bw, 0, sizeof(result->bw)); + return 0; + } output_size = VP8LBitWriterNumBytes(&tmp_bw); if (output_size > data_size) { // compressed size is larger than source! Revert to uncompressed mode. @@ -148,6 +154,7 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height, } } else { VP8LBitWriterWipeOut(&tmp_bw); + memset(&result->bw, 0, sizeof(result->bw)); return 0; } } @@ -162,7 +169,7 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height, header = method | (filter << 2); if (reduce_levels) header |= ALPHA_PREPROCESSED_LEVELS << 4; - VP8BitWriterInit(&result->bw, ALPHA_HEADER_LEN + output_size); + if (!VP8BitWriterInit(&result->bw, ALPHA_HEADER_LEN + output_size)) ok = 0; ok = ok && VP8BitWriterAppend(&result->bw, &header, ALPHA_HEADER_LEN); ok = ok && VP8BitWriterAppend(&result->bw, output, output_size); @@ -303,7 +310,7 @@ static int EncodeAlpha(VP8Encoder* const enc, int ok = 1; const int reduce_levels = (quality < 100); - // quick sanity checks + // quick correctness checks assert((uint64_t)data_size == (uint64_t)width * height); // as per spec assert(enc != NULL && pic != NULL && pic->a != NULL); assert(output != NULL && output_size != NULL); @@ -312,11 +319,11 @@ static int EncodeAlpha(VP8Encoder* const enc, assert(filter >= WEBP_FILTER_NONE && filter <= WEBP_FILTER_FAST); if (quality < 0 || quality > 100) { - return 0; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_INVALID_CONFIGURATION); } if (method < ALPHA_NO_COMPRESSION || method > ALPHA_LOSSLESS_COMPRESSION) { - return 0; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_INVALID_CONFIGURATION); } if (method == ALPHA_NO_COMPRESSION) { @@ -326,7 +333,7 @@ static int EncodeAlpha(VP8Encoder* const enc, quant_alpha = (uint8_t*)WebPSafeMalloc(1ULL, data_size); if (quant_alpha == NULL) { - return 0; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); } // Extract alpha data (width x height) from raw_data (stride x height). @@ -346,6 +353,9 @@ static int EncodeAlpha(VP8Encoder* const enc, ok = ApplyFiltersAndEncode(quant_alpha, width, height, data_size, method, filter, reduce_levels, effort_level, output, output_size, pic->stats); + if (!ok) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); // imprecise + } #if !defined(WEBP_DISABLE_STATS) if (pic->stats != NULL) { // need stats? pic->stats->coded_size += (int)(*output_size); @@ -361,7 +371,7 @@ static int EncodeAlpha(VP8Encoder* const enc, //------------------------------------------------------------------------------ // Main calls -static int CompressAlphaJob(void* arg1, void* dummy) { +static int CompressAlphaJob(void* arg1, void* unused) { VP8Encoder* const enc = (VP8Encoder*)arg1; const WebPConfig* config = enc->config_; uint8_t* alpha_data = NULL; @@ -375,13 +385,13 @@ static int CompressAlphaJob(void* arg1, void* dummy) { filter, effort_level, &alpha_data, &alpha_size)) { return 0; } - if (alpha_size != (uint32_t)alpha_size) { // Sanity check. + if (alpha_size != (uint32_t)alpha_size) { // Soundness check. WebPSafeFree(alpha_data); return 0; } enc->alpha_data_size_ = (uint32_t)alpha_size; enc->alpha_data_ = alpha_data; - (void)dummy; + (void)unused; return 1; } @@ -405,7 +415,7 @@ int VP8EncStartAlpha(VP8Encoder* const enc) { WebPWorker* const worker = &enc->alpha_worker_; // Makes sure worker is good to go. if (!WebPGetWorkerInterface()->Reset(worker)) { - return 0; + return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); } WebPGetWorkerInterface()->Launch(worker); return 1; diff --git a/3rdparty/libwebp/src/enc/analysis_enc.c b/3rdparty/libwebp/src/enc/analysis_enc.c index ebb784261c..962eaa998f 100644 --- a/3rdparty/libwebp/src/enc/analysis_enc.c +++ b/3rdparty/libwebp/src/enc/analysis_enc.c @@ -391,12 +391,14 @@ static int DoSegmentsJob(void* arg1, void* arg2) { return ok; } +#ifdef WEBP_USE_THREAD static void MergeJobs(const SegmentJob* const src, SegmentJob* const dst) { int i; for (i = 0; i <= MAX_ALPHA; ++i) dst->alphas[i] += src->alphas[i]; dst->alpha += src->alpha; dst->uv_alpha += src->uv_alpha; } +#endif // initialize the job struct with some tasks to perform static void InitSegmentJob(VP8Encoder* const enc, SegmentJob* const job, @@ -425,10 +427,10 @@ int VP8EncAnalyze(VP8Encoder* const enc) { (enc->method_ <= 1); // for method 0 - 1, we need preds_[] to be filled. if (do_segments) { const int last_row = enc->mb_h_; - // We give a little more than a half work to the main thread. - const int split_row = (9 * last_row + 15) >> 4; const int total_mb = last_row * enc->mb_w_; #ifdef WEBP_USE_THREAD + // We give a little more than a half work to the main thread. + const int split_row = (9 * last_row + 15) >> 4; const int kMinSplitRow = 2; // minimal rows needed for mt to be worth it const int do_mt = (enc->thread_level_ > 0) && (split_row >= kMinSplitRow); #else @@ -438,6 +440,7 @@ int VP8EncAnalyze(VP8Encoder* const enc) { WebPGetWorkerInterface(); SegmentJob main_job; if (do_mt) { +#ifdef WEBP_USE_THREAD SegmentJob side_job; // Note the use of '&' instead of '&&' because we must call the functions // no matter what. @@ -455,6 +458,7 @@ int VP8EncAnalyze(VP8Encoder* const enc) { } worker_interface->End(&side_job.worker); if (ok) MergeJobs(&side_job, &main_job); // merge results together +#endif // WEBP_USE_THREAD } else { // Even for single-thread case, we use the generic Worker tools. InitSegmentJob(enc, &main_job, 0, last_row); @@ -470,6 +474,10 @@ int VP8EncAnalyze(VP8Encoder* const enc) { } else { // Use only one default segment. ResetAllMBInfo(enc); } + if (!ok) { + return WebPEncodingSetError(enc->pic_, + VP8_ENC_ERROR_OUT_OF_MEMORY); // imprecise + } return ok; } diff --git a/3rdparty/libwebp/src/enc/backward_references_cost_enc.c b/3rdparty/libwebp/src/enc/backward_references_cost_enc.c index 516abd73eb..6968ef3c9f 100644 --- a/3rdparty/libwebp/src/enc/backward_references_cost_enc.c +++ b/3rdparty/libwebp/src/enc/backward_references_cost_enc.c @@ -15,10 +15,11 @@ // #include +#include +#include "src/dsp/lossless_common.h" #include "src/enc/backward_references_enc.h" #include "src/enc/histogram_enc.h" -#include "src/dsp/lossless_common.h" #include "src/utils/color_cache_utils.h" #include "src/utils/utils.h" @@ -30,15 +31,15 @@ extern void VP8LBackwardRefsCursorAdd(VP8LBackwardRefs* const refs, const PixOrCopy v); typedef struct { - double alpha_[VALUES_IN_BYTE]; - double red_[VALUES_IN_BYTE]; - double blue_[VALUES_IN_BYTE]; - double distance_[NUM_DISTANCE_CODES]; - double* literal_; + float alpha_[VALUES_IN_BYTE]; + float red_[VALUES_IN_BYTE]; + float blue_[VALUES_IN_BYTE]; + float distance_[NUM_DISTANCE_CODES]; + float* literal_; } CostModel; static void ConvertPopulationCountTableToBitEstimates( - int num_symbols, const uint32_t population_counts[], double output[]) { + int num_symbols, const uint32_t population_counts[], float output[]) { uint32_t sum = 0; int nonzeros = 0; int i; @@ -51,7 +52,7 @@ static void ConvertPopulationCountTableToBitEstimates( if (nonzeros <= 1) { memset(output, 0, num_symbols * sizeof(*output)); } else { - const double logsum = VP8LFastLog2(sum); + const float logsum = VP8LFastLog2(sum); for (i = 0; i < num_symbols; ++i) { output[i] = logsum - VP8LFastLog2(population_counts[i]); } @@ -75,8 +76,8 @@ static int CostModelBuild(CostModel* const m, int xsize, int cache_bits, } ConvertPopulationCountTableToBitEstimates( - VP8LHistogramNumCodes(histo->palette_code_bits_), - histo->literal_, m->literal_); + VP8LHistogramNumCodes(histo->palette_code_bits_), histo->literal_, + m->literal_); ConvertPopulationCountTableToBitEstimates( VALUES_IN_BYTE, histo->red_, m->red_); ConvertPopulationCountTableToBitEstimates( @@ -92,27 +93,27 @@ static int CostModelBuild(CostModel* const m, int xsize, int cache_bits, return ok; } -static WEBP_INLINE double GetLiteralCost(const CostModel* const m, uint32_t v) { +static WEBP_INLINE float GetLiteralCost(const CostModel* const m, uint32_t v) { return m->alpha_[v >> 24] + m->red_[(v >> 16) & 0xff] + m->literal_[(v >> 8) & 0xff] + m->blue_[v & 0xff]; } -static WEBP_INLINE double GetCacheCost(const CostModel* const m, uint32_t idx) { +static WEBP_INLINE float GetCacheCost(const CostModel* const m, uint32_t idx) { const int literal_idx = VALUES_IN_BYTE + NUM_LENGTH_CODES + idx; return m->literal_[literal_idx]; } -static WEBP_INLINE double GetLengthCost(const CostModel* const m, - uint32_t length) { +static WEBP_INLINE float GetLengthCost(const CostModel* const m, + uint32_t length) { int code, extra_bits; VP8LPrefixEncodeBits(length, &code, &extra_bits); return m->literal_[VALUES_IN_BYTE + code] + extra_bits; } -static WEBP_INLINE double GetDistanceCost(const CostModel* const m, - uint32_t distance) { +static WEBP_INLINE float GetDistanceCost(const CostModel* const m, + uint32_t distance) { int code, extra_bits; VP8LPrefixEncodeBits(distance, &code, &extra_bits); return m->distance_[code] + extra_bits; @@ -122,20 +123,20 @@ static WEBP_INLINE void AddSingleLiteralWithCostModel( const uint32_t* const argb, VP8LColorCache* const hashers, const CostModel* const cost_model, int idx, int use_color_cache, float prev_cost, float* const cost, uint16_t* const dist_array) { - double cost_val = prev_cost; + float cost_val = prev_cost; const uint32_t color = argb[idx]; const int ix = use_color_cache ? VP8LColorCacheContains(hashers, color) : -1; if (ix >= 0) { // use_color_cache is true and hashers contains color - const double mul0 = 0.68; + const float mul0 = 0.68f; cost_val += GetCacheCost(cost_model, ix) * mul0; } else { - const double mul1 = 0.82; + const float mul1 = 0.82f; if (use_color_cache) VP8LColorCacheInsert(hashers, color); cost_val += GetLiteralCost(cost_model, color) * mul1; } if (cost[idx] > cost_val) { - cost[idx] = (float)cost_val; + cost[idx] = cost_val; dist_array[idx] = 1; // only one is inserted. } } @@ -172,7 +173,7 @@ struct CostInterval { // The GetLengthCost(cost_model, k) are cached in a CostCacheInterval. typedef struct { - double cost_; + float cost_; int start_; int end_; // Exclusive. } CostCacheInterval; @@ -187,7 +188,7 @@ typedef struct { int count_; // The number of stored intervals. CostCacheInterval* cache_intervals_; size_t cache_intervals_size_; - double cost_cache_[MAX_LENGTH]; // Contains the GetLengthCost(cost_model, k). + float cost_cache_[MAX_LENGTH]; // Contains the GetLengthCost(cost_model, k). float* costs_; uint16_t* dist_array_; // Most of the time, we only need few intervals -> use a free-list, to avoid @@ -262,10 +263,13 @@ static int CostManagerInit(CostManager* const manager, CostManagerInitFreeList(manager); // Fill in the cost_cache_. - manager->cache_intervals_size_ = 1; - manager->cost_cache_[0] = GetLengthCost(cost_model, 0); - for (i = 1; i < cost_cache_size; ++i) { + // Has to be done in two passes due to a GCC bug on i686 + // related to https://gcc.gnu.org/bugzilla/show_bug.cgi?id=323 + for (i = 0; i < cost_cache_size; ++i) { manager->cost_cache_[i] = GetLengthCost(cost_model, i); + } + manager->cache_intervals_size_ = 1; + for (i = 1; i < cost_cache_size; ++i) { // Get the number of bound intervals. if (manager->cost_cache_[i] != manager->cost_cache_[i - 1]) { ++manager->cache_intervals_size_; @@ -294,7 +298,7 @@ static int CostManagerInit(CostManager* const manager, cur->end_ = 1; cur->cost_ = manager->cost_cache_[0]; for (i = 1; i < cost_cache_size; ++i) { - const double cost_val = manager->cost_cache_[i]; + const float cost_val = manager->cost_cache_[i]; if (cost_val != cur->cost_) { ++cur; // Initialize an interval. @@ -303,6 +307,8 @@ static int CostManagerInit(CostManager* const manager, } cur->end_ = i + 1; } + assert((size_t)(cur - manager->cache_intervals_) + 1 == + manager->cache_intervals_size_); } manager->costs_ = (float*)WebPSafeMalloc(pix_count, sizeof(*manager->costs_)); @@ -311,7 +317,7 @@ static int CostManagerInit(CostManager* const manager, return 0; } // Set the initial costs_ high for every pixel as we will keep the minimum. - for (i = 0; i < pix_count; ++i) manager->costs_[i] = 1e38f; + for (i = 0; i < pix_count; ++i) manager->costs_[i] = FLT_MAX; return 1; } @@ -457,7 +463,7 @@ static WEBP_INLINE void InsertInterval(CostManager* const manager, // If handling the interval or one of its subintervals becomes to heavy, its // contribution is added to the costs right away. static WEBP_INLINE void PushInterval(CostManager* const manager, - double distance_cost, int position, + float distance_cost, int position, int len) { size_t i; CostInterval* interval = manager->head_; @@ -474,7 +480,7 @@ static WEBP_INLINE void PushInterval(CostManager* const manager, const int k = j - position; float cost_tmp; assert(k >= 0 && k < MAX_LENGTH); - cost_tmp = (float)(distance_cost + manager->cost_cache_[k]); + cost_tmp = distance_cost + manager->cost_cache_[k]; if (manager->costs_[j] > cost_tmp) { manager->costs_[j] = cost_tmp; @@ -492,7 +498,7 @@ static WEBP_INLINE void PushInterval(CostManager* const manager, const int end = position + (cost_cache_intervals[i].end_ > len ? len : cost_cache_intervals[i].end_); - const float cost = (float)(distance_cost + cost_cache_intervals[i].cost_); + const float cost = distance_cost + cost_cache_intervals[i].cost_; for (; interval != NULL && interval->start_ < end; interval = interval_next) { @@ -570,22 +576,21 @@ static int BackwardReferencesHashChainDistanceOnly( const int pix_count = xsize * ysize; const int use_color_cache = (cache_bits > 0); const size_t literal_array_size = - sizeof(double) * (NUM_LITERAL_CODES + NUM_LENGTH_CODES + - ((cache_bits > 0) ? (1 << cache_bits) : 0)); + sizeof(float) * (VP8LHistogramNumCodes(cache_bits)); const size_t cost_model_size = sizeof(CostModel) + literal_array_size; CostModel* const cost_model = (CostModel*)WebPSafeCalloc(1ULL, cost_model_size); VP8LColorCache hashers; CostManager* cost_manager = - (CostManager*)WebPSafeMalloc(1ULL, sizeof(*cost_manager)); + (CostManager*)WebPSafeCalloc(1ULL, sizeof(*cost_manager)); int offset_prev = -1, len_prev = -1; - double offset_cost = -1; + float offset_cost = -1.f; int first_offset_is_constant = -1; // initialized with 'impossible' value int reach = 0; if (cost_model == NULL || cost_manager == NULL) goto Error; - cost_model->literal_ = (double*)(cost_model + 1); + cost_model->literal_ = (float*)(cost_model + 1); if (use_color_cache) { cc_init = VP8LColorCacheInit(&hashers, cache_bits); if (!cc_init) goto Error; @@ -675,7 +680,7 @@ static int BackwardReferencesHashChainDistanceOnly( } ok = !refs->error_; -Error: + Error: if (cc_init) VP8LColorCacheClear(&hashers); CostManagerClear(cost_manager); WebPSafeFree(cost_model); diff --git a/3rdparty/libwebp/src/enc/backward_references_enc.c b/3rdparty/libwebp/src/enc/backward_references_enc.c index 519b36a091..dc98bf1719 100644 --- a/3rdparty/libwebp/src/enc/backward_references_enc.c +++ b/3rdparty/libwebp/src/enc/backward_references_enc.c @@ -10,6 +10,8 @@ // Author: Jyrki Alakuijala (jyrki@google.com) // +#include "src/enc/backward_references_enc.h" + #include #include #include @@ -17,10 +19,11 @@ #include "src/dsp/dsp.h" #include "src/dsp/lossless.h" #include "src/dsp/lossless_common.h" -#include "src/enc/backward_references_enc.h" #include "src/enc/histogram_enc.h" +#include "src/enc/vp8i_enc.h" #include "src/utils/color_cache_utils.h" #include "src/utils/utils.h" +#include "src/webp/encode.h" #define MIN_BLOCK_SIZE 256 // minimum block size for backward references @@ -255,10 +258,13 @@ static WEBP_INLINE int MaxFindCopyLength(int len) { int VP8LHashChainFill(VP8LHashChain* const p, int quality, const uint32_t* const argb, int xsize, int ysize, - int low_effort) { + int low_effort, const WebPPicture* const pic, + int percent_range, int* const percent) { const int size = xsize * ysize; const int iter_max = GetMaxItersForQuality(quality); const uint32_t window_size = GetWindowSizeForHashChain(quality, xsize); + int remaining_percent = percent_range; + int percent_start = *percent; int pos; int argb_comp; uint32_t base_position; @@ -276,7 +282,12 @@ int VP8LHashChainFill(VP8LHashChain* const p, int quality, hash_to_first_index = (int32_t*)WebPSafeMalloc(HASH_SIZE, sizeof(*hash_to_first_index)); - if (hash_to_first_index == NULL) return 0; + if (hash_to_first_index == NULL) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } + + percent_range = remaining_percent / 2; + remaining_percent -= percent_range; // Set the int32_t array to -1. memset(hash_to_first_index, 0xff, HASH_SIZE * sizeof(*hash_to_first_index)); @@ -323,12 +334,22 @@ int VP8LHashChainFill(VP8LHashChain* const p, int quality, hash_to_first_index[hash_code] = pos++; argb_comp = argb_comp_next; } + + if (!WebPReportProgress( + pic, percent_start + percent_range * pos / (size - 2), percent)) { + WebPSafeFree(hash_to_first_index); + return 0; + } } // Process the penultimate pixel. chain[pos] = hash_to_first_index[GetPixPairHash64(argb + pos)]; WebPSafeFree(hash_to_first_index); + percent_start += percent_range; + if (!WebPReportProgress(pic, percent_start, percent)) return 0; + percent_range = remaining_percent; + // Find the best match interval at each pixel, defined by an offset to the // pixel and a length. The right-most pixel cannot match anything to the right // (hence a best length of 0) and the left-most pixel nothing to the left @@ -417,8 +438,17 @@ int VP8LHashChainFill(VP8LHashChain* const p, int quality, max_base_position = base_position; } } + + if (!WebPReportProgress(pic, + percent_start + percent_range * + (size - 2 - base_position) / + (size - 2), + percent)) { + return 0; + } } - return 1; + + return WebPReportProgress(pic, percent_start + percent_range, percent); } static WEBP_INLINE void AddSingleLiteral(uint32_t pixel, int use_color_cache, @@ -728,7 +758,7 @@ static int CalculateBestCacheSize(const uint32_t* argb, int quality, int* const best_cache_bits) { int i; const int cache_bits_max = (quality <= 25) ? 0 : *best_cache_bits; - double entropy_min = MAX_ENTROPY; + float entropy_min = MAX_ENTROPY; int cc_init[MAX_COLOR_CACHE_BITS + 1] = { 0 }; VP8LColorCache hashers[MAX_COLOR_CACHE_BITS + 1]; VP8LRefsCursor c = VP8LRefsCursorInit(refs); @@ -813,14 +843,14 @@ static int CalculateBestCacheSize(const uint32_t* argb, int quality, } for (i = 0; i <= cache_bits_max; ++i) { - const double entropy = VP8LHistogramEstimateBits(histos[i]); + const float entropy = VP8LHistogramEstimateBits(histos[i]); if (i == 0 || entropy < entropy_min) { entropy_min = entropy; *best_cache_bits = i; } } ok = 1; -Error: + Error: for (i = 0; i <= cache_bits_max; ++i) { if (cc_init[i]) VP8LColorCacheClear(&hashers[i]); VP8LFreeHistogram(histos[i]); @@ -890,7 +920,7 @@ static int GetBackwardReferences(int width, int height, int i, lz77_type; // Index 0 is for a color cache, index 1 for no cache (if needed). int lz77_types_best[2] = {0, 0}; - double bit_costs_best[2] = {DBL_MAX, DBL_MAX}; + float bit_costs_best[2] = {FLT_MAX, FLT_MAX}; VP8LHashChain hash_chain_box; VP8LBackwardRefs* const refs_tmp = &refs[do_no_cache ? 2 : 1]; int status = 0; @@ -902,7 +932,7 @@ static int GetBackwardReferences(int width, int height, for (lz77_type = 1; lz77_types_to_try; lz77_types_to_try &= ~lz77_type, lz77_type <<= 1) { int res = 0; - double bit_cost = 0.; + float bit_cost = 0.f; if ((lz77_types_to_try & lz77_type) == 0) continue; switch (lz77_type) { case kLZ77RLE: @@ -976,15 +1006,16 @@ static int GetBackwardReferences(int width, int height, const VP8LHashChain* const hash_chain_tmp = (lz77_types_best[i] == kLZ77Standard) ? hash_chain : &hash_chain_box; const int cache_bits = (i == 1) ? 0 : *cache_bits_best; - if (VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits, - hash_chain_tmp, &refs[i], - refs_tmp)) { - double bit_cost_trace; - VP8LHistogramCreate(histo, refs_tmp, cache_bits); - bit_cost_trace = VP8LHistogramEstimateBits(histo); - if (bit_cost_trace < bit_costs_best[i]) { - BackwardRefsSwap(refs_tmp, &refs[i]); - } + float bit_cost_trace; + if (!VP8LBackwardReferencesTraceBackwards(width, height, argb, cache_bits, + hash_chain_tmp, &refs[i], + refs_tmp)) { + goto Error; + } + VP8LHistogramCreate(histo, refs_tmp, cache_bits); + bit_cost_trace = VP8LHistogramEstimateBits(histo); + if (bit_cost_trace < bit_costs_best[i]) { + BackwardRefsSwap(refs_tmp, &refs[i]); } } @@ -1000,31 +1031,35 @@ static int GetBackwardReferences(int width, int height, } status = 1; -Error: + Error: VP8LHashChainClear(&hash_chain_box); VP8LFreeHistogram(histo); return status; } -WebPEncodingError VP8LGetBackwardReferences( +int VP8LGetBackwardReferences( int width, int height, const uint32_t* const argb, int quality, int low_effort, int lz77_types_to_try, int cache_bits_max, int do_no_cache, const VP8LHashChain* const hash_chain, VP8LBackwardRefs* const refs, - int* const cache_bits_best) { + int* const cache_bits_best, const WebPPicture* const pic, int percent_range, + int* const percent) { if (low_effort) { VP8LBackwardRefs* refs_best; *cache_bits_best = cache_bits_max; refs_best = GetBackwardReferencesLowEffort( width, height, argb, cache_bits_best, hash_chain, refs); - if (refs_best == NULL) return VP8_ENC_ERROR_OUT_OF_MEMORY; + if (refs_best == NULL) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } // Set it in first position. BackwardRefsSwap(refs_best, &refs[0]); } else { if (!GetBackwardReferences(width, height, argb, quality, lz77_types_to_try, cache_bits_max, do_no_cache, hash_chain, refs, cache_bits_best)) { - return VP8_ENC_ERROR_OUT_OF_MEMORY; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); } } - return VP8_ENC_OK; + + return WebPReportProgress(pic, *percent + percent_range, percent); } diff --git a/3rdparty/libwebp/src/enc/backward_references_enc.h b/3rdparty/libwebp/src/enc/backward_references_enc.h index 4c0267b41e..4dff1c27b5 100644 --- a/3rdparty/libwebp/src/enc/backward_references_enc.h +++ b/3rdparty/libwebp/src/enc/backward_references_enc.h @@ -134,10 +134,11 @@ struct VP8LHashChain { // Must be called first, to set size. int VP8LHashChainInit(VP8LHashChain* const p, int size); -// Pre-compute the best matches for argb. +// Pre-compute the best matches for argb. pic and percent are for progress. int VP8LHashChainFill(VP8LHashChain* const p, int quality, const uint32_t* const argb, int xsize, int ysize, - int low_effort); + int low_effort, const WebPPicture* const pic, + int percent_range, int* const percent); void VP8LHashChainClear(VP8LHashChain* const p); // release memory static WEBP_INLINE int VP8LHashChainFindOffset(const VP8LHashChain* const p, @@ -227,11 +228,14 @@ enum VP8LLZ77Type { // VP8LBackwardRefs is put in the first element, the best value with no-cache in // the second element. // In both cases, the last element is used as temporary internally. -WebPEncodingError VP8LGetBackwardReferences( +// pic and percent are for progress. +// Returns false in case of error (stored in pic->error_code). +int VP8LGetBackwardReferences( int width, int height, const uint32_t* const argb, int quality, int low_effort, int lz77_types_to_try, int cache_bits_max, int do_no_cache, const VP8LHashChain* const hash_chain, VP8LBackwardRefs* const refs, - int* const cache_bits_best); + int* const cache_bits_best, const WebPPicture* const pic, int percent_range, + int* const percent); #ifdef __cplusplus } diff --git a/3rdparty/libwebp/src/enc/frame_enc.c b/3rdparty/libwebp/src/enc/frame_enc.c index af538d83ba..01860ca757 100644 --- a/3rdparty/libwebp/src/enc/frame_enc.c +++ b/3rdparty/libwebp/src/enc/frame_enc.c @@ -578,7 +578,7 @@ static uint64_t OneStatPass(VP8Encoder* const enc, VP8RDLevel rd_opt, uint64_t size = 0; uint64_t size_p0 = 0; uint64_t distortion = 0; - const uint64_t pixel_count = nb_mbs * 384; + const uint64_t pixel_count = (uint64_t)nb_mbs * 384; VP8IteratorInit(enc, &it); SetLoopParams(enc, s->q); @@ -689,7 +689,7 @@ static int PreLoopInitialize(VP8Encoder* const enc) { } if (!ok) { VP8EncFreeBitWriters(enc); // malloc error occurred - WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); + return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); } return ok; } @@ -719,6 +719,7 @@ static int PostLoopFinalize(VP8EncIterator* const it, int ok) { } else { // Something bad happened -> need to do some memory cleanup. VP8EncFreeBitWriters(enc); + return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); } return ok; } @@ -754,6 +755,11 @@ int VP8EncLoop(VP8Encoder* const enc) { // *then* decide how to code the skip decision if there's one. if (!VP8Decimate(&it, &info, rd_opt) || dont_use_skip) { CodeResiduals(it.bw_, &it, &info); + if (it.bw_->error_) { + // enc->pic_->error_code is set in PostLoopFinalize(). + ok = 0; + break; + } } else { // reset predictors after a skip ResetAfterSkip(&it); } @@ -778,11 +784,12 @@ int VP8EncTokenLoop(VP8Encoder* const enc) { // Roughly refresh the proba eight times per pass int max_count = (enc->mb_w_ * enc->mb_h_) >> 3; int num_pass_left = enc->config_->pass; + int remaining_progress = 40; // percents const int do_search = enc->do_search_; VP8EncIterator it; VP8EncProba* const proba = &enc->proba_; const VP8RDLevel rd_opt = enc->rd_opt_level_; - const uint64_t pixel_count = enc->mb_w_ * enc->mb_h_ * 384; + const uint64_t pixel_count = (uint64_t)enc->mb_w_ * enc->mb_h_ * 384; PassStats stats; int ok; @@ -805,6 +812,9 @@ int VP8EncTokenLoop(VP8Encoder* const enc) { uint64_t size_p0 = 0; uint64_t distortion = 0; int cnt = max_count; + // The final number of passes is not trivial to know in advance. + const int pass_progress = remaining_progress / (2 + num_pass_left); + remaining_progress -= pass_progress; VP8IteratorInit(enc, &it); SetLoopParams(enc, stats.q); if (is_last_pass) { @@ -832,7 +842,7 @@ int VP8EncTokenLoop(VP8Encoder* const enc) { StoreSideInfo(&it); VP8StoreFilterStats(&it); VP8IteratorExport(&it); - ok = VP8IteratorProgress(&it, 20); + ok = VP8IteratorProgress(&it, pass_progress); } VP8IteratorSaveBoundary(&it); } while (ok && VP8IteratorNext(&it)); @@ -878,7 +888,8 @@ int VP8EncTokenLoop(VP8Encoder* const enc) { ok = VP8EmitTokens(&enc->tokens_, enc->parts_ + 0, (const uint8_t*)proba->coeffs_, 1); } - ok = ok && WebPReportProgress(enc->pic_, enc->percent_ + 20, &enc->percent_); + ok = ok && WebPReportProgress(enc->pic_, enc->percent_ + remaining_progress, + &enc->percent_); return PostLoopFinalize(&it, ok); } diff --git a/3rdparty/libwebp/src/enc/histogram_enc.c b/3rdparty/libwebp/src/enc/histogram_enc.c index edc6e4faa4..3ca67b3ad0 100644 --- a/3rdparty/libwebp/src/enc/histogram_enc.c +++ b/3rdparty/libwebp/src/enc/histogram_enc.c @@ -13,15 +13,17 @@ #include "src/webp/config.h" #endif +#include #include -#include "src/enc/backward_references_enc.h" -#include "src/enc/histogram_enc.h" #include "src/dsp/lossless.h" #include "src/dsp/lossless_common.h" +#include "src/enc/backward_references_enc.h" +#include "src/enc/histogram_enc.h" +#include "src/enc/vp8i_enc.h" #include "src/utils/utils.h" -#define MAX_COST 1.e38 +#define MAX_BIT_COST FLT_MAX // Number of partitions for the three dominant (literal, red and blue) symbol // costs. @@ -228,8 +230,8 @@ void VP8LHistogramAddSinglePixOrCopy(VP8LHistogram* const histo, // ----------------------------------------------------------------------------- // Entropy-related functions. -static WEBP_INLINE double BitsEntropyRefine(const VP8LBitEntropy* entropy) { - double mix; +static WEBP_INLINE float BitsEntropyRefine(const VP8LBitEntropy* entropy) { + float mix; if (entropy->nonzeros < 5) { if (entropy->nonzeros <= 1) { return 0; @@ -238,67 +240,67 @@ static WEBP_INLINE double BitsEntropyRefine(const VP8LBitEntropy* entropy) { // Let's mix in a bit of entropy to favor good clustering when // distributions of these are combined. if (entropy->nonzeros == 2) { - return 0.99 * entropy->sum + 0.01 * entropy->entropy; + return 0.99f * entropy->sum + 0.01f * entropy->entropy; } // No matter what the entropy says, we cannot be better than min_limit // with Huffman coding. I am mixing a bit of entropy into the // min_limit since it produces much better (~0.5 %) compression results // perhaps because of better entropy clustering. if (entropy->nonzeros == 3) { - mix = 0.95; + mix = 0.95f; } else { - mix = 0.7; // nonzeros == 4. + mix = 0.7f; // nonzeros == 4. } } else { - mix = 0.627; + mix = 0.627f; } { - double min_limit = 2 * entropy->sum - entropy->max_val; - min_limit = mix * min_limit + (1.0 - mix) * entropy->entropy; + float min_limit = 2.f * entropy->sum - entropy->max_val; + min_limit = mix * min_limit + (1.f - mix) * entropy->entropy; return (entropy->entropy < min_limit) ? min_limit : entropy->entropy; } } -double VP8LBitsEntropy(const uint32_t* const array, int n) { +float VP8LBitsEntropy(const uint32_t* const array, int n) { VP8LBitEntropy entropy; VP8LBitsEntropyUnrefined(array, n, &entropy); return BitsEntropyRefine(&entropy); } -static double InitialHuffmanCost(void) { +static float InitialHuffmanCost(void) { // Small bias because Huffman code length is typically not stored in // full length. static const int kHuffmanCodeOfHuffmanCodeSize = CODE_LENGTH_CODES * 3; - static const double kSmallBias = 9.1; + static const float kSmallBias = 9.1f; return kHuffmanCodeOfHuffmanCodeSize - kSmallBias; } // Finalize the Huffman cost based on streak numbers and length type (<3 or >=3) -static double FinalHuffmanCost(const VP8LStreaks* const stats) { +static float FinalHuffmanCost(const VP8LStreaks* const stats) { // The constants in this function are experimental and got rounded from // their original values in 1/8 when switched to 1/1024. - double retval = InitialHuffmanCost(); + float retval = InitialHuffmanCost(); // Second coefficient: Many zeros in the histogram are covered efficiently // by a run-length encode. Originally 2/8. - retval += stats->counts[0] * 1.5625 + 0.234375 * stats->streaks[0][1]; + retval += stats->counts[0] * 1.5625f + 0.234375f * stats->streaks[0][1]; // Second coefficient: Constant values are encoded less efficiently, but still // RLE'ed. Originally 6/8. - retval += stats->counts[1] * 2.578125 + 0.703125 * stats->streaks[1][1]; + retval += stats->counts[1] * 2.578125f + 0.703125f * stats->streaks[1][1]; // 0s are usually encoded more efficiently than non-0s. // Originally 15/8. - retval += 1.796875 * stats->streaks[0][0]; + retval += 1.796875f * stats->streaks[0][0]; // Originally 26/8. - retval += 3.28125 * stats->streaks[1][0]; + retval += 3.28125f * stats->streaks[1][0]; return retval; } // Get the symbol entropy for the distribution 'population'. // Set 'trivial_sym', if there's only one symbol present in the distribution. -static double PopulationCost(const uint32_t* const population, int length, - uint32_t* const trivial_sym, - uint8_t* const is_used) { +static float PopulationCost(const uint32_t* const population, int length, + uint32_t* const trivial_sym, + uint8_t* const is_used) { VP8LBitEntropy bit_entropy; VP8LStreaks stats; VP8LGetEntropyUnrefined(population, length, &bit_entropy, &stats); @@ -314,11 +316,10 @@ static double PopulationCost(const uint32_t* const population, int length, // trivial_at_end is 1 if the two histograms only have one element that is // non-zero: both the zero-th one, or both the last one. -static WEBP_INLINE double GetCombinedEntropy(const uint32_t* const X, - const uint32_t* const Y, - int length, int is_X_used, - int is_Y_used, - int trivial_at_end) { +static WEBP_INLINE float GetCombinedEntropy(const uint32_t* const X, + const uint32_t* const Y, int length, + int is_X_used, int is_Y_used, + int trivial_at_end) { VP8LStreaks stats; if (trivial_at_end) { // This configuration is due to palettization that transforms an indexed @@ -356,16 +357,18 @@ static WEBP_INLINE double GetCombinedEntropy(const uint32_t* const X, } // Estimates the Entropy + Huffman + other block overhead size cost. -double VP8LHistogramEstimateBits(VP8LHistogram* const p) { - return - PopulationCost(p->literal_, VP8LHistogramNumCodes(p->palette_code_bits_), - NULL, &p->is_used_[0]) - + PopulationCost(p->red_, NUM_LITERAL_CODES, NULL, &p->is_used_[1]) - + PopulationCost(p->blue_, NUM_LITERAL_CODES, NULL, &p->is_used_[2]) - + PopulationCost(p->alpha_, NUM_LITERAL_CODES, NULL, &p->is_used_[3]) - + PopulationCost(p->distance_, NUM_DISTANCE_CODES, NULL, &p->is_used_[4]) - + VP8LExtraCost(p->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES) - + VP8LExtraCost(p->distance_, NUM_DISTANCE_CODES); +float VP8LHistogramEstimateBits(VP8LHistogram* const p) { + return PopulationCost(p->literal_, + VP8LHistogramNumCodes(p->palette_code_bits_), NULL, + &p->is_used_[0]) + + PopulationCost(p->red_, NUM_LITERAL_CODES, NULL, &p->is_used_[1]) + + PopulationCost(p->blue_, NUM_LITERAL_CODES, NULL, &p->is_used_[2]) + + PopulationCost(p->alpha_, NUM_LITERAL_CODES, NULL, &p->is_used_[3]) + + PopulationCost(p->distance_, NUM_DISTANCE_CODES, NULL, + &p->is_used_[4]) + + (float)VP8LExtraCost(p->literal_ + NUM_LITERAL_CODES, + NUM_LENGTH_CODES) + + (float)VP8LExtraCost(p->distance_, NUM_DISTANCE_CODES); } // ----------------------------------------------------------------------------- @@ -373,17 +376,16 @@ double VP8LHistogramEstimateBits(VP8LHistogram* const p) { static int GetCombinedHistogramEntropy(const VP8LHistogram* const a, const VP8LHistogram* const b, - double cost_threshold, - double* cost) { + float cost_threshold, float* cost) { const int palette_code_bits = a->palette_code_bits_; int trivial_at_end = 0; assert(a->palette_code_bits_ == b->palette_code_bits_); *cost += GetCombinedEntropy(a->literal_, b->literal_, VP8LHistogramNumCodes(palette_code_bits), a->is_used_[0], b->is_used_[0], 0); - *cost += VP8LExtraCostCombined(a->literal_ + NUM_LITERAL_CODES, - b->literal_ + NUM_LITERAL_CODES, - NUM_LENGTH_CODES); + *cost += (float)VP8LExtraCostCombined(a->literal_ + NUM_LITERAL_CODES, + b->literal_ + NUM_LITERAL_CODES, + NUM_LENGTH_CODES); if (*cost > cost_threshold) return 0; if (a->trivial_symbol_ != VP8L_NON_TRIVIAL_SYM && @@ -417,8 +419,8 @@ static int GetCombinedHistogramEntropy(const VP8LHistogram* const a, *cost += GetCombinedEntropy(a->distance_, b->distance_, NUM_DISTANCE_CODES, a->is_used_[4], b->is_used_[4], 0); - *cost += - VP8LExtraCostCombined(a->distance_, b->distance_, NUM_DISTANCE_CODES); + *cost += (float)VP8LExtraCostCombined(a->distance_, b->distance_, + NUM_DISTANCE_CODES); if (*cost > cost_threshold) return 0; return 1; @@ -439,12 +441,11 @@ static WEBP_INLINE void HistogramAdd(const VP8LHistogram* const a, // Since the previous score passed is 'cost_threshold', we only need to compare // the partial cost against 'cost_threshold + C(a) + C(b)' to possibly bail-out // early. -static double HistogramAddEval(const VP8LHistogram* const a, - const VP8LHistogram* const b, - VP8LHistogram* const out, - double cost_threshold) { - double cost = 0; - const double sum_cost = a->bit_cost_ + b->bit_cost_; +static float HistogramAddEval(const VP8LHistogram* const a, + const VP8LHistogram* const b, + VP8LHistogram* const out, float cost_threshold) { + float cost = 0; + const float sum_cost = a->bit_cost_ + b->bit_cost_; cost_threshold += sum_cost; if (GetCombinedHistogramEntropy(a, b, cost_threshold, &cost)) { @@ -459,10 +460,10 @@ static double HistogramAddEval(const VP8LHistogram* const a, // Same as HistogramAddEval(), except that the resulting histogram // is not stored. Only the cost C(a+b) - C(a) is evaluated. We omit // the term C(b) which is constant over all the evaluations. -static double HistogramAddThresh(const VP8LHistogram* const a, - const VP8LHistogram* const b, - double cost_threshold) { - double cost; +static float HistogramAddThresh(const VP8LHistogram* const a, + const VP8LHistogram* const b, + float cost_threshold) { + float cost; assert(a != NULL && b != NULL); cost = -a->bit_cost_; GetCombinedHistogramEntropy(a, b, cost_threshold, &cost); @@ -473,24 +474,22 @@ static double HistogramAddThresh(const VP8LHistogram* const a, // The structure to keep track of cost range for the three dominant entropy // symbols. -// TODO(skal): Evaluate if float can be used here instead of double for -// representing the entropy costs. typedef struct { - double literal_max_; - double literal_min_; - double red_max_; - double red_min_; - double blue_max_; - double blue_min_; + float literal_max_; + float literal_min_; + float red_max_; + float red_min_; + float blue_max_; + float blue_min_; } DominantCostRange; static void DominantCostRangeInit(DominantCostRange* const c) { c->literal_max_ = 0.; - c->literal_min_ = MAX_COST; + c->literal_min_ = MAX_BIT_COST; c->red_max_ = 0.; - c->red_min_ = MAX_COST; + c->red_min_ = MAX_BIT_COST; c->blue_max_ = 0.; - c->blue_min_ = MAX_COST; + c->blue_min_ = MAX_BIT_COST; } static void UpdateDominantCostRange( @@ -505,16 +504,15 @@ static void UpdateDominantCostRange( static void UpdateHistogramCost(VP8LHistogram* const h) { uint32_t alpha_sym, red_sym, blue_sym; - const double alpha_cost = - PopulationCost(h->alpha_, NUM_LITERAL_CODES, &alpha_sym, - &h->is_used_[3]); - const double distance_cost = + const float alpha_cost = + PopulationCost(h->alpha_, NUM_LITERAL_CODES, &alpha_sym, &h->is_used_[3]); + const float distance_cost = PopulationCost(h->distance_, NUM_DISTANCE_CODES, NULL, &h->is_used_[4]) + - VP8LExtraCost(h->distance_, NUM_DISTANCE_CODES); + (float)VP8LExtraCost(h->distance_, NUM_DISTANCE_CODES); const int num_codes = VP8LHistogramNumCodes(h->palette_code_bits_); h->literal_cost_ = PopulationCost(h->literal_, num_codes, NULL, &h->is_used_[0]) + - VP8LExtraCost(h->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES); + (float)VP8LExtraCost(h->literal_ + NUM_LITERAL_CODES, NUM_LENGTH_CODES); h->red_cost_ = PopulationCost(h->red_, NUM_LITERAL_CODES, &red_sym, &h->is_used_[1]); h->blue_cost_ = @@ -529,10 +527,10 @@ static void UpdateHistogramCost(VP8LHistogram* const h) { } } -static int GetBinIdForEntropy(double min, double max, double val) { - const double range = max - min; +static int GetBinIdForEntropy(float min, float max, float val) { + const float range = max - min; if (range > 0.) { - const double delta = val - min; + const float delta = val - min; return (int)((NUM_PARTITIONS - 1e-6) * delta / range); } else { return 0; @@ -641,15 +639,11 @@ static void HistogramAnalyzeEntropyBin(VP8LHistogramSet* const image_histo, // Merges some histograms with same bin_id together if it's advantageous. // Sets the remaining histograms to NULL. -static void HistogramCombineEntropyBin(VP8LHistogramSet* const image_histo, - int* num_used, - const uint16_t* const clusters, - uint16_t* const cluster_mappings, - VP8LHistogram* cur_combo, - const uint16_t* const bin_map, - int num_bins, - double combine_cost_factor, - int low_effort) { +static void HistogramCombineEntropyBin( + VP8LHistogramSet* const image_histo, int* num_used, + const uint16_t* const clusters, uint16_t* const cluster_mappings, + VP8LHistogram* cur_combo, const uint16_t* const bin_map, int num_bins, + float combine_cost_factor, int low_effort) { VP8LHistogram** const histograms = image_histo->histograms; int idx; struct { @@ -679,11 +673,10 @@ static void HistogramCombineEntropyBin(VP8LHistogramSet* const image_histo, cluster_mappings[clusters[idx]] = clusters[first]; } else { // try to merge #idx into #first (both share the same bin_id) - const double bit_cost = histograms[idx]->bit_cost_; - const double bit_cost_thresh = -bit_cost * combine_cost_factor; - const double curr_cost_diff = - HistogramAddEval(histograms[first], histograms[idx], - cur_combo, bit_cost_thresh); + const float bit_cost = histograms[idx]->bit_cost_; + const float bit_cost_thresh = -bit_cost * combine_cost_factor; + const float curr_cost_diff = HistogramAddEval( + histograms[first], histograms[idx], cur_combo, bit_cost_thresh); if (curr_cost_diff < bit_cost_thresh) { // Try to merge two histograms only if the combo is a trivial one or // the two candidate histograms are already non-trivial. @@ -731,8 +724,8 @@ static uint32_t MyRand(uint32_t* const seed) { typedef struct { int idx1; int idx2; - double cost_diff; - double cost_combo; + float cost_diff; + float cost_combo; } HistogramPair; typedef struct { @@ -787,10 +780,9 @@ static void HistoQueueUpdateHead(HistoQueue* const histo_queue, // Update the cost diff and combo of a pair of histograms. This needs to be // called when the the histograms have been merged with a third one. static void HistoQueueUpdatePair(const VP8LHistogram* const h1, - const VP8LHistogram* const h2, - double threshold, + const VP8LHistogram* const h2, float threshold, HistogramPair* const pair) { - const double sum_cost = h1->bit_cost_ + h2->bit_cost_; + const float sum_cost = h1->bit_cost_ + h2->bit_cost_; pair->cost_combo = 0.; GetCombinedHistogramEntropy(h1, h2, sum_cost + threshold, &pair->cost_combo); pair->cost_diff = pair->cost_combo - sum_cost; @@ -799,9 +791,9 @@ static void HistoQueueUpdatePair(const VP8LHistogram* const h1, // Create a pair from indices "idx1" and "idx2" provided its cost // is inferior to "threshold", a negative entropy. // It returns the cost of the pair, or 0. if it superior to threshold. -static double HistoQueuePush(HistoQueue* const histo_queue, - VP8LHistogram** const histograms, int idx1, - int idx2, double threshold) { +static float HistoQueuePush(HistoQueue* const histo_queue, + VP8LHistogram** const histograms, int idx1, + int idx2, float threshold) { const VP8LHistogram* h1; const VP8LHistogram* h2; HistogramPair pair; @@ -945,8 +937,8 @@ static int HistogramCombineStochastic(VP8LHistogramSet* const image_histo, ++tries_with_no_success < num_tries_no_success; ++iter) { int* mapping_index; - double best_cost = - (histo_queue.size == 0) ? 0. : histo_queue.queue[0].cost_diff; + float best_cost = + (histo_queue.size == 0) ? 0.f : histo_queue.queue[0].cost_diff; int best_idx1 = -1, best_idx2 = 1; const uint32_t rand_range = (*num_used - 1) * (*num_used); // (*num_used) / 2 was chosen empirically. Less means faster but worse @@ -955,7 +947,7 @@ static int HistogramCombineStochastic(VP8LHistogramSet* const image_histo, // Pick random samples. for (j = 0; *num_used >= 2 && j < num_tries; ++j) { - double curr_cost; + float curr_cost; // Choose two different histograms at random and try to combine them. const uint32_t tmp = MyRand(&seed) % rand_range; uint32_t idx1 = tmp / (*num_used - 1); @@ -1034,7 +1026,7 @@ static int HistogramCombineStochastic(VP8LHistogramSet* const image_histo, *do_greedy = (*num_used <= min_cluster_size); ok = 1; -End: + End: HistoQueueClear(&histo_queue); WebPSafeFree(mappings); return ok; @@ -1057,7 +1049,7 @@ static void HistogramRemap(const VP8LHistogramSet* const in, if (out_size > 1) { for (i = 0; i < in_size; ++i) { int best_out = 0; - double best_bits = MAX_COST; + float best_bits = MAX_BIT_COST; int k; if (in_histo[i] == NULL) { // Arbitrarily set to the previous value if unused to help future LZ77. @@ -1065,7 +1057,7 @@ static void HistogramRemap(const VP8LHistogramSet* const in, continue; } for (k = 0; k < out_size; ++k) { - double cur_bits; + float cur_bits; cur_bits = HistogramAddThresh(out_histo[k], in_histo[i], best_bits); if (k == 0 || cur_bits < best_bits) { best_bits = cur_bits; @@ -1093,13 +1085,13 @@ static void HistogramRemap(const VP8LHistogramSet* const in, } } -static double GetCombineCostFactor(int histo_size, int quality) { - double combine_cost_factor = 0.16; +static float GetCombineCostFactor(int histo_size, int quality) { + float combine_cost_factor = 0.16f; if (quality < 90) { - if (histo_size > 256) combine_cost_factor /= 2.; - if (histo_size > 512) combine_cost_factor /= 2.; - if (histo_size > 1024) combine_cost_factor /= 2.; - if (quality <= 50) combine_cost_factor /= 2.; + if (histo_size > 256) combine_cost_factor /= 2.f; + if (histo_size > 512) combine_cost_factor /= 2.f; + if (histo_size > 1024) combine_cost_factor /= 2.f; + if (quality <= 50) combine_cost_factor /= 2.f; } return combine_cost_factor; } @@ -1169,15 +1161,17 @@ static void RemoveEmptyHistograms(VP8LHistogramSet* const image_histo) { } int VP8LGetHistoImageSymbols(int xsize, int ysize, - const VP8LBackwardRefs* const refs, - int quality, int low_effort, - int histo_bits, int cache_bits, + const VP8LBackwardRefs* const refs, int quality, + int low_effort, int histogram_bits, int cache_bits, VP8LHistogramSet* const image_histo, VP8LHistogram* const tmp_histo, - uint16_t* const histogram_symbols) { - int ok = 0; - const int histo_xsize = histo_bits ? VP8LSubSampleSize(xsize, histo_bits) : 1; - const int histo_ysize = histo_bits ? VP8LSubSampleSize(ysize, histo_bits) : 1; + uint16_t* const histogram_symbols, + const WebPPicture* const pic, int percent_range, + int* const percent) { + const int histo_xsize = + histogram_bits ? VP8LSubSampleSize(xsize, histogram_bits) : 1; + const int histo_ysize = + histogram_bits ? VP8LSubSampleSize(ysize, histogram_bits) : 1; const int image_histo_raw_size = histo_xsize * histo_ysize; VP8LHistogramSet* const orig_histo = VP8LAllocateHistogramSet(image_histo_raw_size, cache_bits); @@ -1187,13 +1181,16 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, const int entropy_combine_num_bins = low_effort ? NUM_PARTITIONS : BIN_SIZE; int entropy_combine; uint16_t* const map_tmp = - WebPSafeMalloc(2 * image_histo_raw_size, sizeof(map_tmp)); + WebPSafeMalloc(2 * image_histo_raw_size, sizeof(*map_tmp)); uint16_t* const cluster_mappings = map_tmp + image_histo_raw_size; int num_used = image_histo_raw_size; - if (orig_histo == NULL || map_tmp == NULL) goto Error; + if (orig_histo == NULL || map_tmp == NULL) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } // Construct the histograms from backward references. - HistogramBuild(xsize, histo_bits, refs, orig_histo); + HistogramBuild(xsize, histogram_bits, refs, orig_histo); // Copies the histograms and computes its bit_cost. // histogram_symbols is optimized HistogramCopyAndAnalyze(orig_histo, image_histo, &num_used, @@ -1204,16 +1201,15 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, if (entropy_combine) { uint16_t* const bin_map = map_tmp; - const double combine_cost_factor = + const float combine_cost_factor = GetCombineCostFactor(image_histo_raw_size, quality); const uint32_t num_clusters = num_used; HistogramAnalyzeEntropyBin(image_histo, bin_map, low_effort); // Collapse histograms with similar entropy. - HistogramCombineEntropyBin(image_histo, &num_used, histogram_symbols, - cluster_mappings, tmp_histo, bin_map, - entropy_combine_num_bins, combine_cost_factor, - low_effort); + HistogramCombineEntropyBin( + image_histo, &num_used, histogram_symbols, cluster_mappings, tmp_histo, + bin_map, entropy_combine_num_bins, combine_cost_factor, low_effort); OptimizeHistogramSymbols(image_histo, cluster_mappings, num_clusters, map_tmp, histogram_symbols); } @@ -1227,11 +1223,13 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, int do_greedy; if (!HistogramCombineStochastic(image_histo, &num_used, threshold_size, &do_greedy)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } if (do_greedy) { RemoveEmptyHistograms(image_histo); if (!HistogramCombineGreedy(image_histo, &num_used)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } } @@ -1241,10 +1239,12 @@ int VP8LGetHistoImageSymbols(int xsize, int ysize, RemoveEmptyHistograms(image_histo); HistogramRemap(orig_histo, image_histo, histogram_symbols); - ok = 1; + if (!WebPReportProgress(pic, *percent + percent_range, percent)) { + goto Error; + } Error: VP8LFreeHistogramSet(orig_histo); WebPSafeFree(map_tmp); - return ok; + return (pic->error_code == VP8_ENC_OK); } diff --git a/3rdparty/libwebp/src/enc/histogram_enc.h b/3rdparty/libwebp/src/enc/histogram_enc.h index 54c2d21783..4c0bb97464 100644 --- a/3rdparty/libwebp/src/enc/histogram_enc.h +++ b/3rdparty/libwebp/src/enc/histogram_enc.h @@ -40,10 +40,10 @@ typedef struct { int palette_code_bits_; uint32_t trivial_symbol_; // True, if histograms for Red, Blue & Alpha // literal symbols are single valued. - double bit_cost_; // cached value of bit cost. - double literal_cost_; // Cached values of dominant entropy costs: - double red_cost_; // literal, red & blue. - double blue_cost_; + float bit_cost_; // cached value of bit cost. + float literal_cost_; // Cached values of dominant entropy costs: + float red_cost_; // literal, red & blue. + float blue_cost_; uint8_t is_used_[5]; // 5 for literal, red, blue, alpha, distance } VP8LHistogram; @@ -64,8 +64,8 @@ void VP8LHistogramCreate(VP8LHistogram* const p, const VP8LBackwardRefs* const refs, int palette_code_bits); -// Return the size of the histogram for a given palette_code_bits. -int VP8LGetHistogramSize(int palette_code_bits); +// Return the size of the histogram for a given cache_bits. +int VP8LGetHistogramSize(int cache_bits); // Set the palette_code_bits and reset the stats. // If init_arrays is true, the arrays are also filled with 0's. @@ -105,21 +105,23 @@ static WEBP_INLINE int VP8LHistogramNumCodes(int palette_code_bits) { ((palette_code_bits > 0) ? (1 << palette_code_bits) : 0); } -// Builds the histogram image. +// Builds the histogram image. pic and percent are for progress. +// Returns false in case of error (stored in pic->error_code). int VP8LGetHistoImageSymbols(int xsize, int ysize, - const VP8LBackwardRefs* const refs, - int quality, int low_effort, - int histogram_bits, int cache_bits, - VP8LHistogramSet* const image_in, + const VP8LBackwardRefs* const refs, int quality, + int low_effort, int histogram_bits, int cache_bits, + VP8LHistogramSet* const image_histo, VP8LHistogram* const tmp_histo, - uint16_t* const histogram_symbols); + uint16_t* const histogram_symbols, + const WebPPicture* const pic, int percent_range, + int* const percent); // Returns the entropy for the symbols in the input array. -double VP8LBitsEntropy(const uint32_t* const array, int n); +float VP8LBitsEntropy(const uint32_t* const array, int n); // Estimate how many bits the combined entropy of literals and distance // approximately maps to. -double VP8LHistogramEstimateBits(VP8LHistogram* const p); +float VP8LHistogramEstimateBits(VP8LHistogram* const p); #ifdef __cplusplus } diff --git a/3rdparty/libwebp/src/enc/picture_csp_enc.c b/3rdparty/libwebp/src/enc/picture_csp_enc.c index 35eede9635..a9280e6c30 100644 --- a/3rdparty/libwebp/src/enc/picture_csp_enc.c +++ b/3rdparty/libwebp/src/enc/picture_csp_enc.c @@ -15,12 +15,19 @@ #include #include +#include "sharpyuv/sharpyuv.h" +#include "sharpyuv/sharpyuv_csp.h" #include "src/enc/vp8i_enc.h" #include "src/utils/random_utils.h" #include "src/utils/utils.h" #include "src/dsp/dsp.h" #include "src/dsp/lossless.h" #include "src/dsp/yuv.h" +#include "src/dsp/cpu.h" + +#if defined(WEBP_USE_THREAD) && !defined(_WIN32) +#include +#endif // Uncomment to disable gamma-compression during RGB->U/V averaging #define USE_GAMMA_COMPRESSION @@ -62,10 +69,12 @@ static int CheckNonOpaque(const uint8_t* alpha, int width, int height, int WebPPictureHasTransparency(const WebPPicture* picture) { if (picture == NULL) return 0; if (picture->use_argb) { - const int alpha_offset = ALPHA_OFFSET; - return CheckNonOpaque((const uint8_t*)picture->argb + alpha_offset, - picture->width, picture->height, - 4, picture->argb_stride * sizeof(*picture->argb)); + if (picture->argb != NULL) { + return CheckNonOpaque((const uint8_t*)picture->argb + ALPHA_OFFSET, + picture->width, picture->height, + 4, picture->argb_stride * sizeof(*picture->argb)); + } + return 0; } return CheckNonOpaque(picture->a, picture->width, picture->height, 1, picture->a_stride); @@ -76,30 +85,31 @@ int WebPPictureHasTransparency(const WebPPicture* picture) { #if defined(USE_GAMMA_COMPRESSION) -// gamma-compensates loss of resolution during chroma subsampling -#define kGamma 0.80 // for now we use a different gamma value than kGammaF -#define kGammaFix 12 // fixed-point precision for linear values -#define kGammaScale ((1 << kGammaFix) - 1) -#define kGammaTabFix 7 // fixed-point fractional bits precision -#define kGammaTabScale (1 << kGammaTabFix) -#define kGammaTabRounder (kGammaTabScale >> 1) -#define kGammaTabSize (1 << (kGammaFix - kGammaTabFix)) +// Gamma correction compensates loss of resolution during chroma subsampling. +#define GAMMA_FIX 12 // fixed-point precision for linear values +#define GAMMA_TAB_FIX 7 // fixed-point fractional bits precision +#define GAMMA_TAB_SIZE (1 << (GAMMA_FIX - GAMMA_TAB_FIX)) +static const double kGamma = 0.80; +static const int kGammaScale = ((1 << GAMMA_FIX) - 1); +static const int kGammaTabScale = (1 << GAMMA_TAB_FIX); +static const int kGammaTabRounder = (1 << GAMMA_TAB_FIX >> 1); -static int kLinearToGammaTab[kGammaTabSize + 1]; +static int kLinearToGammaTab[GAMMA_TAB_SIZE + 1]; static uint16_t kGammaToLinearTab[256]; static volatile int kGammaTablesOk = 0; static void InitGammaTables(void); +extern VP8CPUInfo VP8GetCPUInfo; WEBP_DSP_INIT_FUNC(InitGammaTables) { if (!kGammaTablesOk) { int v; - const double scale = (double)(1 << kGammaTabFix) / kGammaScale; + const double scale = (double)(1 << GAMMA_TAB_FIX) / kGammaScale; const double norm = 1. / 255.; for (v = 0; v <= 255; ++v) { kGammaToLinearTab[v] = (uint16_t)(pow(norm * v, kGamma) * kGammaScale + .5); } - for (v = 0; v <= kGammaTabSize; ++v) { + for (v = 0; v <= GAMMA_TAB_SIZE; ++v) { kLinearToGammaTab[v] = (int)(255. * pow(scale * v, 1. / kGamma) + .5); } kGammaTablesOk = 1; @@ -111,12 +121,12 @@ static WEBP_INLINE uint32_t GammaToLinear(uint8_t v) { } static WEBP_INLINE int Interpolate(int v) { - const int tab_pos = v >> (kGammaTabFix + 2); // integer part + const int tab_pos = v >> (GAMMA_TAB_FIX + 2); // integer part const int x = v & ((kGammaTabScale << 2) - 1); // fractional part const int v0 = kLinearToGammaTab[tab_pos]; const int v1 = kLinearToGammaTab[tab_pos + 1]; const int y = v1 * x + v0 * ((kGammaTabScale << 2) - x); // interpolate - assert(tab_pos + 1 < kGammaTabSize + 1); + assert(tab_pos + 1 < GAMMA_TAB_SIZE + 1); return y; } @@ -124,7 +134,7 @@ static WEBP_INLINE int Interpolate(int v) { // U/V value, suitable for RGBToU/V calls. static WEBP_INLINE int LinearToGamma(uint32_t base_value, int shift) { const int y = Interpolate(base_value << shift); // final uplifted value - return (y + kGammaTabRounder) >> kGammaTabFix; // descale + return (y + kGammaTabRounder) >> GAMMA_TAB_FIX; // descale } #else @@ -158,415 +168,26 @@ static int RGBToV(int r, int g, int b, VP8Random* const rg) { //------------------------------------------------------------------------------ // Sharp RGB->YUV conversion -static const int kNumIterations = 4; static const int kMinDimensionIterativeConversion = 4; -// We could use SFIX=0 and only uint8_t for fixed_y_t, but it produces some -// banding sometimes. Better use extra precision. -#define SFIX 2 // fixed-point precision of RGB and Y/W -typedef int16_t fixed_t; // signed type with extra SFIX precision for UV -typedef uint16_t fixed_y_t; // unsigned type with extra SFIX precision for W - -#define SHALF (1 << SFIX >> 1) -#define MAX_Y_T ((256 << SFIX) - 1) -#define SROUNDER (1 << (YUV_FIX + SFIX - 1)) - -#if defined(USE_GAMMA_COMPRESSION) - -// We use tables of different size and precision for the Rec709 / BT2020 -// transfer function. -#define kGammaF (1./0.45) -static uint32_t kLinearToGammaTabS[kGammaTabSize + 2]; -#define GAMMA_TO_LINEAR_BITS 14 -static uint32_t kGammaToLinearTabS[MAX_Y_T + 1]; // size scales with Y_FIX -static volatile int kGammaTablesSOk = 0; -static void InitGammaTablesS(void); - -WEBP_DSP_INIT_FUNC(InitGammaTablesS) { - assert(2 * GAMMA_TO_LINEAR_BITS < 32); // we use uint32_t intermediate values - if (!kGammaTablesSOk) { - int v; - const double norm = 1. / MAX_Y_T; - const double scale = 1. / kGammaTabSize; - const double a = 0.09929682680944; - const double thresh = 0.018053968510807; - const double final_scale = 1 << GAMMA_TO_LINEAR_BITS; - for (v = 0; v <= MAX_Y_T; ++v) { - const double g = norm * v; - double value; - if (g <= thresh * 4.5) { - value = g / 4.5; - } else { - const double a_rec = 1. / (1. + a); - value = pow(a_rec * (g + a), kGammaF); - } - kGammaToLinearTabS[v] = (uint32_t)(value * final_scale + .5); - } - for (v = 0; v <= kGammaTabSize; ++v) { - const double g = scale * v; - double value; - if (g <= thresh) { - value = 4.5 * g; - } else { - value = (1. + a) * pow(g, 1. / kGammaF) - a; - } - // we already incorporate the 1/2 rounding constant here - kLinearToGammaTabS[v] = - (uint32_t)(MAX_Y_T * value) + (1 << GAMMA_TO_LINEAR_BITS >> 1); - } - // to prevent small rounding errors to cause read-overflow: - kLinearToGammaTabS[kGammaTabSize + 1] = kLinearToGammaTabS[kGammaTabSize]; - kGammaTablesSOk = 1; - } -} - -// return value has a fixed-point precision of GAMMA_TO_LINEAR_BITS -static WEBP_INLINE uint32_t GammaToLinearS(int v) { - return kGammaToLinearTabS[v]; -} - -static WEBP_INLINE uint32_t LinearToGammaS(uint32_t value) { - // 'value' is in GAMMA_TO_LINEAR_BITS fractional precision - const uint32_t v = value * kGammaTabSize; - const uint32_t tab_pos = v >> GAMMA_TO_LINEAR_BITS; - // fractional part, in GAMMA_TO_LINEAR_BITS fixed-point precision - const uint32_t x = v - (tab_pos << GAMMA_TO_LINEAR_BITS); // fractional part - // v0 / v1 are in GAMMA_TO_LINEAR_BITS fixed-point precision (range [0..1]) - const uint32_t v0 = kLinearToGammaTabS[tab_pos + 0]; - const uint32_t v1 = kLinearToGammaTabS[tab_pos + 1]; - // Final interpolation. Note that rounding is already included. - const uint32_t v2 = (v1 - v0) * x; // note: v1 >= v0. - const uint32_t result = v0 + (v2 >> GAMMA_TO_LINEAR_BITS); - return result; -} - -#else - -static void InitGammaTablesS(void) {} -static WEBP_INLINE uint32_t GammaToLinearS(int v) { - return (v << GAMMA_TO_LINEAR_BITS) / MAX_Y_T; -} -static WEBP_INLINE uint32_t LinearToGammaS(uint32_t value) { - return (MAX_Y_T * value) >> GAMMA_TO_LINEAR_BITS; -} - -#endif // USE_GAMMA_COMPRESSION - -//------------------------------------------------------------------------------ - -static uint8_t clip_8b(fixed_t v) { - return (!(v & ~0xff)) ? (uint8_t)v : (v < 0) ? 0u : 255u; -} - -static fixed_y_t clip_y(int y) { - return (!(y & ~MAX_Y_T)) ? (fixed_y_t)y : (y < 0) ? 0 : MAX_Y_T; -} - -//------------------------------------------------------------------------------ - -static int RGBToGray(int r, int g, int b) { - const int luma = 13933 * r + 46871 * g + 4732 * b + YUV_HALF; - return (luma >> YUV_FIX); -} - -static uint32_t ScaleDown(int a, int b, int c, int d) { - const uint32_t A = GammaToLinearS(a); - const uint32_t B = GammaToLinearS(b); - const uint32_t C = GammaToLinearS(c); - const uint32_t D = GammaToLinearS(d); - return LinearToGammaS((A + B + C + D + 2) >> 2); -} - -static WEBP_INLINE void UpdateW(const fixed_y_t* src, fixed_y_t* dst, int w) { - int i; - for (i = 0; i < w; ++i) { - const uint32_t R = GammaToLinearS(src[0 * w + i]); - const uint32_t G = GammaToLinearS(src[1 * w + i]); - const uint32_t B = GammaToLinearS(src[2 * w + i]); - const uint32_t Y = RGBToGray(R, G, B); - dst[i] = (fixed_y_t)LinearToGammaS(Y); - } -} - -static void UpdateChroma(const fixed_y_t* src1, const fixed_y_t* src2, - fixed_t* dst, int uv_w) { - int i; - for (i = 0; i < uv_w; ++i) { - const int r = ScaleDown(src1[0 * uv_w + 0], src1[0 * uv_w + 1], - src2[0 * uv_w + 0], src2[0 * uv_w + 1]); - const int g = ScaleDown(src1[2 * uv_w + 0], src1[2 * uv_w + 1], - src2[2 * uv_w + 0], src2[2 * uv_w + 1]); - const int b = ScaleDown(src1[4 * uv_w + 0], src1[4 * uv_w + 1], - src2[4 * uv_w + 0], src2[4 * uv_w + 1]); - const int W = RGBToGray(r, g, b); - dst[0 * uv_w] = (fixed_t)(r - W); - dst[1 * uv_w] = (fixed_t)(g - W); - dst[2 * uv_w] = (fixed_t)(b - W); - dst += 1; - src1 += 2; - src2 += 2; - } -} - -static void StoreGray(const fixed_y_t* rgb, fixed_y_t* y, int w) { - int i; - for (i = 0; i < w; ++i) { - y[i] = RGBToGray(rgb[0 * w + i], rgb[1 * w + i], rgb[2 * w + i]); - } -} - -//------------------------------------------------------------------------------ - -static WEBP_INLINE fixed_y_t Filter2(int A, int B, int W0) { - const int v0 = (A * 3 + B + 2) >> 2; - return clip_y(v0 + W0); -} - -//------------------------------------------------------------------------------ - -static WEBP_INLINE fixed_y_t UpLift(uint8_t a) { // 8bit -> SFIX - return ((fixed_y_t)a << SFIX) | SHALF; -} - -static void ImportOneRow(const uint8_t* const r_ptr, - const uint8_t* const g_ptr, - const uint8_t* const b_ptr, - int step, - int pic_width, - fixed_y_t* const dst) { - int i; - const int w = (pic_width + 1) & ~1; - for (i = 0; i < pic_width; ++i) { - const int off = i * step; - dst[i + 0 * w] = UpLift(r_ptr[off]); - dst[i + 1 * w] = UpLift(g_ptr[off]); - dst[i + 2 * w] = UpLift(b_ptr[off]); - } - if (pic_width & 1) { // replicate rightmost pixel - dst[pic_width + 0 * w] = dst[pic_width + 0 * w - 1]; - dst[pic_width + 1 * w] = dst[pic_width + 1 * w - 1]; - dst[pic_width + 2 * w] = dst[pic_width + 2 * w - 1]; - } -} - -static void InterpolateTwoRows(const fixed_y_t* const best_y, - const fixed_t* prev_uv, - const fixed_t* cur_uv, - const fixed_t* next_uv, - int w, - fixed_y_t* out1, - fixed_y_t* out2) { - const int uv_w = w >> 1; - const int len = (w - 1) >> 1; // length to filter - int k = 3; - while (k-- > 0) { // process each R/G/B segments in turn - // special boundary case for i==0 - out1[0] = Filter2(cur_uv[0], prev_uv[0], best_y[0]); - out2[0] = Filter2(cur_uv[0], next_uv[0], best_y[w]); - - WebPSharpYUVFilterRow(cur_uv, prev_uv, len, best_y + 0 + 1, out1 + 1); - WebPSharpYUVFilterRow(cur_uv, next_uv, len, best_y + w + 1, out2 + 1); - - // special boundary case for i == w - 1 when w is even - if (!(w & 1)) { - out1[w - 1] = Filter2(cur_uv[uv_w - 1], prev_uv[uv_w - 1], - best_y[w - 1 + 0]); - out2[w - 1] = Filter2(cur_uv[uv_w - 1], next_uv[uv_w - 1], - best_y[w - 1 + w]); - } - out1 += w; - out2 += w; - prev_uv += uv_w; - cur_uv += uv_w; - next_uv += uv_w; - } -} - -static WEBP_INLINE uint8_t ConvertRGBToY(int r, int g, int b) { - const int luma = 16839 * r + 33059 * g + 6420 * b + SROUNDER; - return clip_8b(16 + (luma >> (YUV_FIX + SFIX))); -} - -static WEBP_INLINE uint8_t ConvertRGBToU(int r, int g, int b) { - const int u = -9719 * r - 19081 * g + 28800 * b + SROUNDER; - return clip_8b(128 + (u >> (YUV_FIX + SFIX))); -} - -static WEBP_INLINE uint8_t ConvertRGBToV(int r, int g, int b) { - const int v = +28800 * r - 24116 * g - 4684 * b + SROUNDER; - return clip_8b(128 + (v >> (YUV_FIX + SFIX))); -} - -static int ConvertWRGBToYUV(const fixed_y_t* best_y, const fixed_t* best_uv, - WebPPicture* const picture) { - int i, j; - uint8_t* dst_y = picture->y; - uint8_t* dst_u = picture->u; - uint8_t* dst_v = picture->v; - const fixed_t* const best_uv_base = best_uv; - const int w = (picture->width + 1) & ~1; - const int h = (picture->height + 1) & ~1; - const int uv_w = w >> 1; - const int uv_h = h >> 1; - for (best_uv = best_uv_base, j = 0; j < picture->height; ++j) { - for (i = 0; i < picture->width; ++i) { - const int off = (i >> 1); - const int W = best_y[i]; - const int r = best_uv[off + 0 * uv_w] + W; - const int g = best_uv[off + 1 * uv_w] + W; - const int b = best_uv[off + 2 * uv_w] + W; - dst_y[i] = ConvertRGBToY(r, g, b); - } - best_y += w; - best_uv += (j & 1) * 3 * uv_w; - dst_y += picture->y_stride; - } - for (best_uv = best_uv_base, j = 0; j < uv_h; ++j) { - for (i = 0; i < uv_w; ++i) { - const int off = i; - const int r = best_uv[off + 0 * uv_w]; - const int g = best_uv[off + 1 * uv_w]; - const int b = best_uv[off + 2 * uv_w]; - dst_u[i] = ConvertRGBToU(r, g, b); - dst_v[i] = ConvertRGBToV(r, g, b); - } - best_uv += 3 * uv_w; - dst_u += picture->uv_stride; - dst_v += picture->uv_stride; - } - return 1; -} - //------------------------------------------------------------------------------ // Main function -#define SAFE_ALLOC(W, H, T) ((T*)WebPSafeMalloc((W) * (H), sizeof(T))) - static int PreprocessARGB(const uint8_t* r_ptr, const uint8_t* g_ptr, const uint8_t* b_ptr, int step, int rgb_stride, WebPPicture* const picture) { - // we expand the right/bottom border if needed - const int w = (picture->width + 1) & ~1; - const int h = (picture->height + 1) & ~1; - const int uv_w = w >> 1; - const int uv_h = h >> 1; - uint64_t prev_diff_y_sum = ~0; - int j, iter; - - // TODO(skal): allocate one big memory chunk. But for now, it's easier - // for valgrind debugging to have several chunks. - fixed_y_t* const tmp_buffer = SAFE_ALLOC(w * 3, 2, fixed_y_t); // scratch - fixed_y_t* const best_y_base = SAFE_ALLOC(w, h, fixed_y_t); - fixed_y_t* const target_y_base = SAFE_ALLOC(w, h, fixed_y_t); - fixed_y_t* const best_rgb_y = SAFE_ALLOC(w, 2, fixed_y_t); - fixed_t* const best_uv_base = SAFE_ALLOC(uv_w * 3, uv_h, fixed_t); - fixed_t* const target_uv_base = SAFE_ALLOC(uv_w * 3, uv_h, fixed_t); - fixed_t* const best_rgb_uv = SAFE_ALLOC(uv_w * 3, 1, fixed_t); - fixed_y_t* best_y = best_y_base; - fixed_y_t* target_y = target_y_base; - fixed_t* best_uv = best_uv_base; - fixed_t* target_uv = target_uv_base; - const uint64_t diff_y_threshold = (uint64_t)(3.0 * w * h); - int ok; - - if (best_y_base == NULL || best_uv_base == NULL || - target_y_base == NULL || target_uv_base == NULL || - best_rgb_y == NULL || best_rgb_uv == NULL || - tmp_buffer == NULL) { - ok = WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); - goto End; + const int ok = SharpYuvConvert( + r_ptr, g_ptr, b_ptr, step, rgb_stride, /*rgb_bit_depth=*/8, + picture->y, picture->y_stride, picture->u, picture->uv_stride, picture->v, + picture->uv_stride, /*yuv_bit_depth=*/8, picture->width, + picture->height, SharpYuvGetConversionMatrix(kSharpYuvMatrixWebp)); + if (!ok) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); } - assert(picture->width >= kMinDimensionIterativeConversion); - assert(picture->height >= kMinDimensionIterativeConversion); - - WebPInitConvertARGBToYUV(); - - // Import RGB samples to W/RGB representation. - for (j = 0; j < picture->height; j += 2) { - const int is_last_row = (j == picture->height - 1); - fixed_y_t* const src1 = tmp_buffer + 0 * w; - fixed_y_t* const src2 = tmp_buffer + 3 * w; - - // prepare two rows of input - ImportOneRow(r_ptr, g_ptr, b_ptr, step, picture->width, src1); - if (!is_last_row) { - ImportOneRow(r_ptr + rgb_stride, g_ptr + rgb_stride, b_ptr + rgb_stride, - step, picture->width, src2); - } else { - memcpy(src2, src1, 3 * w * sizeof(*src2)); - } - StoreGray(src1, best_y + 0, w); - StoreGray(src2, best_y + w, w); - - UpdateW(src1, target_y, w); - UpdateW(src2, target_y + w, w); - UpdateChroma(src1, src2, target_uv, uv_w); - memcpy(best_uv, target_uv, 3 * uv_w * sizeof(*best_uv)); - best_y += 2 * w; - best_uv += 3 * uv_w; - target_y += 2 * w; - target_uv += 3 * uv_w; - r_ptr += 2 * rgb_stride; - g_ptr += 2 * rgb_stride; - b_ptr += 2 * rgb_stride; - } - - // Iterate and resolve clipping conflicts. - for (iter = 0; iter < kNumIterations; ++iter) { - const fixed_t* cur_uv = best_uv_base; - const fixed_t* prev_uv = best_uv_base; - uint64_t diff_y_sum = 0; - - best_y = best_y_base; - best_uv = best_uv_base; - target_y = target_y_base; - target_uv = target_uv_base; - for (j = 0; j < h; j += 2) { - fixed_y_t* const src1 = tmp_buffer + 0 * w; - fixed_y_t* const src2 = tmp_buffer + 3 * w; - { - const fixed_t* const next_uv = cur_uv + ((j < h - 2) ? 3 * uv_w : 0); - InterpolateTwoRows(best_y, prev_uv, cur_uv, next_uv, w, src1, src2); - prev_uv = cur_uv; - cur_uv = next_uv; - } - - UpdateW(src1, best_rgb_y + 0 * w, w); - UpdateW(src2, best_rgb_y + 1 * w, w); - UpdateChroma(src1, src2, best_rgb_uv, uv_w); - - // update two rows of Y and one row of RGB - diff_y_sum += WebPSharpYUVUpdateY(target_y, best_rgb_y, best_y, 2 * w); - WebPSharpYUVUpdateRGB(target_uv, best_rgb_uv, best_uv, 3 * uv_w); - - best_y += 2 * w; - best_uv += 3 * uv_w; - target_y += 2 * w; - target_uv += 3 * uv_w; - } - // test exit condition - if (iter > 0) { - if (diff_y_sum < diff_y_threshold) break; - if (diff_y_sum > prev_diff_y_sum) break; - } - prev_diff_y_sum = diff_y_sum; - } - // final reconstruction - ok = ConvertWRGBToYUV(best_y_base, best_uv_base, picture); - - End: - WebPSafeFree(best_y_base); - WebPSafeFree(best_uv_base); - WebPSafeFree(target_y_base); - WebPSafeFree(target_uv_base); - WebPSafeFree(best_rgb_y); - WebPSafeFree(best_rgb_uv); - WebPSafeFree(tmp_buffer); return ok; } -#undef SAFE_ALLOC //------------------------------------------------------------------------------ // "Fast" regular RGB->YUV @@ -591,8 +212,8 @@ static const int kAlphaFix = 19; // and constant are adjusted very tightly to fit 32b arithmetic. // In particular, they use the fact that the operands for 'v / a' are actually // derived as v = (a0.p0 + a1.p1 + a2.p2 + a3.p3) and a = a0 + a1 + a2 + a3 -// with ai in [0..255] and pi in [0..1<> 1); ++y) { @@ -1044,7 +678,7 @@ int WebPPictureYUVAToARGB(WebPPicture* picture) { return WebPEncodingSetError(picture, VP8_ENC_ERROR_INVALID_CONFIGURATION); } // Allocate a new argb buffer (discarding the previous one). - if (!WebPPictureAllocARGB(picture, picture->width, picture->height)) return 0; + if (!WebPPictureAllocARGB(picture)) return 0; picture->use_argb = 1; // Convert @@ -1106,6 +740,8 @@ static int Import(WebPPicture* const picture, const int width = picture->width; const int height = picture->height; + if (abs(rgb_stride) < (import_alpha ? 4 : 3) * width) return 0; + if (!picture->use_argb) { const uint8_t* a_ptr = import_alpha ? rgb + 3 : NULL; return ImportYUVAFromRGBA(r_ptr, g_ptr, b_ptr, a_ptr, step, rgb_stride, @@ -1163,24 +799,24 @@ static int Import(WebPPicture* const picture, #if !defined(WEBP_REDUCE_CSP) int WebPPictureImportBGR(WebPPicture* picture, - const uint8_t* rgb, int rgb_stride) { - return (picture != NULL && rgb != NULL) - ? Import(picture, rgb, rgb_stride, 3, 1, 0) + const uint8_t* bgr, int bgr_stride) { + return (picture != NULL && bgr != NULL) + ? Import(picture, bgr, bgr_stride, 3, 1, 0) : 0; } int WebPPictureImportBGRA(WebPPicture* picture, - const uint8_t* rgba, int rgba_stride) { - return (picture != NULL && rgba != NULL) - ? Import(picture, rgba, rgba_stride, 4, 1, 1) + const uint8_t* bgra, int bgra_stride) { + return (picture != NULL && bgra != NULL) + ? Import(picture, bgra, bgra_stride, 4, 1, 1) : 0; } int WebPPictureImportBGRX(WebPPicture* picture, - const uint8_t* rgba, int rgba_stride) { - return (picture != NULL && rgba != NULL) - ? Import(picture, rgba, rgba_stride, 4, 1, 0) + const uint8_t* bgrx, int bgrx_stride) { + return (picture != NULL && bgrx != NULL) + ? Import(picture, bgrx, bgrx_stride, 4, 1, 0) : 0; } @@ -1201,9 +837,9 @@ int WebPPictureImportRGBA(WebPPicture* picture, } int WebPPictureImportRGBX(WebPPicture* picture, - const uint8_t* rgba, int rgba_stride) { - return (picture != NULL && rgba != NULL) - ? Import(picture, rgba, rgba_stride, 4, 0, 0) + const uint8_t* rgbx, int rgbx_stride) { + return (picture != NULL && rgbx != NULL) + ? Import(picture, rgbx, rgbx_stride, 4, 0, 0) : 0; } diff --git a/3rdparty/libwebp/src/enc/picture_enc.c b/3rdparty/libwebp/src/enc/picture_enc.c index c691622d03..5a2703541f 100644 --- a/3rdparty/libwebp/src/enc/picture_enc.c +++ b/3rdparty/libwebp/src/enc/picture_enc.c @@ -12,10 +12,10 @@ // Author: Skal (pascal.massimino@gmail.com) #include +#include #include #include "src/enc/vp8i_enc.h" -#include "src/dsp/dsp.h" #include "src/utils/utils.h" //------------------------------------------------------------------------------ @@ -45,6 +45,22 @@ int WebPPictureInitInternal(WebPPicture* picture, int version) { //------------------------------------------------------------------------------ +int WebPValidatePicture(const WebPPicture* const picture) { + if (picture == NULL) return 0; + if (picture->width <= 0 || picture->height <= 0) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } + if (picture->width <= 0 || picture->width / 4 > INT_MAX / 4 || + picture->height <= 0 || picture->height / 4 > INT_MAX / 4) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } + if (picture->colorspace != WEBP_YUV420 && + picture->colorspace != WEBP_YUV420A) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_INVALID_CONFIGURATION); + } + return 1; +} + static void WebPPictureResetBufferARGB(WebPPicture* const picture) { picture->memory_argb_ = NULL; picture->argb = NULL; @@ -63,18 +79,17 @@ void WebPPictureResetBuffers(WebPPicture* const picture) { WebPPictureResetBufferYUVA(picture); } -int WebPPictureAllocARGB(WebPPicture* const picture, int width, int height) { +int WebPPictureAllocARGB(WebPPicture* const picture) { void* memory; + const int width = picture->width; + const int height = picture->height; const uint64_t argb_size = (uint64_t)width * height; - assert(picture != NULL); + if (!WebPValidatePicture(picture)) return 0; WebPSafeFree(picture->memory_argb_); WebPPictureResetBufferARGB(picture); - if (width <= 0 || height <= 0) { - return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); - } // allocate a new buffer. memory = WebPSafeMalloc(argb_size + WEBP_ALIGN_CST, sizeof(*picture->argb)); if (memory == NULL) { @@ -86,10 +101,10 @@ int WebPPictureAllocARGB(WebPPicture* const picture, int width, int height) { return 1; } -int WebPPictureAllocYUVA(WebPPicture* const picture, int width, int height) { - const WebPEncCSP uv_csp = - (WebPEncCSP)((int)picture->colorspace & WEBP_CSP_UV_MASK); +int WebPPictureAllocYUVA(WebPPicture* const picture) { const int has_alpha = (int)picture->colorspace & WEBP_CSP_ALPHA_BIT; + const int width = picture->width; + const int height = picture->height; const int y_stride = width; const int uv_width = (int)(((int64_t)width + 1) >> 1); const int uv_height = (int)(((int64_t)height + 1) >> 1); @@ -98,15 +113,11 @@ int WebPPictureAllocYUVA(WebPPicture* const picture, int width, int height) { uint64_t y_size, uv_size, a_size, total_size; uint8_t* mem; - assert(picture != NULL); + if (!WebPValidatePicture(picture)) return 0; WebPSafeFree(picture->memory_); WebPPictureResetBufferYUVA(picture); - if (uv_csp != WEBP_YUV420) { - return WebPEncodingSetError(picture, VP8_ENC_ERROR_INVALID_CONFIGURATION); - } - // alpha a_width = has_alpha ? width : 0; a_stride = a_width; @@ -152,15 +163,12 @@ int WebPPictureAllocYUVA(WebPPicture* const picture, int width, int height) { int WebPPictureAlloc(WebPPicture* picture) { if (picture != NULL) { - const int width = picture->width; - const int height = picture->height; - WebPPictureFree(picture); // erase previous buffer if (!picture->use_argb) { - return WebPPictureAllocYUVA(picture, width, height); + return WebPPictureAllocYUVA(picture); } else { - return WebPPictureAllocARGB(picture, width, height); + return WebPPictureAllocARGB(picture); } } return 1; diff --git a/3rdparty/libwebp/src/enc/picture_rescale_enc.c b/3rdparty/libwebp/src/enc/picture_rescale_enc.c index 58a6ae7b9d..ea90d82548 100644 --- a/3rdparty/libwebp/src/enc/picture_rescale_enc.c +++ b/3rdparty/libwebp/src/enc/picture_rescale_enc.c @@ -13,14 +13,15 @@ #include "src/webp/encode.h" -#if !defined(WEBP_REDUCE_SIZE) - #include #include #include "src/enc/vp8i_enc.h" + +#if !defined(WEBP_REDUCE_SIZE) #include "src/utils/rescaler_utils.h" #include "src/utils/utils.h" +#endif // !defined(WEBP_REDUCE_SIZE) #define HALVE(x) (((x) + 1) >> 1) @@ -56,6 +57,7 @@ static int AdjustAndCheckRectangle(const WebPPicture* const pic, return 1; } +#if !defined(WEBP_REDUCE_SIZE) int WebPPictureCopy(const WebPPicture* src, WebPPicture* dst) { if (src == NULL || dst == NULL) return 0; if (src == dst) return 1; @@ -81,6 +83,7 @@ int WebPPictureCopy(const WebPPicture* src, WebPPicture* dst) { } return 1; } +#endif // !defined(WEBP_REDUCE_SIZE) int WebPPictureIsView(const WebPPicture* picture) { if (picture == NULL) return 0; @@ -120,6 +123,7 @@ int WebPPictureView(const WebPPicture* src, return 1; } +#if !defined(WEBP_REDUCE_SIZE) //------------------------------------------------------------------------------ // Picture cropping @@ -133,7 +137,9 @@ int WebPPictureCrop(WebPPicture* pic, PictureGrabSpecs(pic, &tmp); tmp.width = width; tmp.height = height; - if (!WebPPictureAlloc(&tmp)) return 0; + if (!WebPPictureAlloc(&tmp)) { + return WebPEncodingSetError(pic, tmp.error_code); + } if (!pic->use_argb) { const int y_offset = top * pic->y_stride + left; @@ -164,22 +170,25 @@ int WebPPictureCrop(WebPPicture* pic, //------------------------------------------------------------------------------ // Simple picture rescaler -static void RescalePlane(const uint8_t* src, - int src_width, int src_height, int src_stride, - uint8_t* dst, - int dst_width, int dst_height, int dst_stride, - rescaler_t* const work, - int num_channels) { +static int RescalePlane(const uint8_t* src, + int src_width, int src_height, int src_stride, + uint8_t* dst, + int dst_width, int dst_height, int dst_stride, + rescaler_t* const work, + int num_channels) { WebPRescaler rescaler; int y = 0; - WebPRescalerInit(&rescaler, src_width, src_height, - dst, dst_width, dst_height, dst_stride, - num_channels, work); + if (!WebPRescalerInit(&rescaler, src_width, src_height, + dst, dst_width, dst_height, dst_stride, + num_channels, work)) { + return 0; + } while (y < src_height) { y += WebPRescalerImport(&rescaler, src_height - y, src + y * src_stride, src_stride); WebPRescalerExport(&rescaler); } + return 1; } static void AlphaMultiplyARGB(WebPPicture* const pic, int inverse) { @@ -195,73 +204,76 @@ static void AlphaMultiplyY(WebPPicture* const pic, int inverse) { } } -int WebPPictureRescale(WebPPicture* pic, int width, int height) { +int WebPPictureRescale(WebPPicture* picture, int width, int height) { WebPPicture tmp; int prev_width, prev_height; rescaler_t* work; - if (pic == NULL) return 0; - prev_width = pic->width; - prev_height = pic->height; + if (picture == NULL) return 0; + prev_width = picture->width; + prev_height = picture->height; if (!WebPRescalerGetScaledDimensions( prev_width, prev_height, &width, &height)) { - return 0; + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); } - PictureGrabSpecs(pic, &tmp); + PictureGrabSpecs(picture, &tmp); tmp.width = width; tmp.height = height; - if (!WebPPictureAlloc(&tmp)) return 0; + if (!WebPPictureAlloc(&tmp)) { + return WebPEncodingSetError(picture, tmp.error_code); + } - if (!pic->use_argb) { + if (!picture->use_argb) { work = (rescaler_t*)WebPSafeMalloc(2ULL * width, sizeof(*work)); if (work == NULL) { WebPPictureFree(&tmp); - return 0; + return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); } // If present, we need to rescale alpha first (for AlphaMultiplyY). - if (pic->a != NULL) { + if (picture->a != NULL) { WebPInitAlphaProcessing(); - RescalePlane(pic->a, prev_width, prev_height, pic->a_stride, - tmp.a, width, height, tmp.a_stride, work, 1); + if (!RescalePlane(picture->a, prev_width, prev_height, picture->a_stride, + tmp.a, width, height, tmp.a_stride, work, 1)) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } } // We take transparency into account on the luma plane only. That's not // totally exact blending, but still is a good approximation. - AlphaMultiplyY(pic, 0); - RescalePlane(pic->y, prev_width, prev_height, pic->y_stride, - tmp.y, width, height, tmp.y_stride, work, 1); + AlphaMultiplyY(picture, 0); + if (!RescalePlane(picture->y, prev_width, prev_height, picture->y_stride, + tmp.y, width, height, tmp.y_stride, work, 1) || + !RescalePlane(picture->u, HALVE(prev_width), HALVE(prev_height), + picture->uv_stride, tmp.u, HALVE(width), HALVE(height), + tmp.uv_stride, work, 1) || + !RescalePlane(picture->v, HALVE(prev_width), HALVE(prev_height), + picture->uv_stride, tmp.v, HALVE(width), HALVE(height), + tmp.uv_stride, work, 1)) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } AlphaMultiplyY(&tmp, 1); - - RescalePlane(pic->u, - HALVE(prev_width), HALVE(prev_height), pic->uv_stride, - tmp.u, - HALVE(width), HALVE(height), tmp.uv_stride, work, 1); - RescalePlane(pic->v, - HALVE(prev_width), HALVE(prev_height), pic->uv_stride, - tmp.v, - HALVE(width), HALVE(height), tmp.uv_stride, work, 1); } else { work = (rescaler_t*)WebPSafeMalloc(2ULL * width * 4, sizeof(*work)); if (work == NULL) { WebPPictureFree(&tmp); - return 0; + return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); } // In order to correctly interpolate colors, we need to apply the alpha // weighting first (black-matting), scale the RGB values, and remove // the premultiplication afterward (while preserving the alpha channel). WebPInitAlphaProcessing(); - AlphaMultiplyARGB(pic, 0); - RescalePlane((const uint8_t*)pic->argb, prev_width, prev_height, - pic->argb_stride * 4, - (uint8_t*)tmp.argb, width, height, - tmp.argb_stride * 4, - work, 4); + AlphaMultiplyARGB(picture, 0); + if (!RescalePlane((const uint8_t*)picture->argb, prev_width, prev_height, + picture->argb_stride * 4, (uint8_t*)tmp.argb, width, + height, tmp.argb_stride * 4, work, 4)) { + return WebPEncodingSetError(picture, VP8_ENC_ERROR_BAD_DIMENSION); + } AlphaMultiplyARGB(&tmp, 1); } - WebPPictureFree(pic); + WebPPictureFree(picture); WebPSafeFree(work); - *pic = tmp; + *picture = tmp; return 1; } @@ -273,23 +285,6 @@ int WebPPictureCopy(const WebPPicture* src, WebPPicture* dst) { return 0; } -int WebPPictureIsView(const WebPPicture* picture) { - (void)picture; - return 0; -} - -int WebPPictureView(const WebPPicture* src, - int left, int top, int width, int height, - WebPPicture* dst) { - (void)src; - (void)left; - (void)top; - (void)width; - (void)height; - (void)dst; - return 0; -} - int WebPPictureCrop(WebPPicture* pic, int left, int top, int width, int height) { (void)pic; diff --git a/3rdparty/libwebp/src/enc/picture_tools_enc.c b/3rdparty/libwebp/src/enc/picture_tools_enc.c index 38cb01534a..147cc18608 100644 --- a/3rdparty/libwebp/src/enc/picture_tools_enc.c +++ b/3rdparty/libwebp/src/enc/picture_tools_enc.c @@ -190,27 +190,28 @@ static WEBP_INLINE uint32_t MakeARGB32(int r, int g, int b) { return (0xff000000u | (r << 16) | (g << 8) | b); } -void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb) { +void WebPBlendAlpha(WebPPicture* picture, uint32_t background_rgb) { const int red = (background_rgb >> 16) & 0xff; const int green = (background_rgb >> 8) & 0xff; const int blue = (background_rgb >> 0) & 0xff; int x, y; - if (pic == NULL) return; - if (!pic->use_argb) { - const int uv_width = (pic->width >> 1); // omit last pixel during u/v loop + if (picture == NULL) return; + if (!picture->use_argb) { + // omit last pixel during u/v loop + const int uv_width = (picture->width >> 1); const int Y0 = VP8RGBToY(red, green, blue, YUV_HALF); // VP8RGBToU/V expects the u/v values summed over four pixels const int U0 = VP8RGBToU(4 * red, 4 * green, 4 * blue, 4 * YUV_HALF); const int V0 = VP8RGBToV(4 * red, 4 * green, 4 * blue, 4 * YUV_HALF); - const int has_alpha = pic->colorspace & WEBP_CSP_ALPHA_BIT; - uint8_t* y_ptr = pic->y; - uint8_t* u_ptr = pic->u; - uint8_t* v_ptr = pic->v; - uint8_t* a_ptr = pic->a; + const int has_alpha = picture->colorspace & WEBP_CSP_ALPHA_BIT; + uint8_t* y_ptr = picture->y; + uint8_t* u_ptr = picture->u; + uint8_t* v_ptr = picture->v; + uint8_t* a_ptr = picture->a; if (!has_alpha || a_ptr == NULL) return; // nothing to do - for (y = 0; y < pic->height; ++y) { + for (y = 0; y < picture->height; ++y) { // Luma blending - for (x = 0; x < pic->width; ++x) { + for (x = 0; x < picture->width; ++x) { const uint8_t alpha = a_ptr[x]; if (alpha < 0xff) { y_ptr[x] = BLEND(Y0, y_ptr[x], alpha); @@ -219,7 +220,7 @@ void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb) { // Chroma blending every even line if ((y & 1) == 0) { uint8_t* const a_ptr2 = - (y + 1 == pic->height) ? a_ptr : a_ptr + pic->a_stride; + (y + 1 == picture->height) ? a_ptr : a_ptr + picture->a_stride; for (x = 0; x < uv_width; ++x) { // Average four alpha values into a single blending weight. // TODO(skal): might lead to visible contouring. Can we do better? @@ -229,24 +230,24 @@ void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb) { u_ptr[x] = BLEND_10BIT(U0, u_ptr[x], alpha); v_ptr[x] = BLEND_10BIT(V0, v_ptr[x], alpha); } - if (pic->width & 1) { // rightmost pixel + if (picture->width & 1) { // rightmost pixel const uint32_t alpha = 2 * (a_ptr[2 * x + 0] + a_ptr2[2 * x + 0]); u_ptr[x] = BLEND_10BIT(U0, u_ptr[x], alpha); v_ptr[x] = BLEND_10BIT(V0, v_ptr[x], alpha); } } else { - u_ptr += pic->uv_stride; - v_ptr += pic->uv_stride; + u_ptr += picture->uv_stride; + v_ptr += picture->uv_stride; } - memset(a_ptr, 0xff, pic->width); // reset alpha value to opaque - a_ptr += pic->a_stride; - y_ptr += pic->y_stride; + memset(a_ptr, 0xff, picture->width); // reset alpha value to opaque + a_ptr += picture->a_stride; + y_ptr += picture->y_stride; } } else { - uint32_t* argb = pic->argb; + uint32_t* argb = picture->argb; const uint32_t background = MakeARGB32(red, green, blue); - for (y = 0; y < pic->height; ++y) { - for (x = 0; x < pic->width; ++x) { + for (y = 0; y < picture->height; ++y) { + for (x = 0; x < picture->width; ++x) { const int alpha = (argb[x] >> 24) & 0xff; if (alpha != 0xff) { if (alpha > 0) { @@ -262,7 +263,7 @@ void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb) { } } } - argb += pic->argb_stride; + argb += picture->argb_stride; } } } diff --git a/3rdparty/libwebp/src/enc/predictor_enc.c b/3rdparty/libwebp/src/enc/predictor_enc.c index 2e6762ea0d..b3d44b59d5 100644 --- a/3rdparty/libwebp/src/enc/predictor_enc.c +++ b/3rdparty/libwebp/src/enc/predictor_enc.c @@ -16,6 +16,7 @@ #include "src/dsp/lossless.h" #include "src/dsp/lossless_common.h" +#include "src/enc/vp8i_enc.h" #include "src/enc/vp8li_enc.h" #define MAX_DIFF_COST (1e30f) @@ -31,10 +32,10 @@ static WEBP_INLINE int GetMin(int a, int b) { return (a > b) ? b : a; } // Methods to calculate Entropy (Shannon). static float PredictionCostSpatial(const int counts[256], int weight_0, - double exp_val) { + float exp_val) { const int significant_symbols = 256 >> 4; - const double exp_decay_factor = 0.6; - double bits = weight_0 * counts[0]; + const float exp_decay_factor = 0.6f; + float bits = (float)weight_0 * counts[0]; int i; for (i = 1; i < significant_symbols; ++i) { bits += exp_val * (counts[i] + counts[256 - i]); @@ -46,9 +47,9 @@ static float PredictionCostSpatial(const int counts[256], int weight_0, static float PredictionCostSpatialHistogram(const int accumulated[4][256], const int tile[4][256]) { int i; - double retval = 0; + float retval = 0.f; for (i = 0; i < 4; ++i) { - const double kExpValue = 0.94; + const float kExpValue = 0.94f; retval += PredictionCostSpatial(tile[i], 1, kExpValue); retval += VP8LCombinedShannonEntropy(tile[i], accumulated[i]); } @@ -249,7 +250,7 @@ static WEBP_INLINE void GetResidual( } else if (x == 0) { predict = upper_row[x]; // Top. } else { - predict = pred_func(current_row[x - 1], upper_row + x); + predict = pred_func(¤t_row[x - 1], upper_row + x); } #if (WEBP_NEAR_LOSSLESS == 1) if (max_quantization == 1 || mode == 0 || y == 0 || y == height - 1 || @@ -472,12 +473,15 @@ static void CopyImageWithPrediction(int width, int height, // with respect to predictions. If near_lossless_quality < 100, applies // near lossless processing, shaving off more bits of residuals for lower // qualities. -void VP8LResidualImage(int width, int height, int bits, int low_effort, - uint32_t* const argb, uint32_t* const argb_scratch, - uint32_t* const image, int near_lossless_quality, - int exact, int used_subtract_green) { +int VP8LResidualImage(int width, int height, int bits, int low_effort, + uint32_t* const argb, uint32_t* const argb_scratch, + uint32_t* const image, int near_lossless_quality, + int exact, int used_subtract_green, + const WebPPicture* const pic, int percent_range, + int* const percent) { const int tiles_per_row = VP8LSubSampleSize(width, bits); const int tiles_per_col = VP8LSubSampleSize(height, bits); + int percent_start = *percent; int tile_y; int histo[4][256]; const int max_quantization = 1 << VP8LNearLosslessBits(near_lossless_quality); @@ -491,17 +495,24 @@ void VP8LResidualImage(int width, int height, int bits, int low_effort, for (tile_y = 0; tile_y < tiles_per_col; ++tile_y) { int tile_x; for (tile_x = 0; tile_x < tiles_per_row; ++tile_x) { - const int pred = GetBestPredictorForTile(width, height, tile_x, tile_y, - bits, histo, argb_scratch, argb, max_quantization, exact, - used_subtract_green, image); + const int pred = GetBestPredictorForTile( + width, height, tile_x, tile_y, bits, histo, argb_scratch, argb, + max_quantization, exact, used_subtract_green, image); image[tile_y * tiles_per_row + tile_x] = ARGB_BLACK | (pred << 8); } + + if (!WebPReportProgress( + pic, percent_start + percent_range * tile_y / tiles_per_col, + percent)) { + return 0; + } } } CopyImageWithPrediction(width, height, bits, image, argb_scratch, argb, low_effort, max_quantization, exact, used_subtract_green); + return WebPReportProgress(pic, percent_start + percent_range, percent); } //------------------------------------------------------------------------------ @@ -532,7 +543,7 @@ static float PredictionCostCrossColor(const int accumulated[256], const int counts[256]) { // Favor low entropy, locally and globally. // Favor small absolute values for PredictionCostSpatial - static const double kExpValue = 2.4; + static const float kExpValue = 2.4f; return VP8LCombinedShannonEntropy(counts, accumulated) + PredictionCostSpatial(counts, 3, kExpValue); } @@ -714,11 +725,14 @@ static void CopyTileWithColorTransform(int xsize, int ysize, } } -void VP8LColorSpaceTransform(int width, int height, int bits, int quality, - uint32_t* const argb, uint32_t* image) { +int VP8LColorSpaceTransform(int width, int height, int bits, int quality, + uint32_t* const argb, uint32_t* image, + const WebPPicture* const pic, int percent_range, + int* const percent) { const int max_tile_size = 1 << bits; const int tile_xsize = VP8LSubSampleSize(width, bits); const int tile_ysize = VP8LSubSampleSize(height, bits); + int percent_start = *percent; int accumulated_red_histo[256] = { 0 }; int accumulated_blue_histo[256] = { 0 }; int tile_x, tile_y; @@ -768,5 +782,11 @@ void VP8LColorSpaceTransform(int width, int height, int bits, int quality, } } } + if (!WebPReportProgress( + pic, percent_start + percent_range * tile_y / tile_ysize, + percent)) { + return 0; + } } + return 1; } diff --git a/3rdparty/libwebp/src/enc/quant_enc.c b/3rdparty/libwebp/src/enc/quant_enc.c index 01eb565c7f..6d8202d277 100644 --- a/3rdparty/libwebp/src/enc/quant_enc.c +++ b/3rdparty/libwebp/src/enc/quant_enc.c @@ -533,7 +533,8 @@ static void InitScore(VP8ModeScore* const rd) { rd->score = MAX_COST; } -static void CopyScore(VP8ModeScore* const dst, const VP8ModeScore* const src) { +static void CopyScore(VP8ModeScore* WEBP_RESTRICT const dst, + const VP8ModeScore* WEBP_RESTRICT const src) { dst->D = src->D; dst->SD = src->SD; dst->R = src->R; @@ -542,7 +543,8 @@ static void CopyScore(VP8ModeScore* const dst, const VP8ModeScore* const src) { dst->score = src->score; } -static void AddScore(VP8ModeScore* const dst, const VP8ModeScore* const src) { +static void AddScore(VP8ModeScore* WEBP_RESTRICT const dst, + const VP8ModeScore* WEBP_RESTRICT const src) { dst->D += src->D; dst->SD += src->SD; dst->R += src->R; @@ -585,15 +587,18 @@ static WEBP_INLINE score_t RDScoreTrellis(int lambda, score_t rate, return rate * lambda + RD_DISTO_MULT * distortion; } -static int TrellisQuantizeBlock(const VP8Encoder* const enc, +// Coefficient type. +enum { TYPE_I16_AC = 0, TYPE_I16_DC = 1, TYPE_CHROMA_A = 2, TYPE_I4_AC = 3 }; + +static int TrellisQuantizeBlock(const VP8Encoder* WEBP_RESTRICT const enc, int16_t in[16], int16_t out[16], int ctx0, int coeff_type, - const VP8Matrix* const mtx, + const VP8Matrix* WEBP_RESTRICT const mtx, int lambda) { const ProbaArray* const probas = enc->proba_.coeffs_[coeff_type]; CostArrayPtr const costs = (CostArrayPtr)enc->proba_.remapped_costs_[coeff_type]; - const int first = (coeff_type == 0) ? 1 : 0; + const int first = (coeff_type == TYPE_I16_AC) ? 1 : 0; Node nodes[16][NUM_NODES]; ScoreState score_states[2][NUM_NODES]; ScoreState* ss_cur = &SCORE_STATE(0, MIN_DELTA); @@ -657,16 +662,17 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, // test all alternate level values around level0. for (m = -MIN_DELTA; m <= MAX_DELTA; ++m) { Node* const cur = &NODE(n, m); - int level = level0 + m; + const int level = level0 + m; const int ctx = (level > 2) ? 2 : level; const int band = VP8EncBands[n + 1]; score_t base_score; - score_t best_cur_score = MAX_COST; - int best_prev = 0; // default, in case + score_t best_cur_score; + int best_prev; + score_t cost, score; - ss_cur[m].score = MAX_COST; ss_cur[m].costs = costs[n + 1][ctx]; if (level < 0 || level > thresh_level) { + ss_cur[m].score = MAX_COST; // Node is dead. continue; } @@ -682,18 +688,24 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, } // Inspect all possible non-dead predecessors. Retain only the best one. - for (p = -MIN_DELTA; p <= MAX_DELTA; ++p) { + // The base_score is added to all scores so it is only added for the final + // value after the loop. + cost = VP8LevelCost(ss_prev[-MIN_DELTA].costs, level); + best_cur_score = + ss_prev[-MIN_DELTA].score + RDScoreTrellis(lambda, cost, 0); + best_prev = -MIN_DELTA; + for (p = -MIN_DELTA + 1; p <= MAX_DELTA; ++p) { // Dead nodes (with ss_prev[p].score >= MAX_COST) are automatically // eliminated since their score can't be better than the current best. - const score_t cost = VP8LevelCost(ss_prev[p].costs, level); + cost = VP8LevelCost(ss_prev[p].costs, level); // Examine node assuming it's a non-terminal one. - const score_t score = - base_score + ss_prev[p].score + RDScoreTrellis(lambda, cost, 0); + score = ss_prev[p].score + RDScoreTrellis(lambda, cost, 0); if (score < best_cur_score) { best_cur_score = score; best_prev = p; } } + best_cur_score += base_score; // Store best finding in current node. cur->sign = sign; cur->level = level; @@ -701,11 +713,11 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, ss_cur[m].score = best_cur_score; // Now, record best terminal node (and thus best entry in the graph). - if (level != 0) { + if (level != 0 && best_cur_score < best_score) { const score_t last_pos_cost = (n < 15) ? VP8BitCost(0, probas[band][ctx][0]) : 0; const score_t last_pos_score = RDScoreTrellis(lambda, last_pos_cost, 0); - const score_t score = best_cur_score + last_pos_score; + score = best_cur_score + last_pos_score; if (score < best_score) { best_score = score; best_path[0] = n; // best eob position @@ -717,10 +729,16 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, } // Fresh start - memset(in + first, 0, (16 - first) * sizeof(*in)); - memset(out + first, 0, (16 - first) * sizeof(*out)); + // Beware! We must preserve in[0]/out[0] value for TYPE_I16_AC case. + if (coeff_type == TYPE_I16_AC) { + memset(in + 1, 0, 15 * sizeof(*in)); + memset(out + 1, 0, 15 * sizeof(*out)); + } else { + memset(in, 0, 16 * sizeof(*in)); + memset(out, 0, 16 * sizeof(*out)); + } if (best_path[0] == -1) { - return 0; // skip! + return 0; // skip! } { @@ -751,9 +769,9 @@ static int TrellisQuantizeBlock(const VP8Encoder* const enc, // all at once. Output is the reconstructed block in *yuv_out, and the // quantized levels in *levels. -static int ReconstructIntra16(VP8EncIterator* const it, - VP8ModeScore* const rd, - uint8_t* const yuv_out, +static int ReconstructIntra16(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd, + uint8_t* WEBP_RESTRICT const yuv_out, int mode) { const VP8Encoder* const enc = it->enc_; const uint8_t* const ref = it->yuv_p_ + VP8I16ModeOffsets[mode]; @@ -775,9 +793,9 @@ static int ReconstructIntra16(VP8EncIterator* const it, for (y = 0, n = 0; y < 4; ++y) { for (x = 0; x < 4; ++x, ++n) { const int ctx = it->top_nz_[x] + it->left_nz_[y]; - const int non_zero = - TrellisQuantizeBlock(enc, tmp[n], rd->y_ac_levels[n], ctx, 0, - &dqm->y1_, dqm->lambda_trellis_i16_); + const int non_zero = TrellisQuantizeBlock( + enc, tmp[n], rd->y_ac_levels[n], ctx, TYPE_I16_AC, &dqm->y1_, + dqm->lambda_trellis_i16_); it->top_nz_[x] = it->left_nz_[y] = non_zero; rd->y_ac_levels[n][0] = 0; nz |= non_zero << n; @@ -803,10 +821,10 @@ static int ReconstructIntra16(VP8EncIterator* const it, return nz; } -static int ReconstructIntra4(VP8EncIterator* const it, +static int ReconstructIntra4(VP8EncIterator* WEBP_RESTRICT const it, int16_t levels[16], - const uint8_t* const src, - uint8_t* const yuv_out, + const uint8_t* WEBP_RESTRICT const src, + uint8_t* WEBP_RESTRICT const yuv_out, int mode) { const VP8Encoder* const enc = it->enc_; const uint8_t* const ref = it->yuv_p_ + VP8I4ModeOffsets[mode]; @@ -818,7 +836,7 @@ static int ReconstructIntra4(VP8EncIterator* const it, if (DO_TRELLIS_I4 && it->do_trellis_) { const int x = it->i4_ & 3, y = it->i4_ >> 2; const int ctx = it->top_nz_[x] + it->left_nz_[y]; - nz = TrellisQuantizeBlock(enc, tmp, levels, ctx, 3, &dqm->y1_, + nz = TrellisQuantizeBlock(enc, tmp, levels, ctx, TYPE_I4_AC, &dqm->y1_, dqm->lambda_trellis_i4_); } else { nz = VP8EncQuantizeBlock(tmp, levels, &dqm->y1_); @@ -839,7 +857,8 @@ static int ReconstructIntra4(VP8EncIterator* const it, // Quantize as usual, but also compute and return the quantization error. // Error is already divided by DSHIFT. -static int QuantizeSingle(int16_t* const v, const VP8Matrix* const mtx) { +static int QuantizeSingle(int16_t* WEBP_RESTRICT const v, + const VP8Matrix* WEBP_RESTRICT const mtx) { int V = *v; const int sign = (V < 0); if (sign) V = -V; @@ -853,9 +872,10 @@ static int QuantizeSingle(int16_t* const v, const VP8Matrix* const mtx) { return (sign ? -V : V) >> DSCALE; } -static void CorrectDCValues(const VP8EncIterator* const it, - const VP8Matrix* const mtx, - int16_t tmp[][16], VP8ModeScore* const rd) { +static void CorrectDCValues(const VP8EncIterator* WEBP_RESTRICT const it, + const VP8Matrix* WEBP_RESTRICT const mtx, + int16_t tmp[][16], + VP8ModeScore* WEBP_RESTRICT const rd) { // | top[0] | top[1] // --------+--------+--------- // left[0] | tmp[0] tmp[1] <-> err0 err1 @@ -886,8 +906,8 @@ static void CorrectDCValues(const VP8EncIterator* const it, } } -static void StoreDiffusionErrors(VP8EncIterator* const it, - const VP8ModeScore* const rd) { +static void StoreDiffusionErrors(VP8EncIterator* WEBP_RESTRICT const it, + const VP8ModeScore* WEBP_RESTRICT const rd) { int ch; for (ch = 0; ch <= 1; ++ch) { int8_t* const top = it->top_derr_[it->x_][ch]; @@ -906,8 +926,9 @@ static void StoreDiffusionErrors(VP8EncIterator* const it, //------------------------------------------------------------------------------ -static int ReconstructUV(VP8EncIterator* const it, VP8ModeScore* const rd, - uint8_t* const yuv_out, int mode) { +static int ReconstructUV(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd, + uint8_t* WEBP_RESTRICT const yuv_out, int mode) { const VP8Encoder* const enc = it->enc_; const uint8_t* const ref = it->yuv_p_ + VP8UVModeOffsets[mode]; const uint8_t* const src = it->yuv_in_ + U_OFF_ENC; @@ -927,9 +948,9 @@ static int ReconstructUV(VP8EncIterator* const it, VP8ModeScore* const rd, for (y = 0; y < 2; ++y) { for (x = 0; x < 2; ++x, ++n) { const int ctx = it->top_nz_[4 + ch + x] + it->left_nz_[4 + ch + y]; - const int non_zero = - TrellisQuantizeBlock(enc, tmp[n], rd->uv_levels[n], ctx, 2, - &dqm->uv_, dqm->lambda_trellis_uv_); + const int non_zero = TrellisQuantizeBlock( + enc, tmp[n], rd->uv_levels[n], ctx, TYPE_CHROMA_A, &dqm->uv_, + dqm->lambda_trellis_uv_); it->top_nz_[4 + ch + x] = it->left_nz_[4 + ch + y] = non_zero; nz |= non_zero << n; } @@ -978,7 +999,8 @@ static void SwapOut(VP8EncIterator* const it) { SwapPtr(&it->yuv_out_, &it->yuv_out2_); } -static void PickBestIntra16(VP8EncIterator* const it, VP8ModeScore* rd) { +static void PickBestIntra16(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT rd) { const int kNumBlocks = 16; VP8SegmentInfo* const dqm = &it->enc_->dqm_[it->mb_->segment_]; const int lambda = dqm->lambda_i16_; @@ -1038,7 +1060,7 @@ static void PickBestIntra16(VP8EncIterator* const it, VP8ModeScore* rd) { //------------------------------------------------------------------------------ // return the cost array corresponding to the surrounding prediction modes. -static const uint16_t* GetCostModeI4(VP8EncIterator* const it, +static const uint16_t* GetCostModeI4(VP8EncIterator* WEBP_RESTRICT const it, const uint8_t modes[16]) { const int preds_w = it->enc_->preds_w_; const int x = (it->i4_ & 3), y = it->i4_ >> 2; @@ -1047,7 +1069,8 @@ static const uint16_t* GetCostModeI4(VP8EncIterator* const it, return VP8FixedCostsI4[top][left]; } -static int PickBestIntra4(VP8EncIterator* const it, VP8ModeScore* const rd) { +static int PickBestIntra4(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd) { const VP8Encoder* const enc = it->enc_; const VP8SegmentInfo* const dqm = &enc->dqm_[it->mb_->segment_]; const int lambda = dqm->lambda_i4_; @@ -1143,7 +1166,8 @@ static int PickBestIntra4(VP8EncIterator* const it, VP8ModeScore* const rd) { //------------------------------------------------------------------------------ -static void PickBestUV(VP8EncIterator* const it, VP8ModeScore* const rd) { +static void PickBestUV(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd) { const int kNumBlocks = 8; const VP8SegmentInfo* const dqm = &it->enc_->dqm_[it->mb_->segment_]; const int lambda = dqm->lambda_uv_; @@ -1195,7 +1219,8 @@ static void PickBestUV(VP8EncIterator* const it, VP8ModeScore* const rd) { //------------------------------------------------------------------------------ // Final reconstruction and quantization. -static void SimpleQuantize(VP8EncIterator* const it, VP8ModeScore* const rd) { +static void SimpleQuantize(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd) { const VP8Encoder* const enc = it->enc_; const int is_i16 = (it->mb_->type_ == 1); int nz = 0; @@ -1220,9 +1245,9 @@ static void SimpleQuantize(VP8EncIterator* const it, VP8ModeScore* const rd) { } // Refine intra16/intra4 sub-modes based on distortion only (not rate). -static void RefineUsingDistortion(VP8EncIterator* const it, +static void RefineUsingDistortion(VP8EncIterator* WEBP_RESTRICT const it, int try_both_modes, int refine_uv_mode, - VP8ModeScore* const rd) { + VP8ModeScore* WEBP_RESTRICT const rd) { score_t best_score = MAX_COST; int nz = 0; int mode; @@ -1336,7 +1361,8 @@ static void RefineUsingDistortion(VP8EncIterator* const it, //------------------------------------------------------------------------------ // Entry point -int VP8Decimate(VP8EncIterator* const it, VP8ModeScore* const rd, +int VP8Decimate(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd, VP8RDLevel rd_opt) { int is_skipped; const int method = it->enc_->method_; diff --git a/3rdparty/libwebp/src/enc/syntax_enc.c b/3rdparty/libwebp/src/enc/syntax_enc.c index a9e5a6cf0f..9b8f524d69 100644 --- a/3rdparty/libwebp/src/enc/syntax_enc.c +++ b/3rdparty/libwebp/src/enc/syntax_enc.c @@ -258,7 +258,10 @@ static int EmitPartitionsSize(const VP8Encoder* const enc, buf[3 * p + 1] = (part_size >> 8) & 0xff; buf[3 * p + 2] = (part_size >> 16) & 0xff; } - return p ? pic->writer(buf, 3 * p, pic) : 1; + if (p && !pic->writer(buf, 3 * p, pic)) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE); + } + return 1; } //------------------------------------------------------------------------------ @@ -349,7 +352,7 @@ int VP8EncWrite(VP8Encoder* const enc) { (enc->alpha_data_size_ & 1); riff_size += CHUNK_HEADER_SIZE + padded_alpha_size; } - // Sanity check. + // RIFF size should fit in 32-bits. if (riff_size > 0xfffffffeU) { return WebPEncodingSetError(pic, VP8_ENC_ERROR_FILE_TOO_BIG); } @@ -381,6 +384,7 @@ int VP8EncWrite(VP8Encoder* const enc) { enc->coded_size_ = (int)(CHUNK_HEADER_SIZE + riff_size); ok = ok && WebPReportProgress(pic, final_percent, &enc->percent_); + if (!ok) WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE); return ok; } diff --git a/3rdparty/libwebp/src/enc/vp8i_enc.h b/3rdparty/libwebp/src/enc/vp8i_enc.h index 0e35562a8c..19d9a6edb7 100644 --- a/3rdparty/libwebp/src/enc/vp8i_enc.h +++ b/3rdparty/libwebp/src/enc/vp8i_enc.h @@ -31,8 +31,8 @@ extern "C" { // version numbers #define ENC_MAJ_VERSION 1 -#define ENC_MIN_VERSION 2 -#define ENC_REV_VERSION 0 +#define ENC_MIN_VERSION 3 +#define ENC_REV_VERSION 1 enum { MAX_LF_LEVELS = 64, // Maximum loop filter level MAX_VARIABLE_LEVEL = 67, // last (inclusive) level with variable cost @@ -286,8 +286,7 @@ int VP8IteratorNext(VP8EncIterator* const it); // save the yuv_out_ boundary values to top_/left_ arrays for next iterations. void VP8IteratorSaveBoundary(VP8EncIterator* const it); // Report progression based on macroblock rows. Return 0 for user-abort request. -int VP8IteratorProgress(const VP8EncIterator* const it, - int final_delta_percent); +int VP8IteratorProgress(const VP8EncIterator* const it, int delta); // Intra4x4 iterations void VP8IteratorStartI4(VP8EncIterator* const it); // returns true if not done. @@ -471,7 +470,8 @@ int VP8EncAnalyze(VP8Encoder* const enc); // Sets up segment's quantization values, base_quant_ and filter strengths. void VP8SetSegmentParams(VP8Encoder* const enc, float quality); // Pick best modes and fills the levels. Returns true if skipped. -int VP8Decimate(VP8EncIterator* const it, VP8ModeScore* const rd, +int VP8Decimate(VP8EncIterator* WEBP_RESTRICT const it, + VP8ModeScore* WEBP_RESTRICT const rd, VP8RDLevel rd_opt); // in alpha.c @@ -491,19 +491,24 @@ int VP8FilterStrengthFromDelta(int sharpness, int delta); // misc utils for picture_*.c: +// Returns true if 'picture' is non-NULL and dimensions/colorspace are within +// their valid ranges. If returning false, the 'error_code' in 'picture' is +// updated. +int WebPValidatePicture(const WebPPicture* const picture); + // Remove reference to the ARGB/YUVA buffer (doesn't free anything). void WebPPictureResetBuffers(WebPPicture* const picture); -// Allocates ARGB buffer of given dimension (previous one is always free'd). -// Preserves the YUV(A) buffer. Returns false in case of error (invalid param, -// out-of-memory). -int WebPPictureAllocARGB(WebPPicture* const picture, int width, int height); +// Allocates ARGB buffer according to set width/height (previous one is +// always free'd). Preserves the YUV(A) buffer. Returns false in case of error +// (invalid param, out-of-memory). +int WebPPictureAllocARGB(WebPPicture* const picture); -// Allocates YUVA buffer of given dimension (previous one is always free'd). -// Uses picture->csp to determine whether an alpha buffer is needed. +// Allocates YUVA buffer according to set width/height (previous one is always +// free'd). Uses picture->csp to determine whether an alpha buffer is needed. // Preserves the ARGB buffer. // Returns false in case of error (invalid param, out-of-memory). -int WebPPictureAllocYUVA(WebPPicture* const picture, int width, int height); +int WebPPictureAllocYUVA(WebPPicture* const picture); // Replace samples that are fully transparent by 'color' to help compressibility // (no guarantee, though). Assumes pic->use_argb is true. diff --git a/3rdparty/libwebp/src/enc/vp8l_enc.c b/3rdparty/libwebp/src/enc/vp8l_enc.c index 0b44ebe46e..c43d990d17 100644 --- a/3rdparty/libwebp/src/enc/vp8l_enc.c +++ b/3rdparty/libwebp/src/enc/vp8l_enc.c @@ -15,128 +15,25 @@ #include #include +#include "src/dsp/lossless.h" +#include "src/dsp/lossless_common.h" #include "src/enc/backward_references_enc.h" #include "src/enc/histogram_enc.h" #include "src/enc/vp8i_enc.h" #include "src/enc/vp8li_enc.h" -#include "src/dsp/lossless.h" -#include "src/dsp/lossless_common.h" #include "src/utils/bit_writer_utils.h" #include "src/utils/huffman_encode_utils.h" +#include "src/utils/palette.h" #include "src/utils/utils.h" +#include "src/webp/encode.h" #include "src/webp/format_constants.h" // Maximum number of histogram images (sub-blocks). #define MAX_HUFF_IMAGE_SIZE 2600 -// Palette reordering for smaller sum of deltas (and for smaller storage). - -static int PaletteCompareColorsForQsort(const void* p1, const void* p2) { - const uint32_t a = WebPMemToUint32((uint8_t*)p1); - const uint32_t b = WebPMemToUint32((uint8_t*)p2); - assert(a != b); - return (a < b) ? -1 : 1; -} - -static WEBP_INLINE uint32_t PaletteComponentDistance(uint32_t v) { - return (v <= 128) ? v : (256 - v); -} - -// Computes a value that is related to the entropy created by the -// palette entry diff. -// -// Note that the last & 0xff is a no-operation in the next statement, but -// removed by most compilers and is here only for regularity of the code. -static WEBP_INLINE uint32_t PaletteColorDistance(uint32_t col1, uint32_t col2) { - const uint32_t diff = VP8LSubPixels(col1, col2); - const int kMoreWeightForRGBThanForAlpha = 9; - uint32_t score; - score = PaletteComponentDistance((diff >> 0) & 0xff); - score += PaletteComponentDistance((diff >> 8) & 0xff); - score += PaletteComponentDistance((diff >> 16) & 0xff); - score *= kMoreWeightForRGBThanForAlpha; - score += PaletteComponentDistance((diff >> 24) & 0xff); - return score; -} - -static WEBP_INLINE void SwapColor(uint32_t* const col1, uint32_t* const col2) { - const uint32_t tmp = *col1; - *col1 = *col2; - *col2 = tmp; -} - -static void GreedyMinimizeDeltas(uint32_t palette[], int num_colors) { - // Find greedily always the closest color of the predicted color to minimize - // deltas in the palette. This reduces storage needs since the - // palette is stored with delta encoding. - uint32_t predict = 0x00000000; - int i, k; - for (i = 0; i < num_colors; ++i) { - int best_ix = i; - uint32_t best_score = ~0U; - for (k = i; k < num_colors; ++k) { - const uint32_t cur_score = PaletteColorDistance(palette[k], predict); - if (best_score > cur_score) { - best_score = cur_score; - best_ix = k; - } - } - SwapColor(&palette[best_ix], &palette[i]); - predict = palette[i]; - } -} - -// The palette has been sorted by alpha. This function checks if the other -// components of the palette have a monotonic development with regards to -// position in the palette. If all have monotonic development, there is -// no benefit to re-organize them greedily. A monotonic development -// would be spotted in green-only situations (like lossy alpha) or gray-scale -// images. -static int PaletteHasNonMonotonousDeltas(uint32_t palette[], int num_colors) { - uint32_t predict = 0x000000; - int i; - uint8_t sign_found = 0x00; - for (i = 0; i < num_colors; ++i) { - const uint32_t diff = VP8LSubPixels(palette[i], predict); - const uint8_t rd = (diff >> 16) & 0xff; - const uint8_t gd = (diff >> 8) & 0xff; - const uint8_t bd = (diff >> 0) & 0xff; - if (rd != 0x00) { - sign_found |= (rd < 0x80) ? 1 : 2; - } - if (gd != 0x00) { - sign_found |= (gd < 0x80) ? 8 : 16; - } - if (bd != 0x00) { - sign_found |= (bd < 0x80) ? 64 : 128; - } - predict = palette[i]; - } - return (sign_found & (sign_found << 1)) != 0; // two consequent signs. -} - // ----------------------------------------------------------------------------- // Palette -// If number of colors in the image is less than or equal to MAX_PALETTE_SIZE, -// creates a palette and returns true, else returns false. -static int AnalyzeAndCreatePalette(const WebPPicture* const pic, - int low_effort, - uint32_t palette[MAX_PALETTE_SIZE], - int* const palette_size) { - const int num_colors = WebPGetColorPalette(pic, palette); - if (num_colors > MAX_PALETTE_SIZE) { - *palette_size = 0; - return 0; - } - *palette_size = num_colors; - qsort(palette, num_colors, sizeof(*palette), PaletteCompareColorsForQsort); - if (!low_effort && PaletteHasNonMonotonousDeltas(palette, num_colors)) { - GreedyMinimizeDeltas(palette, num_colors); - } - return 1; -} - // These five modes are evaluated and their respective entropy is computed. typedef enum { kDirect = 0, @@ -165,10 +62,11 @@ typedef enum { kHistoTotal // Must be last. } HistoIx; -static void AddSingleSubGreen(int p, uint32_t* const r, uint32_t* const b) { - const int green = p >> 8; // The upper bits are masked away later. - ++r[((p >> 16) - green) & 0xff]; - ++b[((p >> 0) - green) & 0xff]; +static void AddSingleSubGreen(uint32_t p, + uint32_t* const r, uint32_t* const b) { + const int green = (int)p >> 8; // The upper bits are masked away later. + ++r[(((int)p >> 16) - green) & 0xff]; + ++b[(((int)p >> 0) - green) & 0xff]; } static void AddSingle(uint32_t p, @@ -242,8 +140,8 @@ static int AnalyzeEntropy(const uint32_t* argb, curr_row += argb_stride; } { - double entropy_comp[kHistoTotal]; - double entropy[kNumEntropyIx]; + float entropy_comp[kHistoTotal]; + float entropy[kNumEntropyIx]; int k; int last_mode_to_analyze = use_palette ? kPalette : kSpatialSubGreen; int j; @@ -362,11 +260,14 @@ typedef struct { } CrunchSubConfig; typedef struct { int entropy_idx_; + PaletteSorting palette_sorting_type_; CrunchSubConfig sub_configs_[CRUNCH_SUBCONFIGS_MAX]; int sub_configs_size_; } CrunchConfig; -#define CRUNCH_CONFIGS_MAX kNumEntropyIx +// +2 because we add a palette sorting configuration for kPalette and +// kPaletteAndSpatial. +#define CRUNCH_CONFIGS_MAX (kNumEntropyIx + 2 * kPaletteSortingNum) static int EncoderAnalyze(VP8LEncoder* const enc, CrunchConfig crunch_configs[CRUNCH_CONFIGS_MAX], @@ -386,9 +287,12 @@ static int EncoderAnalyze(VP8LEncoder* const enc, int do_no_cache = 0; assert(pic != NULL && pic->argb != NULL); - use_palette = - AnalyzeAndCreatePalette(pic, low_effort, - enc->palette_, &enc->palette_size_); + // Check whether a palette is possible. + enc->palette_size_ = GetColorPalette(pic, enc->palette_sorted_); + use_palette = (enc->palette_size_ <= MAX_PALETTE_SIZE); + if (!use_palette) { + enc->palette_size_ = 0; + } // Empirical bit sizes. enc->histo_bits_ = GetHistoBits(method, use_palette, @@ -398,6 +302,8 @@ static int EncoderAnalyze(VP8LEncoder* const enc, if (low_effort) { // AnalyzeEntropy is somewhat slow. crunch_configs[0].entropy_idx_ = use_palette ? kPalette : kSpatialSubGreen; + crunch_configs[0].palette_sorting_type_ = + use_palette ? kSortedDefault : kUnusedPalette; n_lz77s = 1; *crunch_configs_size = 1; } else { @@ -418,13 +324,37 @@ static int EncoderAnalyze(VP8LEncoder* const enc, // a palette. if ((i != kPalette && i != kPaletteAndSpatial) || use_palette) { assert(*crunch_configs_size < CRUNCH_CONFIGS_MAX); - crunch_configs[(*crunch_configs_size)++].entropy_idx_ = i; + if (use_palette && (i == kPalette || i == kPaletteAndSpatial)) { + int sorting_method; + for (sorting_method = 0; sorting_method < kPaletteSortingNum; + ++sorting_method) { + const PaletteSorting typed_sorting_method = + (PaletteSorting)sorting_method; + // TODO(vrabaud) kSortedDefault should be tested. It is omitted + // for now for backward compatibility. + if (typed_sorting_method == kUnusedPalette || + typed_sorting_method == kSortedDefault) { + continue; + } + crunch_configs[(*crunch_configs_size)].entropy_idx_ = i; + crunch_configs[(*crunch_configs_size)].palette_sorting_type_ = + typed_sorting_method; + ++*crunch_configs_size; + } + } else { + crunch_configs[(*crunch_configs_size)].entropy_idx_ = i; + crunch_configs[(*crunch_configs_size)].palette_sorting_type_ = + kUnusedPalette; + ++*crunch_configs_size; + } } } } else { // Only choose the guessed best transform. *crunch_configs_size = 1; crunch_configs[0].entropy_idx_ = min_entropy_ix; + crunch_configs[0].palette_sorting_type_ = + use_palette ? kMinimizeDelta : kUnusedPalette; if (config->quality >= 75 && method == 5) { // Test with and without color cache. do_no_cache = 1; @@ -432,6 +362,7 @@ static int EncoderAnalyze(VP8LEncoder* const enc, if (min_entropy_ix == kPalette) { *crunch_configs_size = 2; crunch_configs[1].entropy_idx_ = kPaletteAndSpatial; + crunch_configs[1].palette_sorting_type_ = kMinimizeDelta; } } } @@ -730,11 +661,11 @@ static WEBP_INLINE void WriteHuffmanCodeWithExtraBits( VP8LPutBits(bw, (bits << depth) | symbol, depth + n_bits); } -static WebPEncodingError StoreImageToBitMask( +static int StoreImageToBitMask( VP8LBitWriter* const bw, int width, int histo_bits, const VP8LBackwardRefs* const refs, const uint16_t* histogram_symbols, - const HuffmanTreeCode* const huffman_codes) { + const HuffmanTreeCode* const huffman_codes, const WebPPicture* const pic) { const int histo_xsize = histo_bits ? VP8LSubSampleSize(width, histo_bits) : 1; const int tile_mask = (histo_bits == 0) ? 0 : -(1 << histo_bits); // x and y trace the position in the image. @@ -787,44 +718,52 @@ static WebPEncodingError StoreImageToBitMask( } VP8LRefsCursorNext(&c); } - return bw->error_ ? VP8_ENC_ERROR_OUT_OF_MEMORY : VP8_ENC_OK; + if (bw->error_) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } + return 1; } -// Special case of EncodeImageInternal() for cache-bits=0, histo_bits=31 -static WebPEncodingError EncodeImageNoHuffman( - VP8LBitWriter* const bw, const uint32_t* const argb, - VP8LHashChain* const hash_chain, VP8LBackwardRefs* const refs_array, - int width, int height, int quality, int low_effort) { +// Special case of EncodeImageInternal() for cache-bits=0, histo_bits=31. +// pic and percent are for progress. +static int EncodeImageNoHuffman(VP8LBitWriter* const bw, + const uint32_t* const argb, + VP8LHashChain* const hash_chain, + VP8LBackwardRefs* const refs_array, int width, + int height, int quality, int low_effort, + const WebPPicture* const pic, int percent_range, + int* const percent) { int i; int max_tokens = 0; - WebPEncodingError err = VP8_ENC_OK; VP8LBackwardRefs* refs; HuffmanTreeToken* tokens = NULL; - HuffmanTreeCode huffman_codes[5] = { { 0, NULL, NULL } }; - const uint16_t histogram_symbols[1] = { 0 }; // only one tree, one symbol + HuffmanTreeCode huffman_codes[5] = {{0, NULL, NULL}}; + const uint16_t histogram_symbols[1] = {0}; // only one tree, one symbol int cache_bits = 0; VP8LHistogramSet* histogram_image = NULL; HuffmanTree* const huff_tree = (HuffmanTree*)WebPSafeMalloc( - 3ULL * CODE_LENGTH_CODES, sizeof(*huff_tree)); + 3ULL * CODE_LENGTH_CODES, sizeof(*huff_tree)); if (huff_tree == NULL) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } // Calculate backward references from ARGB image. - if (!VP8LHashChainFill(hash_chain, quality, argb, width, height, - low_effort)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + if (!VP8LHashChainFill(hash_chain, quality, argb, width, height, low_effort, + pic, percent_range / 2, percent)) { + goto Error; + } + if (!VP8LGetBackwardReferences(width, height, argb, quality, /*low_effort=*/0, + kLZ77Standard | kLZ77RLE, cache_bits, + /*do_no_cache=*/0, hash_chain, refs_array, + &cache_bits, pic, + percent_range - percent_range / 2, percent)) { goto Error; } - err = VP8LGetBackwardReferences( - width, height, argb, quality, /*low_effort=*/0, kLZ77Standard | kLZ77RLE, - cache_bits, /*do_no_cache=*/0, hash_chain, refs_array, &cache_bits); - if (err != VP8_ENC_OK) goto Error; refs = &refs_array[0]; histogram_image = VP8LAllocateHistogramSet(1, cache_bits); if (histogram_image == NULL) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } VP8LHistogramSetClear(histogram_image); @@ -835,7 +774,7 @@ static WebPEncodingError EncodeImageNoHuffman( // Create Huffman bit lengths and codes for each histogram image. assert(histogram_image->size == 1); if (!GetHuffBitLengthsAndCodes(histogram_image, huffman_codes)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } @@ -852,7 +791,7 @@ static WebPEncodingError EncodeImageNoHuffman( tokens = (HuffmanTreeToken*)WebPSafeMalloc(max_tokens, sizeof(*tokens)); if (tokens == NULL) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } @@ -864,27 +803,32 @@ static WebPEncodingError EncodeImageNoHuffman( } // Store actual literals. - err = StoreImageToBitMask(bw, width, 0, refs, histogram_symbols, - huffman_codes); + if (!StoreImageToBitMask(bw, width, 0, refs, histogram_symbols, huffman_codes, + pic)) { + goto Error; + } Error: WebPSafeFree(tokens); WebPSafeFree(huff_tree); VP8LFreeHistogramSet(histogram_image); WebPSafeFree(huffman_codes[0].codes); - return err; + return (pic->error_code == VP8_ENC_OK); } -static WebPEncodingError EncodeImageInternal( +// pic and percent are for progress. +static int EncodeImageInternal( VP8LBitWriter* const bw, const uint32_t* const argb, VP8LHashChain* const hash_chain, VP8LBackwardRefs refs_array[4], int width, int height, int quality, int low_effort, int use_cache, const CrunchConfig* const config, int* cache_bits, int histogram_bits, - size_t init_byte_position, int* const hdr_size, int* const data_size) { - WebPEncodingError err = VP8_ENC_ERROR_OUT_OF_MEMORY; + size_t init_byte_position, int* const hdr_size, int* const data_size, + const WebPPicture* const pic, int percent_range, int* const percent) { const uint32_t histogram_image_xysize = VP8LSubSampleSize(width, histogram_bits) * VP8LSubSampleSize(height, histogram_bits); + int remaining_percent = percent_range; + int percent_start = *percent; VP8LHistogramSet* histogram_image = NULL; VP8LHistogram* tmp_histo = NULL; int histogram_image_size = 0; @@ -893,9 +837,8 @@ static WebPEncodingError EncodeImageInternal( 3ULL * CODE_LENGTH_CODES, sizeof(*huff_tree)); HuffmanTreeToken* tokens = NULL; HuffmanTreeCode* huffman_codes = NULL; - uint16_t* const histogram_symbols = - (uint16_t*)WebPSafeMalloc(histogram_image_xysize, - sizeof(*histogram_symbols)); + uint16_t* const histogram_symbols = (uint16_t*)WebPSafeMalloc( + histogram_image_xysize, sizeof(*histogram_symbols)); int sub_configs_idx; int cache_bits_init, write_histogram_image; VP8LBitWriter bw_init = *bw, bw_best; @@ -907,14 +850,27 @@ static WebPEncodingError EncodeImageInternal( assert(hdr_size != NULL); assert(data_size != NULL); - // Make sure we can allocate the different objects. memset(&hash_chain_histogram, 0, sizeof(hash_chain_histogram)); - if (huff_tree == NULL || histogram_symbols == NULL || - !VP8LHashChainInit(&hash_chain_histogram, histogram_image_xysize) || - !VP8LHashChainFill(hash_chain, quality, argb, width, height, - low_effort)) { + if (!VP8LBitWriterInit(&bw_best, 0)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } + + // Make sure we can allocate the different objects. + if (huff_tree == NULL || histogram_symbols == NULL || + !VP8LHashChainInit(&hash_chain_histogram, histogram_image_xysize)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } + + percent_range = remaining_percent / 5; + if (!VP8LHashChainFill(hash_chain, quality, argb, width, height, + low_effort, pic, percent_range, percent)) { + goto Error; + } + percent_start += percent_range; + remaining_percent -= percent_range; + if (use_cache) { // If the value is different from zero, it has been set during the // palette analysis. @@ -923,22 +879,27 @@ static WebPEncodingError EncodeImageInternal( cache_bits_init = 0; } // If several iterations will happen, clone into bw_best. - if (!VP8LBitWriterInit(&bw_best, 0) || - ((config->sub_configs_size_ > 1 || - config->sub_configs_[0].do_no_cache_) && - !VP8LBitWriterClone(bw, &bw_best))) { + if ((config->sub_configs_size_ > 1 || config->sub_configs_[0].do_no_cache_) && + !VP8LBitWriterClone(bw, &bw_best)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } + for (sub_configs_idx = 0; sub_configs_idx < config->sub_configs_size_; ++sub_configs_idx) { const CrunchSubConfig* const sub_config = &config->sub_configs_[sub_configs_idx]; int cache_bits_best, i_cache; - err = VP8LGetBackwardReferences(width, height, argb, quality, low_effort, - sub_config->lz77_, cache_bits_init, - sub_config->do_no_cache_, hash_chain, - &refs_array[0], &cache_bits_best); - if (err != VP8_ENC_OK) goto Error; + int i_remaining_percent = remaining_percent / config->sub_configs_size_; + int i_percent_range = i_remaining_percent / 4; + i_remaining_percent -= i_percent_range; + + if (!VP8LGetBackwardReferences( + width, height, argb, quality, low_effort, sub_config->lz77_, + cache_bits_init, sub_config->do_no_cache_, hash_chain, + &refs_array[0], &cache_bits_best, pic, i_percent_range, percent)) { + goto Error; + } for (i_cache = 0; i_cache < (sub_config->do_no_cache_ ? 2 : 1); ++i_cache) { const int cache_bits_tmp = (i_cache == 0) ? cache_bits_best : 0; @@ -953,11 +914,17 @@ static WebPEncodingError EncodeImageInternal( histogram_image = VP8LAllocateHistogramSet(histogram_image_xysize, cache_bits_tmp); tmp_histo = VP8LAllocateHistogram(cache_bits_tmp); - if (histogram_image == NULL || tmp_histo == NULL || - !VP8LGetHistoImageSymbols(width, height, &refs_array[i_cache], - quality, low_effort, histogram_bits, - cache_bits_tmp, histogram_image, tmp_histo, - histogram_symbols)) { + if (histogram_image == NULL || tmp_histo == NULL) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } + + i_percent_range = i_remaining_percent / 3; + i_remaining_percent -= i_percent_range; + if (!VP8LGetHistoImageSymbols( + width, height, &refs_array[i_cache], quality, low_effort, + histogram_bits, cache_bits_tmp, histogram_image, tmp_histo, + histogram_symbols, pic, i_percent_range, percent)) { goto Error; } // Create Huffman bit lengths and codes for each histogram image. @@ -970,6 +937,7 @@ static WebPEncodingError EncodeImageInternal( // GetHuffBitLengthsAndCodes(). if (huffman_codes == NULL || !GetHuffBitLengthsAndCodes(histogram_image, huffman_codes)) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } // Free combined histograms. @@ -992,12 +960,14 @@ static WebPEncodingError EncodeImageInternal( write_histogram_image = (histogram_image_size > 1); VP8LPutBits(bw, write_histogram_image, 1); if (write_histogram_image) { - uint32_t* const histogram_argb = - (uint32_t*)WebPSafeMalloc(histogram_image_xysize, - sizeof(*histogram_argb)); + uint32_t* const histogram_argb = (uint32_t*)WebPSafeMalloc( + histogram_image_xysize, sizeof(*histogram_argb)); int max_index = 0; uint32_t i; - if (histogram_argb == NULL) goto Error; + if (histogram_argb == NULL) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } for (i = 0; i < histogram_image_xysize; ++i) { const int symbol_index = histogram_symbols[i] & 0xffff; histogram_argb[i] = (symbol_index << 8); @@ -1008,12 +978,17 @@ static WebPEncodingError EncodeImageInternal( histogram_image_size = max_index; VP8LPutBits(bw, histogram_bits - 2, 3); - err = EncodeImageNoHuffman( - bw, histogram_argb, &hash_chain_histogram, &refs_array[2], - VP8LSubSampleSize(width, histogram_bits), - VP8LSubSampleSize(height, histogram_bits), quality, low_effort); + i_percent_range = i_remaining_percent / 2; + i_remaining_percent -= i_percent_range; + if (!EncodeImageNoHuffman( + bw, histogram_argb, &hash_chain_histogram, &refs_array[2], + VP8LSubSampleSize(width, histogram_bits), + VP8LSubSampleSize(height, histogram_bits), quality, low_effort, + pic, i_percent_range, percent)) { + WebPSafeFree(histogram_argb); + goto Error; + } WebPSafeFree(histogram_argb); - if (err != VP8_ENC_OK) goto Error; } // Store Huffman codes. @@ -1028,7 +1003,10 @@ static WebPEncodingError EncodeImageInternal( } } tokens = (HuffmanTreeToken*)WebPSafeMalloc(max_tokens, sizeof(*tokens)); - if (tokens == NULL) goto Error; + if (tokens == NULL) { + WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } for (i = 0; i < 5 * histogram_image_size; ++i) { HuffmanTreeCode* const codes = &huffman_codes[i]; StoreHuffmanCode(bw, huff_tree, tokens, codes); @@ -1037,9 +1015,10 @@ static WebPEncodingError EncodeImageInternal( } // Store actual literals. hdr_size_tmp = (int)(VP8LBitWriterNumBytes(bw) - init_byte_position); - err = StoreImageToBitMask(bw, width, histogram_bits, &refs_array[i_cache], - histogram_symbols, huffman_codes); - if (err != VP8_ENC_OK) goto Error; + if (!StoreImageToBitMask(bw, width, histogram_bits, &refs_array[i_cache], + histogram_symbols, huffman_codes, pic)) { + goto Error; + } // Keep track of the smallest image so far. if (VP8LBitWriterNumBytes(bw) < bw_size_best) { bw_size_best = VP8LBitWriterNumBytes(bw); @@ -1059,7 +1038,10 @@ static WebPEncodingError EncodeImageInternal( } } VP8LBitWriterSwap(bw, &bw_best); - err = VP8_ENC_OK; + + if (!WebPReportProgress(pic, percent_start + remaining_percent, percent)) { + goto Error; + } Error: WebPSafeFree(tokens); @@ -1073,7 +1055,7 @@ static WebPEncodingError EncodeImageInternal( } WebPSafeFree(histogram_symbols); VP8LBitWriterWipeOut(&bw_best); - return err; + return (pic->error_code == VP8_ENC_OK); } // ----------------------------------------------------------------------------- @@ -1082,26 +1064,27 @@ static WebPEncodingError EncodeImageInternal( static void ApplySubtractGreen(VP8LEncoder* const enc, int width, int height, VP8LBitWriter* const bw) { VP8LPutBits(bw, TRANSFORM_PRESENT, 1); - VP8LPutBits(bw, SUBTRACT_GREEN, 2); + VP8LPutBits(bw, SUBTRACT_GREEN_TRANSFORM, 2); VP8LSubtractGreenFromBlueAndRed(enc->argb_, width * height); } -static WebPEncodingError ApplyPredictFilter(const VP8LEncoder* const enc, - int width, int height, - int quality, int low_effort, - int used_subtract_green, - VP8LBitWriter* const bw) { +static int ApplyPredictFilter(const VP8LEncoder* const enc, int width, + int height, int quality, int low_effort, + int used_subtract_green, VP8LBitWriter* const bw, + int percent_range, int* const percent) { const int pred_bits = enc->transform_bits_; const int transform_width = VP8LSubSampleSize(width, pred_bits); const int transform_height = VP8LSubSampleSize(height, pred_bits); // we disable near-lossless quantization if palette is used. - const int near_lossless_strength = enc->use_palette_ ? 100 - : enc->config_->near_lossless; + const int near_lossless_strength = + enc->use_palette_ ? 100 : enc->config_->near_lossless; - VP8LResidualImage(width, height, pred_bits, low_effort, enc->argb_, - enc->argb_scratch_, enc->transform_data_, - near_lossless_strength, enc->config_->exact, - used_subtract_green); + if (!VP8LResidualImage( + width, height, pred_bits, low_effort, enc->argb_, enc->argb_scratch_, + enc->transform_data_, near_lossless_strength, enc->config_->exact, + used_subtract_green, enc->pic_, percent_range / 2, percent)) { + return 0; + } VP8LPutBits(bw, TRANSFORM_PRESENT, 1); VP8LPutBits(bw, PREDICTOR_TRANSFORM, 2); assert(pred_bits >= 2); @@ -1109,19 +1092,23 @@ static WebPEncodingError ApplyPredictFilter(const VP8LEncoder* const enc, return EncodeImageNoHuffman( bw, enc->transform_data_, (VP8LHashChain*)&enc->hash_chain_, (VP8LBackwardRefs*)&enc->refs_[0], transform_width, transform_height, - quality, low_effort); + quality, low_effort, enc->pic_, percent_range - percent_range / 2, + percent); } -static WebPEncodingError ApplyCrossColorFilter(const VP8LEncoder* const enc, - int width, int height, - int quality, int low_effort, - VP8LBitWriter* const bw) { +static int ApplyCrossColorFilter(const VP8LEncoder* const enc, int width, + int height, int quality, int low_effort, + VP8LBitWriter* const bw, int percent_range, + int* const percent) { const int ccolor_transform_bits = enc->transform_bits_; const int transform_width = VP8LSubSampleSize(width, ccolor_transform_bits); const int transform_height = VP8LSubSampleSize(height, ccolor_transform_bits); - VP8LColorSpaceTransform(width, height, ccolor_transform_bits, quality, - enc->argb_, enc->transform_data_); + if (!VP8LColorSpaceTransform(width, height, ccolor_transform_bits, quality, + enc->argb_, enc->transform_data_, enc->pic_, + percent_range / 2, percent)) { + return 0; + } VP8LPutBits(bw, TRANSFORM_PRESENT, 1); VP8LPutBits(bw, CROSS_COLOR_TRANSFORM, 2); assert(ccolor_transform_bits >= 2); @@ -1129,23 +1116,21 @@ static WebPEncodingError ApplyCrossColorFilter(const VP8LEncoder* const enc, return EncodeImageNoHuffman( bw, enc->transform_data_, (VP8LHashChain*)&enc->hash_chain_, (VP8LBackwardRefs*)&enc->refs_[0], transform_width, transform_height, - quality, low_effort); + quality, low_effort, enc->pic_, percent_range - percent_range / 2, + percent); } // ----------------------------------------------------------------------------- -static WebPEncodingError WriteRiffHeader(const WebPPicture* const pic, - size_t riff_size, size_t vp8l_size) { +static int WriteRiffHeader(const WebPPicture* const pic, size_t riff_size, + size_t vp8l_size) { uint8_t riff[RIFF_HEADER_SIZE + CHUNK_HEADER_SIZE + VP8L_SIGNATURE_SIZE] = { 'R', 'I', 'F', 'F', 0, 0, 0, 0, 'W', 'E', 'B', 'P', 'V', 'P', '8', 'L', 0, 0, 0, 0, VP8L_MAGIC_BYTE, }; PutLE32(riff + TAG_SIZE, (uint32_t)riff_size); PutLE32(riff + RIFF_HEADER_SIZE + TAG_SIZE, (uint32_t)vp8l_size); - if (!pic->writer(riff, sizeof(riff), pic)) { - return VP8_ENC_ERROR_BAD_WRITE; - } - return VP8_ENC_OK; + return pic->writer(riff, sizeof(riff), pic); } static int WriteImageSize(const WebPPicture* const pic, @@ -1165,36 +1150,32 @@ static int WriteRealAlphaAndVersion(VP8LBitWriter* const bw, int has_alpha) { return !bw->error_; } -static WebPEncodingError WriteImage(const WebPPicture* const pic, - VP8LBitWriter* const bw, - size_t* const coded_size) { - WebPEncodingError err = VP8_ENC_OK; +static int WriteImage(const WebPPicture* const pic, VP8LBitWriter* const bw, + size_t* const coded_size) { const uint8_t* const webpll_data = VP8LBitWriterFinish(bw); const size_t webpll_size = VP8LBitWriterNumBytes(bw); const size_t vp8l_size = VP8L_SIGNATURE_SIZE + webpll_size; const size_t pad = vp8l_size & 1; const size_t riff_size = TAG_SIZE + CHUNK_HEADER_SIZE + vp8l_size + pad; + *coded_size = 0; - err = WriteRiffHeader(pic, riff_size, vp8l_size); - if (err != VP8_ENC_OK) goto Error; + if (bw->error_) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } - if (!pic->writer(webpll_data, webpll_size, pic)) { - err = VP8_ENC_ERROR_BAD_WRITE; - goto Error; + if (!WriteRiffHeader(pic, riff_size, vp8l_size) || + !pic->writer(webpll_data, webpll_size, pic)) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE); } if (pad) { const uint8_t pad_byte[1] = { 0 }; if (!pic->writer(pad_byte, 1, pic)) { - err = VP8_ENC_ERROR_BAD_WRITE; - goto Error; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_WRITE); } } *coded_size = CHUNK_HEADER_SIZE + riff_size; - return VP8_ENC_OK; - - Error: - return err; + return 1; } // ----------------------------------------------------------------------------- @@ -1210,36 +1191,32 @@ static void ClearTransformBuffer(VP8LEncoder* const enc) { // Flags influencing the memory allocated: // enc->transform_bits_ // enc->use_predict_, enc->use_cross_color_ -static WebPEncodingError AllocateTransformBuffer(VP8LEncoder* const enc, - int width, int height) { - WebPEncodingError err = VP8_ENC_OK; - const uint64_t image_size = width * height; +static int AllocateTransformBuffer(VP8LEncoder* const enc, int width, + int height) { + const uint64_t image_size = (uint64_t)width * height; // VP8LResidualImage needs room for 2 scanlines of uint32 pixels with an extra // pixel in each, plus 2 regular scanlines of bytes. // TODO(skal): Clean up by using arithmetic in bytes instead of words. const uint64_t argb_scratch_size = - enc->use_predict_ - ? (width + 1) * 2 + - (width * 2 + sizeof(uint32_t) - 1) / sizeof(uint32_t) - : 0; + enc->use_predict_ ? (width + 1) * 2 + (width * 2 + sizeof(uint32_t) - 1) / + sizeof(uint32_t) + : 0; const uint64_t transform_data_size = (enc->use_predict_ || enc->use_cross_color_) - ? VP8LSubSampleSize(width, enc->transform_bits_) * + ? (uint64_t)VP8LSubSampleSize(width, enc->transform_bits_) * VP8LSubSampleSize(height, enc->transform_bits_) : 0; const uint64_t max_alignment_in_words = (WEBP_ALIGN_CST + sizeof(uint32_t) - 1) / sizeof(uint32_t); - const uint64_t mem_size = - image_size + max_alignment_in_words + - argb_scratch_size + max_alignment_in_words + - transform_data_size; + const uint64_t mem_size = image_size + max_alignment_in_words + + argb_scratch_size + max_alignment_in_words + + transform_data_size; uint32_t* mem = enc->transform_mem_; if (mem == NULL || mem_size > enc->transform_mem_size_) { ClearTransformBuffer(enc); mem = (uint32_t*)WebPSafeMalloc(mem_size, sizeof(*mem)); if (mem == NULL) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; - goto Error; + return WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); } enc->transform_mem_ = mem; enc->transform_mem_size_ = (size_t)mem_size; @@ -1252,19 +1229,16 @@ static WebPEncodingError AllocateTransformBuffer(VP8LEncoder* const enc, enc->transform_data_ = mem; enc->current_width_ = width; - Error: - return err; + return 1; } -static WebPEncodingError MakeInputImageCopy(VP8LEncoder* const enc) { - WebPEncodingError err = VP8_ENC_OK; +static int MakeInputImageCopy(VP8LEncoder* const enc) { const WebPPicture* const picture = enc->pic_; const int width = picture->width; const int height = picture->height; - err = AllocateTransformBuffer(enc, width, height); - if (err != VP8_ENC_OK) return err; - if (enc->argb_content_ == kEncoderARGB) return VP8_ENC_OK; + if (!AllocateTransformBuffer(enc, width, height)) return 0; + if (enc->argb_content_ == kEncoderARGB) return 1; { uint32_t* dst = enc->argb_; @@ -1278,27 +1252,11 @@ static WebPEncodingError MakeInputImageCopy(VP8LEncoder* const enc) { } enc->argb_content_ = kEncoderARGB; assert(enc->current_width_ == width); - return VP8_ENC_OK; + return 1; } // ----------------------------------------------------------------------------- -static WEBP_INLINE int SearchColorNoIdx(const uint32_t sorted[], uint32_t color, - int hi) { - int low = 0; - if (sorted[low] == color) return low; // loop invariant: sorted[low] != color - while (1) { - const int mid = (low + hi) >> 1; - if (sorted[mid] == color) { - return mid; - } else if (sorted[mid] < color) { - low = mid; - } else { - hi = mid; - } - } -} - #define APPLY_PALETTE_GREEDY_MAX 4 static WEBP_INLINE uint32_t SearchColorGreedy(const uint32_t palette[], @@ -1333,17 +1291,6 @@ static WEBP_INLINE uint32_t ApplyPaletteHash2(uint32_t color) { (32 - PALETTE_INV_SIZE_BITS); } -// Sort palette in increasing order and prepare an inverse mapping array. -static void PrepareMapToPalette(const uint32_t palette[], int num_colors, - uint32_t sorted[], uint32_t idx_map[]) { - int i; - memcpy(sorted, palette, num_colors * sizeof(*sorted)); - qsort(sorted, num_colors, sizeof(*sorted), PaletteCompareColorsForQsort); - for (i = 0; i < num_colors; ++i) { - idx_map[SearchColorNoIdx(sorted, palette[i], num_colors)] = i; - } -} - // Use 1 pixel cache for ARGB pixels. #define APPLY_PALETTE_FOR(COLOR_INDEX) do { \ uint32_t prev_pix = palette[0]; \ @@ -1367,16 +1314,18 @@ static void PrepareMapToPalette(const uint32_t palette[], int num_colors, // using 'row' as a temporary buffer of size 'width'. // We assume that all src[] values have a corresponding entry in the palette. // Note: src[] can be the same as dst[] -static WebPEncodingError ApplyPalette(const uint32_t* src, uint32_t src_stride, - uint32_t* dst, uint32_t dst_stride, - const uint32_t* palette, int palette_size, - int width, int height, int xbits) { +static int ApplyPalette(const uint32_t* src, uint32_t src_stride, uint32_t* dst, + uint32_t dst_stride, const uint32_t* palette, + int palette_size, int width, int height, int xbits, + const WebPPicture* const pic) { // TODO(skal): this tmp buffer is not needed if VP8LBundleColorMap() can be // made to work in-place. uint8_t* const tmp_row = (uint8_t*)WebPSafeMalloc(width, sizeof(*tmp_row)); int x, y; - if (tmp_row == NULL) return VP8_ENC_ERROR_OUT_OF_MEMORY; + if (tmp_row == NULL) { + return WebPEncodingSetError(pic, VP8_ENC_ERROR_OUT_OF_MEMORY); + } if (palette_size < APPLY_PALETTE_GREEDY_MAX) { APPLY_PALETTE_FOR(SearchColorGreedy(palette, palette_size, pix)); @@ -1421,7 +1370,7 @@ static WebPEncodingError ApplyPalette(const uint32_t* src, uint32_t src_stride, } } WebPSafeFree(tmp_row); - return VP8_ENC_OK; + return 1; } #undef APPLY_PALETTE_FOR #undef PALETTE_INV_SIZE_BITS @@ -1429,9 +1378,7 @@ static WebPEncodingError ApplyPalette(const uint32_t* src, uint32_t src_stride, #undef APPLY_PALETTE_GREEDY_MAX // Note: Expects "enc->palette_" to be set properly. -static WebPEncodingError MapImageFromPalette(VP8LEncoder* const enc, - int in_place) { - WebPEncodingError err = VP8_ENC_OK; +static int MapImageFromPalette(VP8LEncoder* const enc, int in_place) { const WebPPicture* const pic = enc->pic_; const int width = pic->width; const int height = pic->height; @@ -1449,19 +1396,22 @@ static WebPEncodingError MapImageFromPalette(VP8LEncoder* const enc, xbits = (palette_size <= 16) ? 1 : 0; } - err = AllocateTransformBuffer(enc, VP8LSubSampleSize(width, xbits), height); - if (err != VP8_ENC_OK) return err; - - err = ApplyPalette(src, src_stride, + if (!AllocateTransformBuffer(enc, VP8LSubSampleSize(width, xbits), height)) { + return 0; + } + if (!ApplyPalette(src, src_stride, enc->argb_, enc->current_width_, - palette, palette_size, width, height, xbits); + palette, palette_size, width, height, xbits, pic)) { + return 0; + } enc->argb_content_ = kEncoderPalette; - return err; + return 1; } // Save palette_[] to bitstream. static WebPEncodingError EncodePalette(VP8LBitWriter* const bw, int low_effort, - VP8LEncoder* const enc) { + VP8LEncoder* const enc, + int percent_range, int* const percent) { int i; uint32_t tmp_palette[MAX_PALETTE_SIZE]; const int palette_size = enc->palette_size_; @@ -1476,7 +1426,7 @@ static WebPEncodingError EncodePalette(VP8LBitWriter* const bw, int low_effort, tmp_palette[0] = palette[0]; return EncodeImageNoHuffman(bw, tmp_palette, &enc->hash_chain_, &enc->refs_[0], palette_size, 1, /*quality=*/20, - low_effort); + low_effort, enc->pic_, percent_range, percent); } // ----------------------------------------------------------------------------- @@ -1520,7 +1470,6 @@ typedef struct { CrunchConfig crunch_configs_[CRUNCH_CONFIGS_MAX]; int num_crunch_configs_; int red_and_blue_always_zero_; - WebPEncodingError err_; WebPAuxStats* stats_; } StreamEncodeContext; @@ -1537,7 +1486,6 @@ static int EncodeStreamHook(void* input, void* data2) { #if !defined(WEBP_DISABLE_STATS) WebPAuxStats* const stats = params->stats_; #endif - WebPEncodingError err = VP8_ENC_OK; const int quality = (int)config->quality; const int low_effort = (config->method == 0); #if (WEBP_NEAR_LOSSLESS == 1) @@ -1545,6 +1493,7 @@ static int EncodeStreamHook(void* input, void* data2) { #endif const int height = picture->height; const size_t byte_position = VP8LBitWriterNumBytes(bw); + int percent = 2; // for WebPProgressHook #if (WEBP_NEAR_LOSSLESS == 1) int use_near_lossless = 0; #endif @@ -1558,12 +1507,13 @@ static int EncodeStreamHook(void* input, void* data2) { if (!VP8LBitWriterInit(&bw_best, 0) || (num_crunch_configs > 1 && !VP8LBitWriterClone(bw, &bw_best))) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } for (idx = 0; idx < num_crunch_configs; ++idx) { const int entropy_idx = crunch_configs[idx].entropy_idx_; + int remaining_percent = 97 / num_crunch_configs, percent_range; enc->use_palette_ = (entropy_idx == kPalette) || (entropy_idx == kPaletteAndSpatial); enc->use_subtract_green_ = @@ -1571,7 +1521,8 @@ static int EncodeStreamHook(void* input, void* data2) { enc->use_predict_ = (entropy_idx == kSpatial) || (entropy_idx == kSpatialSubGreen) || (entropy_idx == kPaletteAndSpatial); - if (low_effort) { + // When using a palette, R/B==0, hence no need to test for cross-color. + if (low_effort || enc->use_palette_) { enc->use_cross_color_ = 0; } else { enc->use_cross_color_ = red_and_blue_always_zero ? 0 : enc->use_predict_; @@ -1586,11 +1537,10 @@ static int EncodeStreamHook(void* input, void* data2) { use_near_lossless = (config->near_lossless < 100) && !enc->use_palette_ && !enc->use_predict_; if (use_near_lossless) { - err = AllocateTransformBuffer(enc, width, height); - if (err != VP8_ENC_OK) goto Error; + if (!AllocateTransformBuffer(enc, width, height)) goto Error; if ((enc->argb_content_ != kEncoderNearLossless) && !VP8ApplyNearLossless(picture, config->near_lossless, enc->argb_)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } enc->argb_content_ = kEncoderNearLossless; @@ -1603,10 +1553,18 @@ static int EncodeStreamHook(void* input, void* data2) { // Encode palette if (enc->use_palette_) { - err = EncodePalette(bw, low_effort, enc); - if (err != VP8_ENC_OK) goto Error; - err = MapImageFromPalette(enc, use_delta_palette); - if (err != VP8_ENC_OK) goto Error; + if (!PaletteSort(crunch_configs[idx].palette_sorting_type_, enc->pic_, + enc->palette_sorted_, enc->palette_size_, + enc->palette_)) { + WebPEncodingSetError(enc->pic_, VP8_ENC_ERROR_OUT_OF_MEMORY); + goto Error; + } + percent_range = remaining_percent / 4; + if (!EncodePalette(bw, low_effort, enc, percent_range, &percent)) { + goto Error; + } + remaining_percent -= percent_range; + if (!MapImageFromPalette(enc, use_delta_palette)) goto Error; // If using a color cache, do not have it bigger than the number of // colors. if (use_cache && enc->palette_size_ < (1 << MAX_COLOR_CACHE_BITS)) { @@ -1617,8 +1575,7 @@ static int EncodeStreamHook(void* input, void* data2) { // In case image is not packed. if (enc->argb_content_ != kEncoderNearLossless && enc->argb_content_ != kEncoderPalette) { - err = MakeInputImageCopy(enc); - if (err != VP8_ENC_OK) goto Error; + if (!MakeInputImageCopy(enc)) goto Error; } // ----------------------------------------------------------------------- @@ -1629,15 +1586,22 @@ static int EncodeStreamHook(void* input, void* data2) { } if (enc->use_predict_) { - err = ApplyPredictFilter(enc, enc->current_width_, height, quality, - low_effort, enc->use_subtract_green_, bw); - if (err != VP8_ENC_OK) goto Error; + percent_range = remaining_percent / 3; + if (!ApplyPredictFilter(enc, enc->current_width_, height, quality, + low_effort, enc->use_subtract_green_, bw, + percent_range, &percent)) { + goto Error; + } + remaining_percent -= percent_range; } if (enc->use_cross_color_) { - err = ApplyCrossColorFilter(enc, enc->current_width_, height, quality, - low_effort, bw); - if (err != VP8_ENC_OK) goto Error; + percent_range = remaining_percent / 2; + if (!ApplyCrossColorFilter(enc, enc->current_width_, height, quality, + low_effort, bw, percent_range, &percent)) { + goto Error; + } + remaining_percent -= percent_range; } } @@ -1645,12 +1609,13 @@ static int EncodeStreamHook(void* input, void* data2) { // ------------------------------------------------------------------------- // Encode and write the transformed image. - err = EncodeImageInternal(bw, enc->argb_, &enc->hash_chain_, enc->refs_, - enc->current_width_, height, quality, low_effort, - use_cache, &crunch_configs[idx], - &enc->cache_bits_, enc->histo_bits_, - byte_position, &hdr_size, &data_size); - if (err != VP8_ENC_OK) goto Error; + if (!EncodeImageInternal( + bw, enc->argb_, &enc->hash_chain_, enc->refs_, enc->current_width_, + height, quality, low_effort, use_cache, &crunch_configs[idx], + &enc->cache_bits_, enc->histo_bits_, byte_position, &hdr_size, + &data_size, picture, remaining_percent, &percent)) { + goto Error; + } // If we are better than what we already have. if (VP8LBitWriterNumBytes(bw) < best_size) { @@ -1680,18 +1645,15 @@ static int EncodeStreamHook(void* input, void* data2) { } VP8LBitWriterSwap(&bw_best, bw); -Error: + Error: VP8LBitWriterWipeOut(&bw_best); - params->err_ = err; // The hook should return false in case of error. - return (err == VP8_ENC_OK); + return (params->picture_->error_code == VP8_ENC_OK); } -WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, - const WebPPicture* const picture, - VP8LBitWriter* const bw_main, - int use_cache) { - WebPEncodingError err = VP8_ENC_OK; +int VP8LEncodeStream(const WebPConfig* const config, + const WebPPicture* const picture, + VP8LBitWriter* const bw_main, int use_cache) { VP8LEncoder* const enc_main = VP8LEncoderNew(config, picture); VP8LEncoder* enc_side = NULL; CrunchConfig crunch_configs[CRUNCH_CONFIGS_MAX]; @@ -1703,15 +1665,23 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, // The main thread uses picture->stats, the side thread uses stats_side. WebPAuxStats stats_side; VP8LBitWriter bw_side; + WebPPicture picture_side; const WebPWorkerInterface* const worker_interface = WebPGetWorkerInterface(); int ok_main; + if (enc_main == NULL || !VP8LBitWriterInit(&bw_side, 0)) { + VP8LEncoderDelete(enc_main); + return WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); + } + + // Avoid "garbage value" error from Clang's static analysis tool. + WebPPictureInit(&picture_side); + // Analyze image (entropy, num_palettes etc) - if (enc_main == NULL || - !EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main, + if (!EncoderAnalyze(enc_main, crunch_configs, &num_crunch_configs_main, &red_and_blue_always_zero) || - !EncoderInit(enc_main) || !VP8LBitWriterInit(&bw_side, 0)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + !EncoderInit(enc_main)) { + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } @@ -1740,25 +1710,32 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, StreamEncodeContext* const param = (idx == 0) ? ¶ms_main : ¶ms_side; param->config_ = config; - param->picture_ = picture; param->use_cache_ = use_cache; param->red_and_blue_always_zero_ = red_and_blue_always_zero; if (idx == 0) { + param->picture_ = picture; param->stats_ = picture->stats; param->bw_ = bw_main; param->enc_ = enc_main; } else { + // Create a side picture (error_code is not thread-safe). + if (!WebPPictureView(picture, /*left=*/0, /*top=*/0, picture->width, + picture->height, &picture_side)) { + assert(0); + } + picture_side.progress_hook = NULL; // Progress hook is not thread-safe. + param->picture_ = &picture_side; // No need to free a view afterwards. param->stats_ = (picture->stats == NULL) ? NULL : &stats_side; // Create a side bit writer. if (!VP8LBitWriterClone(bw_main, &bw_side)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } param->bw_ = &bw_side; // Create a side encoder. - enc_side = VP8LEncoderNew(config, picture); + enc_side = VP8LEncoderNew(config, &picture_side); if (enc_side == NULL || !EncoderInit(enc_side)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } // Copy the values that were computed for the main encoder. @@ -1767,6 +1744,8 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, enc_side->palette_size_ = enc_main->palette_size_; memcpy(enc_side->palette_, enc_main->palette_, sizeof(enc_main->palette_)); + memcpy(enc_side->palette_sorted_, enc_main->palette_sorted_, + sizeof(enc_main->palette_sorted_)); param->enc_ = enc_side; } // Create the workers. @@ -1780,7 +1759,7 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, // Start the second thread if needed. if (num_crunch_configs_side != 0) { if (!worker_interface->Reset(&worker_side)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } #if !defined(WEBP_DISABLE_STATS) @@ -1790,8 +1769,6 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, memcpy(&stats_side, picture->stats, sizeof(stats_side)); } #endif - // This line is only useful to remove a Clang static analyzer warning. - params_side.err_ = VP8_ENC_OK; worker_interface->Launch(&worker_side); } // Execute the main thread. @@ -1803,7 +1780,10 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, const int ok_side = worker_interface->Sync(&worker_side); worker_interface->End(&worker_side); if (!ok_main || !ok_side) { - err = ok_main ? params_side.err_ : params_main.err_; + if (picture->error_code == VP8_ENC_OK) { + assert(picture_side.error_code != VP8_ENC_OK); + WebPEncodingSetError(picture, picture_side.error_code); + } goto Error; } if (VP8LBitWriterNumBytes(&bw_side) < VP8LBitWriterNumBytes(bw_main)) { @@ -1814,18 +1794,13 @@ WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, } #endif } - } else { - if (!ok_main) { - err = params_main.err_; - goto Error; - } } -Error: + Error: VP8LBitWriterWipeOut(&bw_side); VP8LEncoderDelete(enc_main); VP8LEncoderDelete(enc_side); - return err; + return (picture->error_code == VP8_ENC_OK); } #undef CRUNCH_CONFIGS_MAX @@ -1838,15 +1813,12 @@ int VP8LEncodeImage(const WebPConfig* const config, size_t coded_size; int percent = 0; int initial_size; - WebPEncodingError err = VP8_ENC_OK; VP8LBitWriter bw; if (picture == NULL) return 0; if (config == NULL || picture->argb == NULL) { - err = VP8_ENC_ERROR_NULL_PARAMETER; - WebPEncodingSetError(picture, err); - return 0; + return WebPEncodingSetError(picture, VP8_ENC_ERROR_NULL_PARAMETER); } width = picture->width; @@ -1856,13 +1828,13 @@ int VP8LEncodeImage(const WebPConfig* const config, initial_size = (config->image_hint == WEBP_HINT_GRAPH) ? width * height : width * height * 2; if (!VP8LBitWriterInit(&bw, initial_size)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } if (!WebPReportProgress(picture, 1, &percent)) { UserAbort: - err = VP8_ENC_ERROR_USER_ABORT; + WebPEncodingSetError(picture, VP8_ENC_ERROR_USER_ABORT); goto Error; } // Reset stats (for pure lossless coding) @@ -1878,28 +1850,26 @@ int VP8LEncodeImage(const WebPConfig* const config, // Write image size. if (!WriteImageSize(picture, &bw)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } has_alpha = WebPPictureHasTransparency(picture); // Write the non-trivial Alpha flag and lossless version. if (!WriteRealAlphaAndVersion(&bw, has_alpha)) { - err = VP8_ENC_ERROR_OUT_OF_MEMORY; + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); goto Error; } - if (!WebPReportProgress(picture, 5, &percent)) goto UserAbort; + if (!WebPReportProgress(picture, 2, &percent)) goto UserAbort; // Encode main image stream. - err = VP8LEncodeStream(config, picture, &bw, 1 /*use_cache*/); - if (err != VP8_ENC_OK) goto Error; + if (!VP8LEncodeStream(config, picture, &bw, 1 /*use_cache*/)) goto Error; - if (!WebPReportProgress(picture, 90, &percent)) goto UserAbort; + if (!WebPReportProgress(picture, 99, &percent)) goto UserAbort; // Finish the RIFF chunk. - err = WriteImage(picture, &bw, &coded_size); - if (err != VP8_ENC_OK) goto Error; + if (!WriteImage(picture, &bw, &coded_size)) goto Error; if (!WebPReportProgress(picture, 100, &percent)) goto UserAbort; @@ -1918,13 +1888,11 @@ int VP8LEncodeImage(const WebPConfig* const config, } Error: - if (bw.error_) err = VP8_ENC_ERROR_OUT_OF_MEMORY; - VP8LBitWriterWipeOut(&bw); - if (err != VP8_ENC_OK) { - WebPEncodingSetError(picture, err); - return 0; + if (bw.error_) { + WebPEncodingSetError(picture, VP8_ENC_ERROR_OUT_OF_MEMORY); } - return 1; + VP8LBitWriterWipeOut(&bw); + return (picture->error_code == VP8_ENC_OK); } //------------------------------------------------------------------------------ diff --git a/3rdparty/libwebp/src/enc/vp8li_enc.h b/3rdparty/libwebp/src/enc/vp8li_enc.h index 94210ce9f3..3d35e1612d 100644 --- a/3rdparty/libwebp/src/enc/vp8li_enc.h +++ b/3rdparty/libwebp/src/enc/vp8li_enc.h @@ -69,6 +69,8 @@ typedef struct { int use_palette_; int palette_size_; uint32_t palette_[MAX_PALETTE_SIZE]; + // Sorted version of palette_ for cache purposes. + uint32_t palette_sorted_[MAX_PALETTE_SIZE]; // Some 'scratch' (potentially large) objects. struct VP8LBackwardRefs refs_[4]; // Backward Refs array for temporaries. @@ -87,9 +89,10 @@ int VP8LEncodeImage(const WebPConfig* const config, // Encodes the main image stream using the supplied bit writer. // If 'use_cache' is false, disables the use of color cache. -WebPEncodingError VP8LEncodeStream(const WebPConfig* const config, - const WebPPicture* const picture, - VP8LBitWriter* const bw, int use_cache); +// Returns false in case of error (stored in picture->error_code). +int VP8LEncodeStream(const WebPConfig* const config, + const WebPPicture* const picture, VP8LBitWriter* const bw, + int use_cache); #if (WEBP_NEAR_LOSSLESS == 1) // in near_lossless.c @@ -101,13 +104,18 @@ int VP8ApplyNearLossless(const WebPPicture* const picture, int quality, //------------------------------------------------------------------------------ // Image transforms in predictor.c. -void VP8LResidualImage(int width, int height, int bits, int low_effort, - uint32_t* const argb, uint32_t* const argb_scratch, - uint32_t* const image, int near_lossless, int exact, - int used_subtract_green); +// pic and percent are for progress. +// Returns false in case of error (stored in pic->error_code). +int VP8LResidualImage(int width, int height, int bits, int low_effort, + uint32_t* const argb, uint32_t* const argb_scratch, + uint32_t* const image, int near_lossless, int exact, + int used_subtract_green, const WebPPicture* const pic, + int percent_range, int* const percent); -void VP8LColorSpaceTransform(int width, int height, int bits, int quality, - uint32_t* const argb, uint32_t* image); +int VP8LColorSpaceTransform(int width, int height, int bits, int quality, + uint32_t* const argb, uint32_t* image, + const WebPPicture* const pic, int percent_range, + int* const percent); //------------------------------------------------------------------------------ diff --git a/3rdparty/libwebp/src/enc/webp_enc.c b/3rdparty/libwebp/src/enc/webp_enc.c index ce2db2e94b..583fe6a8bb 100644 --- a/3rdparty/libwebp/src/enc/webp_enc.c +++ b/3rdparty/libwebp/src/enc/webp_enc.c @@ -307,7 +307,10 @@ int WebPEncodingSetError(const WebPPicture* const pic, WebPEncodingError error) { assert((int)error < VP8_ENC_ERROR_LAST); assert((int)error >= VP8_ENC_OK); - ((WebPPicture*)pic)->error_code = error; + // The oldest error reported takes precedence over the new one. + if (pic->error_code == VP8_ENC_OK) { + ((WebPPicture*)pic)->error_code = error; + } return 0; } @@ -317,8 +320,7 @@ int WebPReportProgress(const WebPPicture* const pic, *percent_store = percent; if (pic->progress_hook && !pic->progress_hook(percent, pic)) { // user abort requested - WebPEncodingSetError(pic, VP8_ENC_ERROR_USER_ABORT); - return 0; + return WebPEncodingSetError(pic, VP8_ENC_ERROR_USER_ABORT); } } return 1; // ok @@ -329,16 +331,14 @@ int WebPEncode(const WebPConfig* config, WebPPicture* pic) { int ok = 0; if (pic == NULL) return 0; - WebPEncodingSetError(pic, VP8_ENC_OK); // all ok so far + pic->error_code = VP8_ENC_OK; // all ok so far if (config == NULL) { // bad params return WebPEncodingSetError(pic, VP8_ENC_ERROR_NULL_PARAMETER); } if (!WebPValidateConfig(config)) { return WebPEncodingSetError(pic, VP8_ENC_ERROR_INVALID_CONFIGURATION); } - if (pic->width <= 0 || pic->height <= 0) { - return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_DIMENSION); - } + if (!WebPValidatePicture(pic)) return 0; if (pic->width > WEBP_MAX_DIMENSION || pic->height > WEBP_MAX_DIMENSION) { return WebPEncodingSetError(pic, VP8_ENC_ERROR_BAD_DIMENSION); } diff --git a/3rdparty/libwebp/src/mux/anim_encode.c b/3rdparty/libwebp/src/mux/anim_encode.c index 7be99068f6..d1c61a2f1e 100644 --- a/3rdparty/libwebp/src/mux/anim_encode.c +++ b/3rdparty/libwebp/src/mux/anim_encode.c @@ -248,9 +248,6 @@ WebPAnimEncoder* WebPAnimEncoderNewInternal( enc = (WebPAnimEncoder*)WebPSafeCalloc(1, sizeof(*enc)); if (enc == NULL) return NULL; - // sanity inits, so we can call WebPAnimEncoderDelete(): - enc->encoded_frames_ = NULL; - enc->mux_ = NULL; MarkNoError(enc); // Dimensions and options. @@ -421,7 +418,7 @@ static void MinimizeChangeRectangle(const WebPPicture* const src, const int max_allowed_diff_lossy = QualityToMaxDiff(quality); const int max_allowed_diff = is_lossless ? 0 : max_allowed_diff_lossy; - // Sanity checks. + // Assumption/correctness checks. assert(src->width == dst->width && src->height == dst->height); assert(rect->x_offset_ + rect->width_ <= dst->width); assert(rect->y_offset_ + rect->height_ <= dst->height); @@ -596,16 +593,17 @@ int WebPAnimEncoderRefineRect( int is_lossless, float quality, int* const x_offset, int* const y_offset, int* const width, int* const height) { FrameRectangle rect; - const int right = clip(*x_offset + *width, 0, curr_canvas->width); - const int left = clip(*x_offset, 0, curr_canvas->width - 1); - const int bottom = clip(*y_offset + *height, 0, curr_canvas->height); - const int top = clip(*y_offset, 0, curr_canvas->height - 1); + int right, left, bottom, top; if (prev_canvas == NULL || curr_canvas == NULL || prev_canvas->width != curr_canvas->width || prev_canvas->height != curr_canvas->height || !prev_canvas->use_argb || !curr_canvas->use_argb) { return 0; } + right = clip(*x_offset + *width, 0, curr_canvas->width); + left = clip(*x_offset, 0, curr_canvas->width - 1); + bottom = clip(*y_offset + *height, 0, curr_canvas->height); + top = clip(*y_offset, 0, curr_canvas->height - 1); rect.x_offset_ = left; rect.y_offset_ = top; rect.width_ = clip(right - left, 0, curr_canvas->width - rect.x_offset_); @@ -949,7 +947,8 @@ static int IncreasePreviousDuration(WebPAnimEncoder* const enc, int duration) { int new_duration; assert(enc->count_ >= 1); - assert(prev_enc_frame->sub_frame_.duration == + assert(!prev_enc_frame->is_key_frame_ || + prev_enc_frame->sub_frame_.duration == prev_enc_frame->key_frame_.duration); assert(prev_enc_frame->sub_frame_.duration == (prev_enc_frame->sub_frame_.duration & (MAX_DURATION - 1))); @@ -966,7 +965,7 @@ static int IncreasePreviousDuration(WebPAnimEncoder* const enc, int duration) { 0x10, 0x88, 0x88, 0x08 }; const WebPData lossless_1x1 = { - lossless_1x1_bytes, sizeof(lossless_1x1_bytes) + lossless_1x1_bytes, sizeof(lossless_1x1_bytes) }; const uint8_t lossy_1x1_bytes[] = { 0x52, 0x49, 0x46, 0x46, 0x40, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, @@ -1358,6 +1357,12 @@ int WebPAnimEncoderAdd(WebPAnimEncoder* enc, WebPPicture* frame, int timestamp, if (!IncreasePreviousDuration(enc, (int)prev_frame_duration)) { return 0; } + // IncreasePreviousDuration() may add a frame to avoid exceeding + // MAX_DURATION which could cause CacheFrame() to over read encoded_frames_ + // before the next flush. + if (enc->count_ == enc->size_ && !FlushFrames(enc)) { + return 0; + } } else { enc->first_timestamp_ = timestamp; } diff --git a/3rdparty/libwebp/src/mux/muxedit.c b/3rdparty/libwebp/src/mux/muxedit.c index ccf14b2a0c..aab479cc6c 100644 --- a/3rdparty/libwebp/src/mux/muxedit.c +++ b/3rdparty/libwebp/src/mux/muxedit.c @@ -70,6 +70,7 @@ void WebPMuxDelete(WebPMux* mux) { err = ChunkAssignData(&chunk, data, copy_data, tag); \ if (err == WEBP_MUX_OK) { \ err = ChunkSetHead(&chunk, (LIST)); \ + if (err != WEBP_MUX_OK) ChunkRelease(&chunk); \ } \ return err; \ } @@ -235,7 +236,6 @@ WebPMuxError WebPMuxSetImage(WebPMux* mux, const WebPData* bitstream, WebPMuxImage wpi; WebPMuxError err; - // Sanity checks. if (mux == NULL || bitstream == NULL || bitstream->bytes == NULL || bitstream->size > MAX_CHUNK_PAYLOAD) { return WEBP_MUX_INVALID_ARGUMENT; @@ -267,7 +267,6 @@ WebPMuxError WebPMuxPushFrame(WebPMux* mux, const WebPMuxFrameInfo* info, WebPMuxImage wpi; WebPMuxError err; - // Sanity checks. if (mux == NULL || info == NULL) return WEBP_MUX_INVALID_ARGUMENT; if (info->id != WEBP_CHUNK_ANMF) return WEBP_MUX_INVALID_ARGUMENT; @@ -556,7 +555,8 @@ static WebPMuxError MuxCleanup(WebPMux* const mux) { if (num_frames == 1) { WebPMuxImage* frame = NULL; err = MuxImageGetNth((const WebPMuxImage**)&mux->images_, 1, &frame); - assert(err == WEBP_MUX_OK); // We know that one frame does exist. + if (err != WEBP_MUX_OK) return err; + // We know that one frame does exist. assert(frame != NULL); if (frame->header_ != NULL && ((mux->canvas_width_ == 0 && mux->canvas_height_ == 0) || diff --git a/3rdparty/libwebp/src/mux/muxi.h b/3rdparty/libwebp/src/mux/muxi.h index 2289822e8f..fc44d6f2fe 100644 --- a/3rdparty/libwebp/src/mux/muxi.h +++ b/3rdparty/libwebp/src/mux/muxi.h @@ -28,8 +28,8 @@ extern "C" { // Defines and constants. #define MUX_MAJ_VERSION 1 -#define MUX_MIN_VERSION 2 -#define MUX_REV_VERSION 0 +#define MUX_MIN_VERSION 3 +#define MUX_REV_VERSION 1 // Chunk object. typedef struct WebPChunk WebPChunk; diff --git a/3rdparty/libwebp/src/mux/muxinternal.c b/3rdparty/libwebp/src/mux/muxinternal.c index b9ee6717d3..75b6b416b9 100644 --- a/3rdparty/libwebp/src/mux/muxinternal.c +++ b/3rdparty/libwebp/src/mux/muxinternal.c @@ -155,17 +155,18 @@ WebPMuxError ChunkSetHead(WebPChunk* const chunk, WebPMuxError ChunkAppend(WebPChunk* const chunk, WebPChunk*** const chunk_list) { + WebPMuxError err; assert(chunk_list != NULL && *chunk_list != NULL); if (**chunk_list == NULL) { - ChunkSetHead(chunk, *chunk_list); + err = ChunkSetHead(chunk, *chunk_list); } else { WebPChunk* last_chunk = **chunk_list; while (last_chunk->next_ != NULL) last_chunk = last_chunk->next_; - ChunkSetHead(chunk, &last_chunk->next_); - *chunk_list = &last_chunk->next_; + err = ChunkSetHead(chunk, &last_chunk->next_); + if (err == WEBP_MUX_OK) *chunk_list = &last_chunk->next_; } - return WEBP_MUX_OK; + return err; } //------------------------------------------------------------------------------ diff --git a/3rdparty/libwebp/src/mux/muxread.c b/3rdparty/libwebp/src/mux/muxread.c index 0101fde15d..9862ec68ee 100644 --- a/3rdparty/libwebp/src/mux/muxread.c +++ b/3rdparty/libwebp/src/mux/muxread.c @@ -56,7 +56,7 @@ static WebPMuxError ChunkVerifyAndAssign(WebPChunk* chunk, uint32_t chunk_size; WebPData chunk_data; - // Sanity checks. + // Correctness checks. if (data_size < CHUNK_HEADER_SIZE) return WEBP_MUX_NOT_ENOUGH_DATA; chunk_size = GetLE32(data + TAG_SIZE); if (chunk_size > MAX_CHUNK_PAYLOAD) return WEBP_MUX_BAD_DATA; @@ -116,9 +116,12 @@ static int MuxImageParse(const WebPChunk* const chunk, int copy_data, // Each of ANMF chunk contain a header at the beginning. So, its size should // be at least 'hdr_size'. if (size < hdr_size) goto Fail; - ChunkAssignData(&subchunk, &temp, copy_data, chunk->tag_); + if (ChunkAssignData(&subchunk, &temp, copy_data, + chunk->tag_) != WEBP_MUX_OK) { + goto Fail; + } } - ChunkSetHead(&subchunk, &wpi->header_); + if (ChunkSetHead(&subchunk, &wpi->header_) != WEBP_MUX_OK) goto Fail; wpi->is_partial_ = 1; // Waiting for ALPH and/or VP8/VP8L chunks. // Rest of the chunks. @@ -186,7 +189,6 @@ WebPMux* WebPMuxCreateInternal(const WebPData* bitstream, int copy_data, WebPChunk** chunk_list_ends[WEBP_CHUNK_NIL + 1] = { NULL }; ChunkInit(&chunk); - // Sanity checks. if (WEBP_ABI_IS_INCOMPATIBLE(version, WEBP_MUX_ABI_VERSION)) { return NULL; // version mismatch } @@ -481,7 +483,6 @@ WebPMuxError WebPMuxGetFrame( WebPMuxError err; WebPMuxImage* wpi; - // Sanity checks. if (mux == NULL || frame == NULL) { return WEBP_MUX_INVALID_ARGUMENT; } diff --git a/3rdparty/libwebp/src/utils/bit_reader_inl_utils.h b/3rdparty/libwebp/src/utils/bit_reader_inl_utils.h index 46b3880706..24f3af7b54 100644 --- a/3rdparty/libwebp/src/utils/bit_reader_inl_utils.h +++ b/3rdparty/libwebp/src/utils/bit_reader_inl_utils.h @@ -55,7 +55,7 @@ void VP8LoadFinalBytes(VP8BitReader* const br); // makes sure br->value_ has at least BITS bits worth of data static WEBP_UBSAN_IGNORE_UNDEF WEBP_INLINE -void VP8LoadNewBytes(VP8BitReader* const br) { +void VP8LoadNewBytes(VP8BitReader* WEBP_RESTRICT const br) { assert(br != NULL && br->buf_ != NULL); // Read 'BITS' bits at a time if possible. if (br->buf_ < br->buf_max_) { @@ -104,7 +104,7 @@ void VP8LoadNewBytes(VP8BitReader* const br) { } // Read a bit with proba 'prob'. Speed-critical function! -static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, +static WEBP_INLINE int VP8GetBit(VP8BitReader* WEBP_RESTRICT const br, int prob, const char label[]) { // Don't move this declaration! It makes a big speed difference to store // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't @@ -137,7 +137,8 @@ static WEBP_INLINE int VP8GetBit(VP8BitReader* const br, // simplified version of VP8GetBit() for prob=0x80 (note shift is always 1 here) static WEBP_UBSAN_IGNORE_UNSIGNED_OVERFLOW WEBP_INLINE -int VP8GetSigned(VP8BitReader* const br, int v, const char label[]) { +int VP8GetSigned(VP8BitReader* WEBP_RESTRICT const br, int v, + const char label[]) { if (br->bits_ < 0) { VP8LoadNewBytes(br); } @@ -147,15 +148,15 @@ int VP8GetSigned(VP8BitReader* const br, int v, const char label[]) { const range_t value = (range_t)(br->value_ >> pos); const int32_t mask = (int32_t)(split - value) >> 31; // -1 or 0 br->bits_ -= 1; - br->range_ += mask; + br->range_ += (range_t)mask; br->range_ |= 1; - br->value_ -= (bit_t)((split + 1) & mask) << pos; + br->value_ -= (bit_t)((split + 1) & (uint32_t)mask) << pos; BT_TRACK(br); return (v ^ mask) - mask; } } -static WEBP_INLINE int VP8GetBitAlt(VP8BitReader* const br, +static WEBP_INLINE int VP8GetBitAlt(VP8BitReader* WEBP_RESTRICT const br, int prob, const char label[]) { // Don't move this declaration! It makes a big speed difference to store // 'range' *before* calling VP8LoadNewBytes(), even if this function doesn't diff --git a/3rdparty/libwebp/src/utils/bit_reader_utils.c b/3rdparty/libwebp/src/utils/bit_reader_utils.c index 857cd60988..a26557aa49 100644 --- a/3rdparty/libwebp/src/utils/bit_reader_utils.c +++ b/3rdparty/libwebp/src/utils/bit_reader_utils.c @@ -15,6 +15,7 @@ #include "src/webp/config.h" #endif +#include "src/dsp/cpu.h" #include "src/utils/bit_reader_inl_utils.h" #include "src/utils/utils.h" @@ -121,7 +122,7 @@ int32_t VP8GetSignedValue(VP8BitReader* const br, int bits, #define VP8L_LOG8_WBITS 4 // Number of bytes needed to store VP8L_WBITS bits. -#if defined(__arm__) || defined(_M_ARM) || defined(__aarch64__) || \ +#if defined(__arm__) || defined(_M_ARM) || WEBP_AARCH64 || \ defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64__) || defined(_M_X64) #define VP8L_USE_FAST_LOAD diff --git a/3rdparty/libwebp/src/utils/bit_reader_utils.h b/3rdparty/libwebp/src/utils/bit_reader_utils.h index e64156e318..25ff31e5d9 100644 --- a/3rdparty/libwebp/src/utils/bit_reader_utils.h +++ b/3rdparty/libwebp/src/utils/bit_reader_utils.h @@ -19,6 +19,7 @@ #ifdef _MSC_VER #include // _byteswap_ulong #endif +#include "src/dsp/cpu.h" #include "src/webp/types.h" // Warning! This macro triggers quite some MACRO wizardry around func signature! @@ -64,7 +65,7 @@ extern "C" { #define BITS 56 #elif defined(__arm__) || defined(_M_ARM) // ARM #define BITS 24 -#elif defined(__aarch64__) // ARM 64bit +#elif WEBP_AARCH64 // ARM 64bit #define BITS 56 #elif defined(__mips__) // MIPS #define BITS 24 diff --git a/3rdparty/libwebp/src/utils/bit_writer_utils.c b/3rdparty/libwebp/src/utils/bit_writer_utils.c index bef0e31ca5..2f408508f1 100644 --- a/3rdparty/libwebp/src/utils/bit_writer_utils.c +++ b/3rdparty/libwebp/src/utils/bit_writer_utils.c @@ -278,7 +278,7 @@ void VP8LPutBitsFlushBits(VP8LBitWriter* const bw) { // If needed, make some room by flushing some bits out. if (bw->cur_ + VP8L_WRITER_BYTES > bw->end_) { const uint64_t extra_size = (bw->end_ - bw->buf_) + MIN_EXTRA_SIZE; - if (extra_size != (size_t)extra_size || + if (!CheckSizeOverflow(extra_size) || !VP8LBitWriterResize(bw, (size_t)extra_size)) { bw->cur_ = bw->buf_; bw->error_ = 1; @@ -314,7 +314,7 @@ void VP8LPutBitsInternal(VP8LBitWriter* const bw, uint32_t bits, int n_bits) { while (used >= VP8L_WRITER_BITS) { if (bw->cur_ + VP8L_WRITER_BYTES > bw->end_) { const uint64_t extra_size = (bw->end_ - bw->buf_) + MIN_EXTRA_SIZE; - if (extra_size != (size_t)extra_size || + if (!CheckSizeOverflow(extra_size) || !VP8LBitWriterResize(bw, (size_t)extra_size)) { bw->cur_ = bw->buf_; bw->error_ = 1; diff --git a/3rdparty/libwebp/src/utils/color_cache_utils.c b/3rdparty/libwebp/src/utils/color_cache_utils.c index b09f538e8b..7b5222b6e5 100644 --- a/3rdparty/libwebp/src/utils/color_cache_utils.c +++ b/3rdparty/libwebp/src/utils/color_cache_utils.c @@ -20,22 +20,22 @@ //------------------------------------------------------------------------------ // VP8LColorCache. -int VP8LColorCacheInit(VP8LColorCache* const cc, int hash_bits) { +int VP8LColorCacheInit(VP8LColorCache* const color_cache, int hash_bits) { const int hash_size = 1 << hash_bits; - assert(cc != NULL); + assert(color_cache != NULL); assert(hash_bits > 0); - cc->colors_ = (uint32_t*)WebPSafeCalloc((uint64_t)hash_size, - sizeof(*cc->colors_)); - if (cc->colors_ == NULL) return 0; - cc->hash_shift_ = 32 - hash_bits; - cc->hash_bits_ = hash_bits; + color_cache->colors_ = (uint32_t*)WebPSafeCalloc( + (uint64_t)hash_size, sizeof(*color_cache->colors_)); + if (color_cache->colors_ == NULL) return 0; + color_cache->hash_shift_ = 32 - hash_bits; + color_cache->hash_bits_ = hash_bits; return 1; } -void VP8LColorCacheClear(VP8LColorCache* const cc) { - if (cc != NULL) { - WebPSafeFree(cc->colors_); - cc->colors_ = NULL; +void VP8LColorCacheClear(VP8LColorCache* const color_cache) { + if (color_cache != NULL) { + WebPSafeFree(color_cache->colors_); + color_cache->colors_ = NULL; } } diff --git a/3rdparty/libwebp/src/utils/huffman_encode_utils.c b/3rdparty/libwebp/src/utils/huffman_encode_utils.c index 6f3b1bbe02..585db91951 100644 --- a/3rdparty/libwebp/src/utils/huffman_encode_utils.c +++ b/3rdparty/libwebp/src/utils/huffman_encode_utils.c @@ -161,7 +161,7 @@ static void SetBitDepths(const HuffmanTree* const tree, // especially when population counts are longer than 2**tree_limit, but // we are not planning to use this with extremely long blocks. // -// See http://en.wikipedia.org/wiki/Huffman_coding +// See https://en.wikipedia.org/wiki/Huffman_coding static void GenerateOptimalTree(const uint32_t* const histogram, int histogram_size, HuffmanTree* tree, int tree_depth_limit, @@ -404,8 +404,7 @@ static void ConvertBitDepthsToSymbols(HuffmanTreeCode* const tree) { // Main entry point void VP8LCreateHuffmanTree(uint32_t* const histogram, int tree_depth_limit, - uint8_t* const buf_rle, - HuffmanTree* const huff_tree, + uint8_t* const buf_rle, HuffmanTree* const huff_tree, HuffmanTreeCode* const huff_code) { const int num_symbols = huff_code->num_symbols; memset(buf_rle, 0, num_symbols * sizeof(*buf_rle)); diff --git a/3rdparty/libwebp/src/utils/huffman_encode_utils.h b/3rdparty/libwebp/src/utils/huffman_encode_utils.h index 3e6763ce49..3f7f1d8074 100644 --- a/3rdparty/libwebp/src/utils/huffman_encode_utils.h +++ b/3rdparty/libwebp/src/utils/huffman_encode_utils.h @@ -51,7 +51,7 @@ int VP8LCreateCompressedHuffmanTree(const HuffmanTreeCode* const tree, // huffman code tree. void VP8LCreateHuffmanTree(uint32_t* const histogram, int tree_depth_limit, uint8_t* const buf_rle, HuffmanTree* const huff_tree, - HuffmanTreeCode* const tree); + HuffmanTreeCode* const huff_code); #ifdef __cplusplus } diff --git a/3rdparty/libwebp/src/utils/huffman_utils.c b/3rdparty/libwebp/src/utils/huffman_utils.c index 0cba0fbb7d..cf73abd437 100644 --- a/3rdparty/libwebp/src/utils/huffman_utils.c +++ b/3rdparty/libwebp/src/utils/huffman_utils.c @@ -142,7 +142,7 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, { int step; // step size to replicate values in current table - uint32_t low = -1; // low bits for current root entry + uint32_t low = 0xffffffffu; // low bits for current root entry uint32_t mask = total_size - 1; // mask for low bits uint32_t key = 0; // reversed prefix code int num_nodes = 1; // number of Huffman tree nodes @@ -177,21 +177,24 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, if (num_open < 0) { return 0; } - if (root_table == NULL) continue; for (; count[len] > 0; --count[len]) { HuffmanCode code; if ((key & mask) != low) { - table += table_size; + if (root_table != NULL) table += table_size; table_bits = NextTableBitSize(count, len, root_bits); table_size = 1 << table_bits; total_size += table_size; low = key & mask; - root_table[low].bits = (uint8_t)(table_bits + root_bits); - root_table[low].value = (uint16_t)((table - root_table) - low); + if (root_table != NULL) { + root_table[low].bits = (uint8_t)(table_bits + root_bits); + root_table[low].value = (uint16_t)((table - root_table) - low); + } + } + if (root_table != NULL) { + code.bits = (uint8_t)(len - root_bits); + code.value = (uint16_t)sorted[symbol++]; + ReplicateValue(&table[key >> root_bits], step, table_size, code); } - code.bits = (uint8_t)(len - root_bits); - code.value = (uint16_t)sorted[symbol++]; - ReplicateValue(&table[key >> root_bits], step, table_size, code); key = GetNextKey(key, len); } } @@ -211,25 +214,83 @@ static int BuildHuffmanTable(HuffmanCode* const root_table, int root_bits, ((1 << MAX_CACHE_BITS) + NUM_LITERAL_CODES + NUM_LENGTH_CODES) // Cut-off value for switching between heap and stack allocation. #define SORTED_SIZE_CUTOFF 512 -int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits, +int VP8LBuildHuffmanTable(HuffmanTables* const root_table, int root_bits, const int code_lengths[], int code_lengths_size) { - int total_size; + const int total_size = + BuildHuffmanTable(NULL, root_bits, code_lengths, code_lengths_size, NULL); assert(code_lengths_size <= MAX_CODE_LENGTHS_SIZE); - if (root_table == NULL) { - total_size = BuildHuffmanTable(NULL, root_bits, - code_lengths, code_lengths_size, NULL); - } else if (code_lengths_size <= SORTED_SIZE_CUTOFF) { + if (total_size == 0 || root_table == NULL) return total_size; + + if (root_table->curr_segment->curr_table + total_size >= + root_table->curr_segment->start + root_table->curr_segment->size) { + // If 'root_table' does not have enough memory, allocate a new segment. + // The available part of root_table->curr_segment is left unused because we + // need a contiguous buffer. + const int segment_size = root_table->curr_segment->size; + struct HuffmanTablesSegment* next = + (HuffmanTablesSegment*)WebPSafeMalloc(1, sizeof(*next)); + if (next == NULL) return 0; + // Fill the new segment. + // We need at least 'total_size' but if that value is small, it is better to + // allocate a big chunk to prevent more allocations later. 'segment_size' is + // therefore chosen (any other arbitrary value could be chosen). + next->size = total_size > segment_size ? total_size : segment_size; + next->start = + (HuffmanCode*)WebPSafeMalloc(next->size, sizeof(*next->start)); + if (next->start == NULL) { + WebPSafeFree(next); + return 0; + } + next->curr_table = next->start; + next->next = NULL; + // Point to the new segment. + root_table->curr_segment->next = next; + root_table->curr_segment = next; + } + if (code_lengths_size <= SORTED_SIZE_CUTOFF) { // use local stack-allocated array. uint16_t sorted[SORTED_SIZE_CUTOFF]; - total_size = BuildHuffmanTable(root_table, root_bits, - code_lengths, code_lengths_size, sorted); - } else { // rare case. Use heap allocation. + BuildHuffmanTable(root_table->curr_segment->curr_table, root_bits, + code_lengths, code_lengths_size, sorted); + } else { // rare case. Use heap allocation. uint16_t* const sorted = (uint16_t*)WebPSafeMalloc(code_lengths_size, sizeof(*sorted)); if (sorted == NULL) return 0; - total_size = BuildHuffmanTable(root_table, root_bits, - code_lengths, code_lengths_size, sorted); + BuildHuffmanTable(root_table->curr_segment->curr_table, root_bits, + code_lengths, code_lengths_size, sorted); WebPSafeFree(sorted); } return total_size; } + +int VP8LHuffmanTablesAllocate(int size, HuffmanTables* huffman_tables) { + // Have 'segment' point to the first segment for now, 'root'. + HuffmanTablesSegment* const root = &huffman_tables->root; + huffman_tables->curr_segment = root; + // Allocate root. + root->start = (HuffmanCode*)WebPSafeMalloc(size, sizeof(*root->start)); + if (root->start == NULL) return 0; + root->curr_table = root->start; + root->next = NULL; + root->size = size; + return 1; +} + +void VP8LHuffmanTablesDeallocate(HuffmanTables* const huffman_tables) { + HuffmanTablesSegment *current, *next; + if (huffman_tables == NULL) return; + // Free the root node. + current = &huffman_tables->root; + next = current->next; + WebPSafeFree(current->start); + current->start = NULL; + current->next = NULL; + current = next; + // Free the following nodes. + while (current != NULL) { + next = current->next; + WebPSafeFree(current->start); + WebPSafeFree(current); + current = next; + } +} diff --git a/3rdparty/libwebp/src/utils/huffman_utils.h b/3rdparty/libwebp/src/utils/huffman_utils.h index 13b7ad1ac4..98415c5328 100644 --- a/3rdparty/libwebp/src/utils/huffman_utils.h +++ b/3rdparty/libwebp/src/utils/huffman_utils.h @@ -43,6 +43,29 @@ typedef struct { // or non-literal symbol otherwise } HuffmanCode32; +// Contiguous memory segment of HuffmanCodes. +typedef struct HuffmanTablesSegment { + HuffmanCode* start; + // Pointer to where we are writing into the segment. Starts at 'start' and + // cannot go beyond 'start' + 'size'. + HuffmanCode* curr_table; + // Pointer to the next segment in the chain. + struct HuffmanTablesSegment* next; + int size; +} HuffmanTablesSegment; + +// Chained memory segments of HuffmanCodes. +typedef struct HuffmanTables { + HuffmanTablesSegment root; + // Currently processed segment. At first, this is 'root'. + HuffmanTablesSegment* curr_segment; +} HuffmanTables; + +// Allocates a HuffmanTables with 'size' contiguous HuffmanCodes. Returns 0 on +// memory allocation error, 1 otherwise. +int VP8LHuffmanTablesAllocate(int size, HuffmanTables* huffman_tables); +void VP8LHuffmanTablesDeallocate(HuffmanTables* const huffman_tables); + #define HUFFMAN_PACKED_BITS 6 #define HUFFMAN_PACKED_TABLE_SIZE (1u << HUFFMAN_PACKED_BITS) @@ -78,9 +101,7 @@ void VP8LHtreeGroupsFree(HTreeGroup* const htree_groups); // the huffman table. // Returns built table size or 0 in case of error (invalid tree or // memory error). -// If root_table is NULL, it returns 0 if a lookup cannot be built, something -// > 0 otherwise (but not the table size). -int VP8LBuildHuffmanTable(HuffmanCode* const root_table, int root_bits, +int VP8LBuildHuffmanTable(HuffmanTables* const root_table, int root_bits, const int code_lengths[], int code_lengths_size); #ifdef __cplusplus diff --git a/3rdparty/libwebp/src/utils/palette.c b/3rdparty/libwebp/src/utils/palette.c new file mode 100644 index 0000000000..515da21019 --- /dev/null +++ b/3rdparty/libwebp/src/utils/palette.c @@ -0,0 +1,402 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for palette analysis. +// +// Author: Vincent Rabaud (vrabaud@google.com) + +#include "src/utils/palette.h" + +#include +#include + +#include "src/dsp/lossless_common.h" +#include "src/utils/color_cache_utils.h" +#include "src/utils/utils.h" +#include "src/webp/encode.h" +#include "src/webp/format_constants.h" + +// ----------------------------------------------------------------------------- + +// Palette reordering for smaller sum of deltas (and for smaller storage). + +static int PaletteCompareColorsForQsort(const void* p1, const void* p2) { + const uint32_t a = WebPMemToUint32((uint8_t*)p1); + const uint32_t b = WebPMemToUint32((uint8_t*)p2); + assert(a != b); + return (a < b) ? -1 : 1; +} + +static WEBP_INLINE uint32_t PaletteComponentDistance(uint32_t v) { + return (v <= 128) ? v : (256 - v); +} + +// Computes a value that is related to the entropy created by the +// palette entry diff. +// +// Note that the last & 0xff is a no-operation in the next statement, but +// removed by most compilers and is here only for regularity of the code. +static WEBP_INLINE uint32_t PaletteColorDistance(uint32_t col1, uint32_t col2) { + const uint32_t diff = VP8LSubPixels(col1, col2); + const int kMoreWeightForRGBThanForAlpha = 9; + uint32_t score; + score = PaletteComponentDistance((diff >> 0) & 0xff); + score += PaletteComponentDistance((diff >> 8) & 0xff); + score += PaletteComponentDistance((diff >> 16) & 0xff); + score *= kMoreWeightForRGBThanForAlpha; + score += PaletteComponentDistance((diff >> 24) & 0xff); + return score; +} + +static WEBP_INLINE void SwapColor(uint32_t* const col1, uint32_t* const col2) { + const uint32_t tmp = *col1; + *col1 = *col2; + *col2 = tmp; +} + +int SearchColorNoIdx(const uint32_t sorted[], uint32_t color, int num_colors) { + int low = 0, hi = num_colors; + if (sorted[low] == color) return low; // loop invariant: sorted[low] != color + while (1) { + const int mid = (low + hi) >> 1; + if (sorted[mid] == color) { + return mid; + } else if (sorted[mid] < color) { + low = mid; + } else { + hi = mid; + } + } + assert(0); + return 0; +} + +void PrepareMapToPalette(const uint32_t palette[], uint32_t num_colors, + uint32_t sorted[], uint32_t idx_map[]) { + uint32_t i; + memcpy(sorted, palette, num_colors * sizeof(*sorted)); + qsort(sorted, num_colors, sizeof(*sorted), PaletteCompareColorsForQsort); + for (i = 0; i < num_colors; ++i) { + idx_map[SearchColorNoIdx(sorted, palette[i], num_colors)] = i; + } +} + +//------------------------------------------------------------------------------ + +#define COLOR_HASH_SIZE (MAX_PALETTE_SIZE * 4) +#define COLOR_HASH_RIGHT_SHIFT 22 // 32 - log2(COLOR_HASH_SIZE). + +int GetColorPalette(const WebPPicture* const pic, uint32_t* const palette) { + int i; + int x, y; + int num_colors = 0; + uint8_t in_use[COLOR_HASH_SIZE] = {0}; + uint32_t colors[COLOR_HASH_SIZE] = {0}; + const uint32_t* argb = pic->argb; + const int width = pic->width; + const int height = pic->height; + uint32_t last_pix = ~argb[0]; // so we're sure that last_pix != argb[0] + assert(pic != NULL); + assert(pic->use_argb); + + for (y = 0; y < height; ++y) { + for (x = 0; x < width; ++x) { + int key; + if (argb[x] == last_pix) { + continue; + } + last_pix = argb[x]; + key = VP8LHashPix(last_pix, COLOR_HASH_RIGHT_SHIFT); + while (1) { + if (!in_use[key]) { + colors[key] = last_pix; + in_use[key] = 1; + ++num_colors; + if (num_colors > MAX_PALETTE_SIZE) { + return MAX_PALETTE_SIZE + 1; // Exact count not needed. + } + break; + } else if (colors[key] == last_pix) { + break; // The color is already there. + } else { + // Some other color sits here, so do linear conflict resolution. + ++key; + key &= (COLOR_HASH_SIZE - 1); // Key mask. + } + } + } + argb += pic->argb_stride; + } + + if (palette != NULL) { // Fill the colors into palette. + num_colors = 0; + for (i = 0; i < COLOR_HASH_SIZE; ++i) { + if (in_use[i]) { + palette[num_colors] = colors[i]; + ++num_colors; + } + } + qsort(palette, num_colors, sizeof(*palette), PaletteCompareColorsForQsort); + } + return num_colors; +} + +#undef COLOR_HASH_SIZE +#undef COLOR_HASH_RIGHT_SHIFT + +// ----------------------------------------------------------------------------- + +// The palette has been sorted by alpha. This function checks if the other +// components of the palette have a monotonic development with regards to +// position in the palette. If all have monotonic development, there is +// no benefit to re-organize them greedily. A monotonic development +// would be spotted in green-only situations (like lossy alpha) or gray-scale +// images. +static int PaletteHasNonMonotonousDeltas(const uint32_t* const palette, + int num_colors) { + uint32_t predict = 0x000000; + int i; + uint8_t sign_found = 0x00; + for (i = 0; i < num_colors; ++i) { + const uint32_t diff = VP8LSubPixels(palette[i], predict); + const uint8_t rd = (diff >> 16) & 0xff; + const uint8_t gd = (diff >> 8) & 0xff; + const uint8_t bd = (diff >> 0) & 0xff; + if (rd != 0x00) { + sign_found |= (rd < 0x80) ? 1 : 2; + } + if (gd != 0x00) { + sign_found |= (gd < 0x80) ? 8 : 16; + } + if (bd != 0x00) { + sign_found |= (bd < 0x80) ? 64 : 128; + } + predict = palette[i]; + } + return (sign_found & (sign_found << 1)) != 0; // two consequent signs. +} + +static void PaletteSortMinimizeDeltas(const uint32_t* const palette_sorted, + int num_colors, uint32_t* const palette) { + uint32_t predict = 0x00000000; + int i, k; + memcpy(palette, palette_sorted, num_colors * sizeof(*palette)); + if (!PaletteHasNonMonotonousDeltas(palette_sorted, num_colors)) return; + // Find greedily always the closest color of the predicted color to minimize + // deltas in the palette. This reduces storage needs since the + // palette is stored with delta encoding. + for (i = 0; i < num_colors; ++i) { + int best_ix = i; + uint32_t best_score = ~0U; + for (k = i; k < num_colors; ++k) { + const uint32_t cur_score = PaletteColorDistance(palette[k], predict); + if (best_score > cur_score) { + best_score = cur_score; + best_ix = k; + } + } + SwapColor(&palette[best_ix], &palette[i]); + predict = palette[i]; + } +} + +// ----------------------------------------------------------------------------- +// Modified Zeng method from "A Survey on Palette Reordering +// Methods for Improving the Compression of Color-Indexed Images" by Armando J. +// Pinho and Antonio J. R. Neves. + +// Finds the biggest cooccurrence in the matrix. +static void CoOccurrenceFindMax(const uint32_t* const cooccurrence, + uint32_t num_colors, uint8_t* const c1, + uint8_t* const c2) { + // Find the index that is most frequently located adjacent to other + // (different) indexes. + uint32_t best_sum = 0u; + uint32_t i, j, best_cooccurrence; + *c1 = 0u; + for (i = 0; i < num_colors; ++i) { + uint32_t sum = 0; + for (j = 0; j < num_colors; ++j) sum += cooccurrence[i * num_colors + j]; + if (sum > best_sum) { + best_sum = sum; + *c1 = i; + } + } + // Find the index that is most frequently found adjacent to *c1. + *c2 = 0u; + best_cooccurrence = 0u; + for (i = 0; i < num_colors; ++i) { + if (cooccurrence[*c1 * num_colors + i] > best_cooccurrence) { + best_cooccurrence = cooccurrence[*c1 * num_colors + i]; + *c2 = i; + } + } + assert(*c1 != *c2); +} + +// Builds the cooccurrence matrix +static int CoOccurrenceBuild(const WebPPicture* const pic, + const uint32_t* const palette, uint32_t num_colors, + uint32_t* cooccurrence) { + uint32_t *lines, *line_top, *line_current, *line_tmp; + int x, y; + const uint32_t* src = pic->argb; + uint32_t prev_pix = ~src[0]; + uint32_t prev_idx = 0u; + uint32_t idx_map[MAX_PALETTE_SIZE] = {0}; + uint32_t palette_sorted[MAX_PALETTE_SIZE]; + lines = (uint32_t*)WebPSafeMalloc(2 * pic->width, sizeof(*lines)); + if (lines == NULL) { + return 0; + } + line_top = &lines[0]; + line_current = &lines[pic->width]; + PrepareMapToPalette(palette, num_colors, palette_sorted, idx_map); + for (y = 0; y < pic->height; ++y) { + for (x = 0; x < pic->width; ++x) { + const uint32_t pix = src[x]; + if (pix != prev_pix) { + prev_idx = idx_map[SearchColorNoIdx(palette_sorted, pix, num_colors)]; + prev_pix = pix; + } + line_current[x] = prev_idx; + // 4-connectivity is what works best as mentioned in "On the relation + // between Memon's and the modified Zeng's palette reordering methods". + if (x > 0 && prev_idx != line_current[x - 1]) { + const uint32_t left_idx = line_current[x - 1]; + ++cooccurrence[prev_idx * num_colors + left_idx]; + ++cooccurrence[left_idx * num_colors + prev_idx]; + } + if (y > 0 && prev_idx != line_top[x]) { + const uint32_t top_idx = line_top[x]; + ++cooccurrence[prev_idx * num_colors + top_idx]; + ++cooccurrence[top_idx * num_colors + prev_idx]; + } + } + line_tmp = line_top; + line_top = line_current; + line_current = line_tmp; + src += pic->argb_stride; + } + WebPSafeFree(lines); + return 1; +} + +struct Sum { + uint8_t index; + uint32_t sum; +}; + +static int PaletteSortModifiedZeng(const WebPPicture* const pic, + const uint32_t* const palette_in, + uint32_t num_colors, + uint32_t* const palette) { + uint32_t i, j, ind; + uint8_t remapping[MAX_PALETTE_SIZE]; + uint32_t* cooccurrence; + struct Sum sums[MAX_PALETTE_SIZE]; + uint32_t first, last; + uint32_t num_sums; + // TODO(vrabaud) check whether one color images should use palette or not. + if (num_colors <= 1) return 1; + // Build the co-occurrence matrix. + cooccurrence = + (uint32_t*)WebPSafeCalloc(num_colors * num_colors, sizeof(*cooccurrence)); + if (cooccurrence == NULL) { + return 0; + } + if (!CoOccurrenceBuild(pic, palette_in, num_colors, cooccurrence)) { + WebPSafeFree(cooccurrence); + return 0; + } + + // Initialize the mapping list with the two best indices. + CoOccurrenceFindMax(cooccurrence, num_colors, &remapping[0], &remapping[1]); + + // We need to append and prepend to the list of remapping. To this end, we + // actually define the next start/end of the list as indices in a vector (with + // a wrap around when the end is reached). + first = 0; + last = 1; + num_sums = num_colors - 2; // -2 because we know the first two values + if (num_sums > 0) { + // Initialize the sums with the first two remappings and find the best one + struct Sum* best_sum = &sums[0]; + best_sum->index = 0u; + best_sum->sum = 0u; + for (i = 0, j = 0; i < num_colors; ++i) { + if (i == remapping[0] || i == remapping[1]) continue; + sums[j].index = i; + sums[j].sum = cooccurrence[i * num_colors + remapping[0]] + + cooccurrence[i * num_colors + remapping[1]]; + if (sums[j].sum > best_sum->sum) best_sum = &sums[j]; + ++j; + } + + while (num_sums > 0) { + const uint8_t best_index = best_sum->index; + // Compute delta to know if we need to prepend or append the best index. + int32_t delta = 0; + const int32_t n = num_colors - num_sums; + for (ind = first, j = 0; (ind + j) % num_colors != last + 1; ++j) { + const uint16_t l_j = remapping[(ind + j) % num_colors]; + delta += (n - 1 - 2 * (int32_t)j) * + (int32_t)cooccurrence[best_index * num_colors + l_j]; + } + if (delta > 0) { + first = (first == 0) ? num_colors - 1 : first - 1; + remapping[first] = best_index; + } else { + ++last; + remapping[last] = best_index; + } + // Remove best_sum from sums. + *best_sum = sums[num_sums - 1]; + --num_sums; + // Update all the sums and find the best one. + best_sum = &sums[0]; + for (i = 0; i < num_sums; ++i) { + sums[i].sum += cooccurrence[best_index * num_colors + sums[i].index]; + if (sums[i].sum > best_sum->sum) best_sum = &sums[i]; + } + } + } + assert((last + 1) % num_colors == first); + WebPSafeFree(cooccurrence); + + // Re-map the palette. + for (i = 0; i < num_colors; ++i) { + palette[i] = palette_in[remapping[(first + i) % num_colors]]; + } + return 1; +} + +// ----------------------------------------------------------------------------- + +int PaletteSort(PaletteSorting method, const struct WebPPicture* const pic, + const uint32_t* const palette_sorted, uint32_t num_colors, + uint32_t* const palette) { + switch (method) { + case kSortedDefault: + // Nothing to do, we have already sorted the palette. + memcpy(palette, palette_sorted, num_colors * sizeof(*palette)); + return 1; + case kMinimizeDelta: + PaletteSortMinimizeDeltas(palette_sorted, num_colors, palette); + return 1; + case kModifiedZeng: + return PaletteSortModifiedZeng(pic, palette_sorted, num_colors, palette); + case kUnusedPalette: + case kPaletteSortingNum: + break; + } + + assert(0); + return 0; +} diff --git a/3rdparty/libwebp/src/utils/palette.h b/3rdparty/libwebp/src/utils/palette.h new file mode 100644 index 0000000000..34479e463f --- /dev/null +++ b/3rdparty/libwebp/src/utils/palette.h @@ -0,0 +1,60 @@ +// Copyright 2023 Google Inc. All Rights Reserved. +// +// Use of this source code is governed by a BSD-style license +// that can be found in the COPYING file in the root of the source +// tree. An additional intellectual property rights grant can be found +// in the file PATENTS. All contributing project authors may +// be found in the AUTHORS file in the root of the source tree. +// ----------------------------------------------------------------------------- +// +// Utilities for palette analysis. +// +// Author: Vincent Rabaud (vrabaud@google.com) + +#ifndef WEBP_UTILS_PALETTE_H_ +#define WEBP_UTILS_PALETTE_H_ + +#include "src/webp/types.h" + +struct WebPPicture; + +// The different ways a palette can be sorted. +typedef enum PaletteSorting { + kSortedDefault = 0, + // Sorts by minimizing L1 deltas between consecutive colors, giving more + // weight to RGB colors. + kMinimizeDelta = 1, + // Implements the modified Zeng method from "A Survey on Palette Reordering + // Methods for Improving the Compression of Color-Indexed Images" by Armando + // J. Pinho and Antonio J. R. Neves. + kModifiedZeng = 2, + kUnusedPalette = 3, + kPaletteSortingNum = 4 +} PaletteSorting; + +// Returns the index of 'color' in the sorted palette 'sorted' of size +// 'num_colors'. +int SearchColorNoIdx(const uint32_t sorted[], uint32_t color, int num_colors); + +// Sort palette in increasing order and prepare an inverse mapping array. +void PrepareMapToPalette(const uint32_t palette[], uint32_t num_colors, + uint32_t sorted[], uint32_t idx_map[]); + +// Returns count of unique colors in 'pic', assuming pic->use_argb is true. +// If the unique color count is more than MAX_PALETTE_SIZE, returns +// MAX_PALETTE_SIZE+1. +// If 'palette' is not NULL and the number of unique colors is less than or +// equal to MAX_PALETTE_SIZE, also outputs the actual unique colors into +// 'palette' in a sorted order. Note: 'palette' is assumed to be an array +// already allocated with at least MAX_PALETTE_SIZE elements. +int GetColorPalette(const struct WebPPicture* const pic, + uint32_t* const palette); + +// Sorts the palette according to the criterion defined by 'method'. +// 'palette_sorted' is the input palette sorted lexicographically, as done in +// PrepareMapToPalette. Returns 0 on memory allocation error. +int PaletteSort(PaletteSorting method, const struct WebPPicture* const pic, + const uint32_t* const palette_sorted, uint32_t num_colors, + uint32_t* const palette); + +#endif // WEBP_UTILS_PALETTE_H_ diff --git a/3rdparty/libwebp/src/utils/quant_levels_dec_utils.c b/3rdparty/libwebp/src/utils/quant_levels_dec_utils.c index f65b6cdbb6..97e7893704 100644 --- a/3rdparty/libwebp/src/utils/quant_levels_dec_utils.c +++ b/3rdparty/libwebp/src/utils/quant_levels_dec_utils.c @@ -30,7 +30,7 @@ #define DFIX 4 // extra precision for ordered dithering #define DSIZE 4 // dithering size (must be a power of two) -// cf. http://en.wikipedia.org/wiki/Ordered_dithering +// cf. https://en.wikipedia.org/wiki/Ordered_dithering static const uint8_t kOrderedDither[DSIZE][DSIZE] = { { 0, 8, 2, 10 }, // coefficients are in DFIX fixed-point precision { 12, 4, 14, 6 }, diff --git a/3rdparty/libwebp/src/utils/rescaler_utils.c b/3rdparty/libwebp/src/utils/rescaler_utils.c index 4bcae24af5..a0581a14b1 100644 --- a/3rdparty/libwebp/src/utils/rescaler_utils.c +++ b/3rdparty/libwebp/src/utils/rescaler_utils.c @@ -12,66 +12,74 @@ // Author: Skal (pascal.massimino@gmail.com) #include +#include #include #include #include "src/dsp/dsp.h" #include "src/utils/rescaler_utils.h" +#include "src/utils/utils.h" //------------------------------------------------------------------------------ -void WebPRescalerInit(WebPRescaler* const wrk, int src_width, int src_height, - uint8_t* const dst, - int dst_width, int dst_height, int dst_stride, - int num_channels, rescaler_t* const work) { +int WebPRescalerInit(WebPRescaler* const rescaler, + int src_width, int src_height, + uint8_t* const dst, + int dst_width, int dst_height, int dst_stride, + int num_channels, rescaler_t* const work) { const int x_add = src_width, x_sub = dst_width; const int y_add = src_height, y_sub = dst_height; - wrk->x_expand = (src_width < dst_width); - wrk->y_expand = (src_height < dst_height); - wrk->src_width = src_width; - wrk->src_height = src_height; - wrk->dst_width = dst_width; - wrk->dst_height = dst_height; - wrk->src_y = 0; - wrk->dst_y = 0; - wrk->dst = dst; - wrk->dst_stride = dst_stride; - wrk->num_channels = num_channels; + const uint64_t total_size = 2ull * dst_width * num_channels * sizeof(*work); + if (!CheckSizeOverflow(total_size)) return 0; + + rescaler->x_expand = (src_width < dst_width); + rescaler->y_expand = (src_height < dst_height); + rescaler->src_width = src_width; + rescaler->src_height = src_height; + rescaler->dst_width = dst_width; + rescaler->dst_height = dst_height; + rescaler->src_y = 0; + rescaler->dst_y = 0; + rescaler->dst = dst; + rescaler->dst_stride = dst_stride; + rescaler->num_channels = num_channels; // for 'x_expand', we use bilinear interpolation - wrk->x_add = wrk->x_expand ? (x_sub - 1) : x_add; - wrk->x_sub = wrk->x_expand ? (x_add - 1) : x_sub; - if (!wrk->x_expand) { // fx_scale is not used otherwise - wrk->fx_scale = WEBP_RESCALER_FRAC(1, wrk->x_sub); + rescaler->x_add = rescaler->x_expand ? (x_sub - 1) : x_add; + rescaler->x_sub = rescaler->x_expand ? (x_add - 1) : x_sub; + if (!rescaler->x_expand) { // fx_scale is not used otherwise + rescaler->fx_scale = WEBP_RESCALER_FRAC(1, rescaler->x_sub); } // vertical scaling parameters - wrk->y_add = wrk->y_expand ? y_add - 1 : y_add; - wrk->y_sub = wrk->y_expand ? y_sub - 1 : y_sub; - wrk->y_accum = wrk->y_expand ? wrk->y_sub : wrk->y_add; - if (!wrk->y_expand) { + rescaler->y_add = rescaler->y_expand ? y_add - 1 : y_add; + rescaler->y_sub = rescaler->y_expand ? y_sub - 1 : y_sub; + rescaler->y_accum = rescaler->y_expand ? rescaler->y_sub : rescaler->y_add; + if (!rescaler->y_expand) { // This is WEBP_RESCALER_FRAC(dst_height, x_add * y_add) without the cast. - // Its value is <= WEBP_RESCALER_ONE, because dst_height <= wrk->y_add, and - // wrk->x_add >= 1; - const uint64_t ratio = - (uint64_t)dst_height * WEBP_RESCALER_ONE / (wrk->x_add * wrk->y_add); + // Its value is <= WEBP_RESCALER_ONE, because dst_height <= rescaler->y_add + // and rescaler->x_add >= 1; + const uint64_t num = (uint64_t)dst_height * WEBP_RESCALER_ONE; + const uint64_t den = (uint64_t)rescaler->x_add * rescaler->y_add; + const uint64_t ratio = num / den; if (ratio != (uint32_t)ratio) { // When ratio == WEBP_RESCALER_ONE, we can't represent the ratio with the // current fixed-point precision. This happens when src_height == - // wrk->y_add (which == src_height), and wrk->x_add == 1. + // rescaler->y_add (which == src_height), and rescaler->x_add == 1. // => We special-case fxy_scale = 0, in WebPRescalerExportRow(). - wrk->fxy_scale = 0; + rescaler->fxy_scale = 0; } else { - wrk->fxy_scale = (uint32_t)ratio; + rescaler->fxy_scale = (uint32_t)ratio; } - wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->y_sub); + rescaler->fy_scale = WEBP_RESCALER_FRAC(1, rescaler->y_sub); } else { - wrk->fy_scale = WEBP_RESCALER_FRAC(1, wrk->x_add); - // wrk->fxy_scale is unused here. + rescaler->fy_scale = WEBP_RESCALER_FRAC(1, rescaler->x_add); + // rescaler->fxy_scale is unused here. } - wrk->irow = work; - wrk->frow = work + num_channels * dst_width; - memset(work, 0, 2 * dst_width * num_channels * sizeof(*work)); + rescaler->irow = work; + rescaler->frow = work + num_channels * dst_width; + memset(work, 0, (size_t)total_size); WebPRescalerDspInit(); + return 1; } int WebPRescalerGetScaledDimensions(int src_width, int src_height, @@ -82,6 +90,7 @@ int WebPRescalerGetScaledDimensions(int src_width, int src_height, { int width = *scaled_width; int height = *scaled_height; + const int max_size = INT_MAX / 2; // if width is unspecified, scale original proportionally to height ratio. if (width == 0 && src_height > 0) { @@ -94,7 +103,7 @@ int WebPRescalerGetScaledDimensions(int src_width, int src_height, (int)(((uint64_t)src_height * width + src_width - 1) / src_width); } // Check if the overall dimensions still make sense. - if (width <= 0 || height <= 0) { + if (width <= 0 || height <= 0 || width > max_size || height > max_size) { return 0; } @@ -107,31 +116,34 @@ int WebPRescalerGetScaledDimensions(int src_width, int src_height, //------------------------------------------------------------------------------ // all-in-one calls -int WebPRescaleNeededLines(const WebPRescaler* const wrk, int max_num_lines) { - const int num_lines = (wrk->y_accum + wrk->y_sub - 1) / wrk->y_sub; +int WebPRescaleNeededLines(const WebPRescaler* const rescaler, + int max_num_lines) { + const int num_lines = + (rescaler->y_accum + rescaler->y_sub - 1) / rescaler->y_sub; return (num_lines > max_num_lines) ? max_num_lines : num_lines; } -int WebPRescalerImport(WebPRescaler* const wrk, int num_lines, +int WebPRescalerImport(WebPRescaler* const rescaler, int num_lines, const uint8_t* src, int src_stride) { int total_imported = 0; - while (total_imported < num_lines && !WebPRescalerHasPendingOutput(wrk)) { - if (wrk->y_expand) { - rescaler_t* const tmp = wrk->irow; - wrk->irow = wrk->frow; - wrk->frow = tmp; + while (total_imported < num_lines && + !WebPRescalerHasPendingOutput(rescaler)) { + if (rescaler->y_expand) { + rescaler_t* const tmp = rescaler->irow; + rescaler->irow = rescaler->frow; + rescaler->frow = tmp; } - WebPRescalerImportRow(wrk, src); - if (!wrk->y_expand) { // Accumulate the contribution of the new row. + WebPRescalerImportRow(rescaler, src); + if (!rescaler->y_expand) { // Accumulate the contribution of the new row. int x; - for (x = 0; x < wrk->num_channels * wrk->dst_width; ++x) { - wrk->irow[x] += wrk->frow[x]; + for (x = 0; x < rescaler->num_channels * rescaler->dst_width; ++x) { + rescaler->irow[x] += rescaler->frow[x]; } } - ++wrk->src_y; + ++rescaler->src_y; src += src_stride; ++total_imported; - wrk->y_accum -= wrk->y_sub; + rescaler->y_accum -= rescaler->y_sub; } return total_imported; } diff --git a/3rdparty/libwebp/src/utils/rescaler_utils.h b/3rdparty/libwebp/src/utils/rescaler_utils.h index ca41e42c4a..ef201ef86c 100644 --- a/3rdparty/libwebp/src/utils/rescaler_utils.h +++ b/3rdparty/libwebp/src/utils/rescaler_utils.h @@ -47,12 +47,13 @@ struct WebPRescaler { }; // Initialize a rescaler given scratch area 'work' and dimensions of src & dst. -void WebPRescalerInit(WebPRescaler* const rescaler, - int src_width, int src_height, - uint8_t* const dst, - int dst_width, int dst_height, int dst_stride, - int num_channels, - rescaler_t* const work); +// Returns false in case of error. +int WebPRescalerInit(WebPRescaler* const rescaler, + int src_width, int src_height, + uint8_t* const dst, + int dst_width, int dst_height, int dst_stride, + int num_channels, + rescaler_t* const work); // If either 'scaled_width' or 'scaled_height' (but not both) is 0 the value // will be calculated preserving the aspect ratio, otherwise the values are diff --git a/3rdparty/libwebp/src/utils/utils.c b/3rdparty/libwebp/src/utils/utils.c index 6080e19e21..408ce88f67 100644 --- a/3rdparty/libwebp/src/utils/utils.c +++ b/3rdparty/libwebp/src/utils/utils.c @@ -11,19 +11,19 @@ // // Author: Skal (pascal.massimino@gmail.com) +#include "src/utils/utils.h" + #include #include // for memcpy() -#include "src/webp/decode.h" + +#include "src/utils/palette.h" #include "src/webp/encode.h" -#include "src/webp/format_constants.h" // for MAX_PALETTE_SIZE -#include "src/utils/color_cache_utils.h" -#include "src/utils/utils.h" // If PRINT_MEM_INFO is defined, extra info (like total memory used, number of // alloc/free etc) is printed. For debugging/tuning purpose only (it's slow, // and not multi-thread safe!). // An interesting alternative is valgrind's 'massif' tool: -// http://valgrind.org/docs/manual/ms-manual.html +// https://valgrind.org/docs/manual/ms-manual.html // Here is an example command line: /* valgrind --tool=massif --massif-out-file=massif.out \ --stacks=yes --alloc-fn=WebPSafeMalloc --alloc-fn=WebPSafeCalloc @@ -101,6 +101,9 @@ static void Increment(int* const v) { #if defined(MALLOC_LIMIT) { const char* const malloc_limit_str = getenv("MALLOC_LIMIT"); +#if MALLOC_LIMIT > 1 + mem_limit = (size_t)MALLOC_LIMIT; +#endif if (malloc_limit_str != NULL) { mem_limit = atoi(malloc_limit_str); } @@ -169,16 +172,16 @@ static int CheckSizeArgumentsOverflow(uint64_t nmemb, size_t size) { const uint64_t total_size = nmemb * size; if (nmemb == 0) return 1; if ((uint64_t)size > WEBP_MAX_ALLOCABLE_MEMORY / nmemb) return 0; - if (total_size != (size_t)total_size) return 0; + if (!CheckSizeOverflow(total_size)) return 0; #if defined(PRINT_MEM_INFO) && defined(MALLOC_FAIL_AT) if (countdown_to_fail > 0 && --countdown_to_fail == 0) { return 0; // fake fail! } #endif -#if defined(MALLOC_LIMIT) +#if defined(PRINT_MEM_INFO) && defined(MALLOC_LIMIT) if (mem_limit > 0) { const uint64_t new_total_mem = (uint64_t)total_mem + total_size; - if (new_total_mem != (size_t)new_total_mem || + if (!CheckSizeOverflow(new_total_mem) || new_total_mem > mem_limit) { return 0; // fake fail! } @@ -249,66 +252,10 @@ void WebPCopyPixels(const WebPPicture* const src, WebPPicture* const dst) { //------------------------------------------------------------------------------ -#define COLOR_HASH_SIZE (MAX_PALETTE_SIZE * 4) -#define COLOR_HASH_RIGHT_SHIFT 22 // 32 - log2(COLOR_HASH_SIZE). - int WebPGetColorPalette(const WebPPicture* const pic, uint32_t* const palette) { - int i; - int x, y; - int num_colors = 0; - uint8_t in_use[COLOR_HASH_SIZE] = { 0 }; - uint32_t colors[COLOR_HASH_SIZE]; - const uint32_t* argb = pic->argb; - const int width = pic->width; - const int height = pic->height; - uint32_t last_pix = ~argb[0]; // so we're sure that last_pix != argb[0] - assert(pic != NULL); - assert(pic->use_argb); - - for (y = 0; y < height; ++y) { - for (x = 0; x < width; ++x) { - int key; - if (argb[x] == last_pix) { - continue; - } - last_pix = argb[x]; - key = VP8LHashPix(last_pix, COLOR_HASH_RIGHT_SHIFT); - while (1) { - if (!in_use[key]) { - colors[key] = last_pix; - in_use[key] = 1; - ++num_colors; - if (num_colors > MAX_PALETTE_SIZE) { - return MAX_PALETTE_SIZE + 1; // Exact count not needed. - } - break; - } else if (colors[key] == last_pix) { - break; // The color is already there. - } else { - // Some other color sits here, so do linear conflict resolution. - ++key; - key &= (COLOR_HASH_SIZE - 1); // Key mask. - } - } - } - argb += pic->argb_stride; - } - - if (palette != NULL) { // Fill the colors into palette. - num_colors = 0; - for (i = 0; i < COLOR_HASH_SIZE; ++i) { - if (in_use[i]) { - palette[num_colors] = colors[i]; - ++num_colors; - } - } - } - return num_colors; + return GetColorPalette(pic, palette); } -#undef COLOR_HASH_SIZE -#undef COLOR_HASH_RIGHT_SHIFT - //------------------------------------------------------------------------------ #if defined(WEBP_NEED_LOG_TABLE_8BIT) diff --git a/3rdparty/libwebp/src/utils/utils.h b/3rdparty/libwebp/src/utils/utils.h index 2a3ec92678..b2241fbf9b 100644 --- a/3rdparty/libwebp/src/utils/utils.h +++ b/3rdparty/libwebp/src/utils/utils.h @@ -20,9 +20,7 @@ #endif #include -#include -#include "src/dsp/dsp.h" #include "src/webp/types.h" #ifdef __cplusplus @@ -42,6 +40,10 @@ extern "C" { #endif #endif // WEBP_MAX_ALLOCABLE_MEMORY +static WEBP_INLINE int CheckSizeOverflow(uint64_t size) { + return size == (size_t)size; +} + // size-checking safe malloc/calloc: verify that the requested size is not too // large, or return NULL. You don't need to call these for constructs like // malloc(sizeof(foo)), but only if there's picture-dependent size involved @@ -60,7 +62,8 @@ WEBP_EXTERN void WebPSafeFree(void* const ptr); // Alignment #define WEBP_ALIGN_CST 31 -#define WEBP_ALIGN(PTR) (((uintptr_t)(PTR) + WEBP_ALIGN_CST) & ~WEBP_ALIGN_CST) +#define WEBP_ALIGN(PTR) (((uintptr_t)(PTR) + WEBP_ALIGN_CST) & \ + ~(uintptr_t)WEBP_ALIGN_CST) #include // memcpy() is the safe way of moving potentially unaligned 32b memory. @@ -69,10 +72,19 @@ static WEBP_INLINE uint32_t WebPMemToUint32(const uint8_t* const ptr) { memcpy(&A, ptr, sizeof(A)); return A; } + +static WEBP_INLINE int32_t WebPMemToInt32(const uint8_t* const ptr) { + return (int32_t)WebPMemToUint32(ptr); +} + static WEBP_INLINE void WebPUint32ToMem(uint8_t* const ptr, uint32_t val) { memcpy(ptr, &val, sizeof(val)); } +static WEBP_INLINE void WebPInt32ToMem(uint8_t* const ptr, int val) { + WebPUint32ToMem(ptr, (uint32_t)val); +} + //------------------------------------------------------------------------------ // Reading/writing data. @@ -107,24 +119,33 @@ static WEBP_INLINE void PutLE32(uint8_t* const data, uint32_t val) { PutLE16(data + 2, (int)(val >> 16)); } -// Returns (int)floor(log2(n)). n must be > 0. // use GNU builtins where available. #if defined(__GNUC__) && \ ((__GNUC__ == 3 && __GNUC_MINOR__ >= 4) || __GNUC__ >= 4) +// Returns (int)floor(log2(n)). n must be > 0. static WEBP_INLINE int BitsLog2Floor(uint32_t n) { return 31 ^ __builtin_clz(n); } +// counts the number of trailing zero +static WEBP_INLINE int BitsCtz(uint32_t n) { return __builtin_ctz(n); } #elif defined(_MSC_VER) && _MSC_VER > 1310 && \ (defined(_M_X64) || defined(_M_IX86)) #include #pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanForward) static WEBP_INLINE int BitsLog2Floor(uint32_t n) { - unsigned long first_set_bit; + unsigned long first_set_bit; // NOLINT (runtime/int) _BitScanReverse(&first_set_bit, n); return first_set_bit; } -#else // default: use the C-version. +static WEBP_INLINE int BitsCtz(uint32_t n) { + unsigned long first_set_bit; // NOLINT (runtime/int) + _BitScanForward(&first_set_bit, n); + return first_set_bit; +} +#else // default: use the (slow) C-version. +#define WEBP_HAVE_SLOW_CLZ_CTZ // signal that the Clz/Ctz function are slow // Returns 31 ^ clz(n) = log2(n). This is the default C-implementation, either // based on table or not. Can be used as fallback if clz() is not available. #define WEBP_NEED_LOG_TABLE_8BIT @@ -139,6 +160,15 @@ static WEBP_INLINE int WebPLog2FloorC(uint32_t n) { } static WEBP_INLINE int BitsLog2Floor(uint32_t n) { return WebPLog2FloorC(n); } + +static WEBP_INLINE int BitsCtz(uint32_t n) { + int i; + for (i = 0; i < 32; ++i, n >>= 1) { + if (n & 1) return i; + } + return 32; +} + #endif //------------------------------------------------------------------------------ @@ -166,6 +196,7 @@ WEBP_EXTERN void WebPCopyPixels(const struct WebPPicture* const src, // MAX_PALETTE_SIZE, also outputs the actual unique colors into 'palette'. // Note: 'palette' is assumed to be an array already allocated with at least // MAX_PALETTE_SIZE elements. +// TODO(vrabaud) remove whenever we can break the ABI. WEBP_EXTERN int WebPGetColorPalette(const struct WebPPicture* const pic, uint32_t* const palette); diff --git a/3rdparty/libwebp/src/webp/decode.h b/3rdparty/libwebp/src/webp/decode.h index 44fcd64a84..9d968061d1 100644 --- a/3rdparty/libwebp/src/webp/decode.h +++ b/3rdparty/libwebp/src/webp/decode.h @@ -81,11 +81,12 @@ WEBP_EXTERN uint8_t* WebPDecodeBGR(const uint8_t* data, size_t data_size, // returned is the Y samples buffer. Upon return, *u and *v will point to // the U and V chroma data. These U and V buffers need NOT be passed to // WebPFree(), unlike the returned Y luma one. The dimension of the U and V -// planes are both (*width + 1) / 2 and (*height + 1)/ 2. +// planes are both (*width + 1) / 2 and (*height + 1) / 2. // Upon return, the Y buffer has a stride returned as '*stride', while U and V // have a common stride returned as '*uv_stride'. -// Return NULL in case of error. -// (*) Also named Y'CbCr. See: http://en.wikipedia.org/wiki/YCbCr +// 'width' and 'height' may be NULL, the other pointers must not be. +// Returns NULL in case of error. +// (*) Also named Y'CbCr. See: https://en.wikipedia.org/wiki/YCbCr WEBP_EXTERN uint8_t* WebPDecodeYUV(const uint8_t* data, size_t data_size, int* width, int* height, uint8_t** u, uint8_t** v, @@ -250,23 +251,24 @@ typedef enum VP8StatusCode { // WebPIDecoder object. This object can be left in a SUSPENDED state if the // picture is only partially decoded, pending additional input. // Code example: -// -// WebPInitDecBuffer(&output_buffer); -// output_buffer.colorspace = mode; -// ... -// WebPIDecoder* idec = WebPINewDecoder(&output_buffer); -// while (additional_data_is_available) { -// // ... (get additional data in some new_data[] buffer) -// status = WebPIAppend(idec, new_data, new_data_size); -// if (status != VP8_STATUS_OK && status != VP8_STATUS_SUSPENDED) { -// break; // an error occurred. -// } -// -// // The above call decodes the current available buffer. -// // Part of the image can now be refreshed by calling -// // WebPIDecGetRGB()/WebPIDecGetYUVA() etc. -// } -// WebPIDelete(idec); +/* + WebPInitDecBuffer(&output_buffer); + output_buffer.colorspace = mode; + ... + WebPIDecoder* idec = WebPINewDecoder(&output_buffer); + while (additional_data_is_available) { + // ... (get additional data in some new_data[] buffer) + status = WebPIAppend(idec, new_data, new_data_size); + if (status != VP8_STATUS_OK && status != VP8_STATUS_SUSPENDED) { + break; // an error occurred. + } + + // The above call decodes the current available buffer. + // Part of the image can now be refreshed by calling + // WebPIDecGetRGB()/WebPIDecGetYUVA() etc. + } + WebPIDelete(idec); +*/ // Creates a new incremental decoder with the supplied buffer parameter. // This output_buffer can be passed NULL, in which case a default output buffer @@ -388,7 +390,7 @@ WEBP_EXTERN const WebPDecBuffer* WebPIDecodedArea( CHECK(WebPGetFeatures(data, data_size, &config.input) == VP8_STATUS_OK); // C) Adjust 'config', if needed - config.no_fancy_upsampling = 1; + config.options.no_fancy_upsampling = 1; config.output.colorspace = MODE_BGRA; // etc. diff --git a/3rdparty/libwebp/src/webp/encode.h b/3rdparty/libwebp/src/webp/encode.h index b4c599df87..56b68e2f10 100644 --- a/3rdparty/libwebp/src/webp/encode.h +++ b/3rdparty/libwebp/src/webp/encode.h @@ -441,7 +441,7 @@ WEBP_EXTERN int WebPPictureCrop(WebPPicture* picture, // the original dimension will be lost). Picture 'dst' need not be initialized // with WebPPictureInit() if it is different from 'src', since its content will // be overwritten. -// Returns false in case of memory allocation error or invalid parameters. +// Returns false in case of invalid parameters. WEBP_EXTERN int WebPPictureView(const WebPPicture* src, int left, int top, int width, int height, WebPPicture* dst); @@ -455,7 +455,7 @@ WEBP_EXTERN int WebPPictureIsView(const WebPPicture* picture); // dimension will be calculated preserving the aspect ratio. // No gamma correction is applied. // Returns false in case of error (invalid parameter or insufficient memory). -WEBP_EXTERN int WebPPictureRescale(WebPPicture* pic, int width, int height); +WEBP_EXTERN int WebPPictureRescale(WebPPicture* picture, int width, int height); // Colorspace conversion function to import RGB samples. // Previous buffer will be free'd, if any. @@ -526,7 +526,7 @@ WEBP_EXTERN int WebPPictureHasTransparency(const WebPPicture* picture); // Remove the transparency information (if present) by blending the color with // the background color 'background_rgb' (specified as 24bit RGB triplet). // After this call, all alpha values are reset to 0xff. -WEBP_EXTERN void WebPBlendAlpha(WebPPicture* pic, uint32_t background_rgb); +WEBP_EXTERN void WebPBlendAlpha(WebPPicture* picture, uint32_t background_rgb); //------------------------------------------------------------------------------ // Main call diff --git a/3rdparty/libwebp/src/webp/format_constants.h b/3rdparty/libwebp/src/webp/format_constants.h index eca6981a47..999035c5d2 100644 --- a/3rdparty/libwebp/src/webp/format_constants.h +++ b/3rdparty/libwebp/src/webp/format_constants.h @@ -55,7 +55,7 @@ typedef enum { PREDICTOR_TRANSFORM = 0, CROSS_COLOR_TRANSFORM = 1, - SUBTRACT_GREEN = 2, + SUBTRACT_GREEN_TRANSFORM = 2, COLOR_INDEXING_TRANSFORM = 3 } VP8LImageTransformType; diff --git a/3rdparty/libwebp/src/webp/types.h b/3rdparty/libwebp/src/webp/types.h index 47f7f2b007..f255432e41 100644 --- a/3rdparty/libwebp/src/webp/types.h +++ b/3rdparty/libwebp/src/webp/types.h @@ -42,7 +42,11 @@ typedef long long int int64_t; # if defined(__GNUC__) && __GNUC__ >= 4 # define WEBP_EXTERN extern __attribute__ ((visibility ("default"))) # else -# define WEBP_EXTERN extern +# if defined(_MSC_VER) && defined(WEBP_DLL) +# define WEBP_EXTERN __declspec(dllexport) +# else +# define WEBP_EXTERN extern +# endif # endif /* __GNUC__ >= 4 */ #endif /* WEBP_EXTERN */ diff --git a/cmake/OpenCVCompilerOptimizations.cmake b/cmake/OpenCVCompilerOptimizations.cmake index 28929c3890..90ad7d783c 100644 --- a/cmake/OpenCVCompilerOptimizations.cmake +++ b/cmake/OpenCVCompilerOptimizations.cmake @@ -224,7 +224,7 @@ if(X86 OR X86_64) ocv_update(CPU_SSE2_IMPLIES "SSE") endif() - if(CV_ICC) + if(CV_ICC OR CV_ICX) macro(ocv_intel_compiler_optimization_option name unix_flags msvc_flags) ocv_update(CPU_${name}_FLAGS_NAME "${name}") if(MSVC) @@ -260,7 +260,7 @@ if(X86 OR X86_64) ocv_intel_compiler_optimization_option(AVX512_CNL "-xCANNONLAKE" "/Qx:CANNONLAKE") ocv_intel_compiler_optimization_option(AVX512_CLX "-xCASCADELAKE" "/Qx:CASCADELAKE") ocv_intel_compiler_optimization_option(AVX512_ICL "-xICELAKE-CLIENT" "/Qx:ICELAKE-CLIENT") - elseif(CV_GCC OR CV_CLANG) + elseif(CV_GCC OR CV_CLANG OR CV_ICX) ocv_update(CPU_AVX2_FLAGS_ON "-mavx2") ocv_update(CPU_FP16_FLAGS_ON "-mf16c") ocv_update(CPU_AVX_FLAGS_ON "-mavx") @@ -657,7 +657,7 @@ macro(ocv_compiler_optimization_options) endmacro() macro(ocv_compiler_optimization_options_finalize) - if((CV_GCC OR CV_CLANG) AND (X86 OR X86_64)) + if((CV_GCC OR CV_CLANG OR CV_ICX) AND (X86 OR X86_64)) if(NOT APPLE AND CMAKE_SIZEOF_VOID_P EQUAL 4) if(OPENCV_EXTRA_CXX_FLAGS MATCHES "-m(sse2|avx)") add_extra_compiler_option(-mfpmath=sse) # !! important - be on the same wave with x64 compilers @@ -944,7 +944,7 @@ macro(ocv_add_dispatched_file_force_all) endmacro() -if(CV_DISABLE_OPTIMIZATION OR CV_ICC) +if(CV_DISABLE_OPTIMIZATION OR CV_ICC OR CX_ICX) ocv_update(CV_ENABLE_UNROLLED 0) else() ocv_update(CV_ENABLE_UNROLLED 1) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 8bd8668130..427189c079 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -105,6 +105,19 @@ elseif(CV_ICC) add_extra_compiler_option("-fp-model precise") endif() endif() +elseif(CV_ICX) + # ICX uses -ffast-math by default. + # use own flags, if no one of the flags provided by user: -fp-model, -ffast-math -fno-fast-math + if(NOT " ${CMAKE_CXX_FLAGS} ${OPENCV_EXTRA_FLAGS} ${OPENCV_EXTRA_CXX_FLAGS}" MATCHES " /fp:" + AND NOT " ${CMAKE_CXX_FLAGS} ${OPENCV_EXTRA_FLAGS} ${OPENCV_EXTRA_CXX_FLAGS}" MATCHES " -fp-model" + AND NOT " ${CMAKE_CXX_FLAGS} ${OPENCV_EXTRA_FLAGS} ${OPENCV_EXTRA_CXX_FLAGS}" MATCHES " -ffast-math" + AND NOT " ${CMAKE_CXX_FLAGS} ${OPENCV_EXTRA_FLAGS} ${OPENCV_EXTRA_CXX_FLAGS}" MATCHES " -fno-fast-math" + ) + if(NOT ENABLE_FAST_MATH) + add_extra_compiler_option(-fno-fast-math) + add_extra_compiler_option(-fp-model=precise) + endif() + endif() elseif(CV_GCC OR CV_CLANG) if(ENABLE_FAST_MATH) add_extra_compiler_option(-ffast-math) @@ -112,7 +125,7 @@ elseif(CV_GCC OR CV_CLANG) endif() endif() -if(CV_GCC OR CV_CLANG) +if(CV_GCC OR CV_CLANG OR CV_ICX) # High level of warnings. add_extra_compiler_option(-W) if (NOT MSVC) @@ -336,7 +349,7 @@ if(COMMAND ocv_compiler_optimization_options_finalize) endif() # set default visibility to hidden -if((CV_GCC OR CV_CLANG) +if((CV_GCC OR CV_CLANG OR CV_ICX) AND NOT MSVC AND NOT OPENCV_SKIP_VISIBILITY_HIDDEN AND NOT " ${CMAKE_CXX_FLAGS} ${OPENCV_EXTRA_FLAGS} ${OPENCV_EXTRA_CXX_FLAGS}" MATCHES " -fvisibility") diff --git a/cmake/OpenCVDetectCUDA.cmake b/cmake/OpenCVDetectCUDA.cmake index 4a562bdaf9..752ae0edcf 100644 --- a/cmake/OpenCVDetectCUDA.cmake +++ b/cmake/OpenCVDetectCUDA.cmake @@ -228,7 +228,7 @@ if(CUDA_FOUND) endif() endmacro() - set(__cuda_arch_ptx "") + set(__cuda_arch_ptx ${CUDA_ARCH_PTX}) if(CUDA_GENERATION STREQUAL "Fermi") set(__cuda_arch_bin ${_arch_fermi}) elseif(CUDA_GENERATION STREQUAL "Kepler") @@ -259,7 +259,7 @@ if(CUDA_FOUND) set(__cuda_arch_bin ${CUDA_ARCH_BIN}) endif() - if(NOT DEFINED __cuda_arch_bin) + if(NOT DEFINED __cuda_arch_bin AND NOT DEFINED __cuda_arch_ptx) if(ARM) set(__cuda_arch_bin "3.2") set(__cuda_arch_ptx "") @@ -295,6 +295,7 @@ if(CUDA_FOUND) ${_arch_lovelace} ${_arch_hopper} ) + list(GET __cuda_arch_bin -1 __cuda_arch_ptx) endif() endif() diff --git a/cmake/OpenCVDetectCXXCompiler.cmake b/cmake/OpenCVDetectCXXCompiler.cmake index 8fe89b3fe0..4295f96568 100644 --- a/cmake/OpenCVDetectCXXCompiler.cmake +++ b/cmake/OpenCVDetectCXXCompiler.cmake @@ -68,6 +68,23 @@ if(MSVC AND CMAKE_C_COMPILER MATCHES "icc|icl") set(CV_ICC __INTEL_COMPILER_FOR_WINDOWS) endif() +# ---------------------------------------------------------------------------- +# Detect Intel ICXC compiler +# ---------------------------------------------------------------------------- +if(UNIX) + if(__INTEL_COMPILER) + set(CV_ICX __INTEL_LLVM_COMPILER) + elseif(CMAKE_C_COMPILER MATCHES "icx") + set(CV_ICX icx_matches_c_compiler) + elseif(CMAKE_CXX_COMPILER MATCHES "icpx") + set(CV_ICX icpx_matches_cxx_compiler) + endif() +endif() + +if(MSVC AND CMAKE_CXX_COMPILER MATCHES ".*(dpcpp-cl|dpcpp|icx-cl|icpx|icx)(.exe)?$") + set(CV_ICX __INTEL_LLVM_COMPILER_WINDOWS) +endif() + if(NOT DEFINED CMAKE_CXX_COMPILER_VERSION AND NOT OPENCV_SUPPRESS_MESSAGE_MISSING_COMPILER_VERSION) message(WARNING "OpenCV: Compiler version is not available: CMAKE_CXX_COMPILER_VERSION is not set") diff --git a/cmake/OpenCVFindOpenBLAS.cmake b/cmake/OpenCVFindOpenBLAS.cmake index d1db034908..4e3f0cc210 100644 --- a/cmake/OpenCVFindOpenBLAS.cmake +++ b/cmake/OpenCVFindOpenBLAS.cmake @@ -73,7 +73,7 @@ SET(Open_BLAS_LIB_SEARCH_PATHS ) FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS} NO_DEFAULT_PATH) -FIND_LIBRARY(OpenBLAS_LIB NAMES openblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS} NO_DEFAULT_PATH) +FIND_LIBRARY(OpenBLAS_LIB NAMES openblas libopenblas PATHS ${Open_BLAS_LIB_SEARCH_PATHS} NO_DEFAULT_PATH) SET(OpenBLAS_FOUND ON) diff --git a/modules/calib/src/checkchessboard.cpp b/modules/calib/src/checkchessboard.cpp index 47995297c2..ac708a660a 100644 --- a/modules/calib/src/checkchessboard.cpp +++ b/modules/calib/src/checkchessboard.cpp @@ -53,15 +53,12 @@ static void icvGetQuadrangleHypotheses(const std::vector >::const_iterator iter_t; - iter_t i; - for (i = contours.begin(); i != contours.end(); ++i) + for (size_t i = 0; i < contours.size(); ++i) { - const iter_t::difference_type idx = i - contours.begin(); - if (hierarchy.at(idx)[3] != -1) + if (hierarchy.at(i)[3] != -1) continue; // skip holes - const std::vector< cv::Point > & c = *i; + const std::vector< cv::Point > & c = contours[i]; cv::RotatedRect box = cv::minAreaRect(c); float box_size = MAX(box.size.width, box.size.height); diff --git a/modules/core/include/opencv2/core.hpp b/modules/core/include/opencv2/core.hpp index 89046d7907..d76120238c 100644 --- a/modules/core/include/opencv2/core.hpp +++ b/modules/core/include/opencv2/core.hpp @@ -349,6 +349,9 @@ be set to the default -1. In this case, the output array will have the same dept array, be it src1, src2 or both. @note Saturation is not applied when the output array has the depth CV_32S. You may even get result of an incorrect sign in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`add(src,X)` means `add(src,(X,X,X,X))`. +`add(src,(X,))` means `add(src,(X,0,0,0))`. @param src1 first input array or a scalar. @param src2 second input array or a scalar. @param dst output array that has the same size and number of channels as the input array(s); the @@ -390,6 +393,9 @@ in the first case, when src1.depth() == src2.depth(), dtype can be set to the de case the output array will have the same depth as the input array, be it src1, src2 or both. @note Saturation is not applied when the output array has the depth CV_32S. You may even get result of an incorrect sign in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`subtract(src,X)` means `subtract(src,(X,X,X,X))`. +`subtract(src,(X,))` means `subtract(src,(X,0,0,0))`. @param src1 first input array or a scalar. @param src2 second input array or a scalar. @param dst output array of the same size and the same number of channels as the input array. @@ -415,6 +421,9 @@ For a not-per-element matrix product, see gemm . @note Saturation is not applied when the output array has the depth CV_32S. You may even get result of an incorrect sign in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`multiply(src,X)` means `multiply(src,(X,X,X,X))`. +`multiply(src,(X,))` means `multiply(src,(X,0,0,0))`. @param src1 first input array. @param src2 second input array of the same size and the same type as src1. @param dst output array of the same size and type as src1. @@ -443,6 +452,9 @@ Expect correct IEEE-754 behaviour for floating-point data (with NaN, Inf result @note Saturation is not applied when the output array has the depth CV_32S. You may even get result of an incorrect sign in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`divide(src,X)` means `divide(src,(X,X,X,X))`. +`divide(src,(X,))` means `divide(src,(X,0,0,0))`. @param src1 first input array. @param src2 second input array of the same size and type as src1. @param scale scalar factor. @@ -1412,6 +1424,9 @@ The function cv::absdiff calculates: multi-channel arrays, each channel is processed independently. @note Saturation is not applied when the arrays have the depth CV_32S. You may even get a negative value in the case of overflow. +@note (Python) Be careful to difference behaviour between src1/src2 are single number and they are tuple/array. +`absdiff(src,X)` means `absdiff(src,(X,X,X,X))`. +`absdiff(src,(X,))` means `absdiff(src,(X,0,0,0))`. @param src1 first input array or a scalar. @param src2 second input array or a scalar. @param dst output array that has the same size and type as input arrays. diff --git a/modules/core/include/opencv2/core/hal/intrin.hpp b/modules/core/include/opencv2/core/hal/intrin.hpp index eaad52fc01..738ffb2d22 100644 --- a/modules/core/include/opencv2/core/hal/intrin.hpp +++ b/modules/core/include/opencv2/core/hal/intrin.hpp @@ -760,7 +760,22 @@ namespace CV__SIMD_NAMESPACE { inline _Tpvec v_add(const _Tpvec& f1, const _Tpvec& f2, const Args&... vf) { \ return v_add(f1 + f2, vf...); \ } + #define OPENCV_HAL_WRAP_SHIFT_OP(_Tpvec) \ + inline _Tpvec v_shr(const _Tpvec& a, int n) \ + { \ + return a >> n; \ + } \ + inline _Tpvec v_shl(const _Tpvec& a, int n) \ + { \ + return a << n; \ + } + OPENCV_HAL_WRAP_SHIFT_OP(v_uint16) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint32) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint64) + OPENCV_HAL_WRAP_SHIFT_OP(v_int16) + OPENCV_HAL_WRAP_SHIFT_OP(v_int32) + OPENCV_HAL_WRAP_SHIFT_OP(v_int64) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint8) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint16) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_uint32) @@ -784,6 +799,12 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int32x4) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int64x2) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float32x4) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint16x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint32x4) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint64x2) + OPENCV_HAL_WRAP_SHIFT_OP(v_int16x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_int32x4) + OPENCV_HAL_WRAP_SHIFT_OP(v_int64x2) #if CV_SIMD_64F OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float64x2) #endif @@ -799,6 +820,12 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int32x8) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_int64x4) OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float32x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint16x16) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint32x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_uint64x4) + OPENCV_HAL_WRAP_SHIFT_OP(v_int16x16) + OPENCV_HAL_WRAP_SHIFT_OP(v_int32x8) + OPENCV_HAL_WRAP_SHIFT_OP(v_int64x4) #if CV_SIMD_64F OPENCV_HAL_WRAP_BIN_OP_ADDSUB(v_float64x4) #endif @@ -816,7 +843,9 @@ namespace CV__SIMD_NAMESPACE { inline _Tpvec v_xor(const _Tpvec& a, const _Tpvec& b) \ { \ return a ^ b; \ - } \ + } + + #define OPENCV_HAL_WRAP_NOT_OP(_Tpvec) \ inline _Tpvec v_not(const _Tpvec& a) \ { \ return ~a; \ @@ -830,6 +859,18 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int16) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int32) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int64) + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float32) + OPENCV_HAL_WRAP_NOT_OP(v_uint8) + OPENCV_HAL_WRAP_NOT_OP(v_uint16) + OPENCV_HAL_WRAP_NOT_OP(v_uint32) + OPENCV_HAL_WRAP_NOT_OP(v_uint64) + OPENCV_HAL_WRAP_NOT_OP(v_int8) + OPENCV_HAL_WRAP_NOT_OP(v_int16) + OPENCV_HAL_WRAP_NOT_OP(v_int32) + OPENCV_HAL_WRAP_NOT_OP(v_int64) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float64) + #endif #if CV_SIMD_WIDTH != 16/*128*/ && CV_SIMD128 OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_uint8x16) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_uint16x8) @@ -839,6 +880,18 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int16x8) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int32x4) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int64x2) + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float32x4) + OPENCV_HAL_WRAP_NOT_OP(v_uint8x16) + OPENCV_HAL_WRAP_NOT_OP(v_uint16x8) + OPENCV_HAL_WRAP_NOT_OP(v_uint32x4) + OPENCV_HAL_WRAP_NOT_OP(v_uint64x2) + OPENCV_HAL_WRAP_NOT_OP(v_int8x16) + OPENCV_HAL_WRAP_NOT_OP(v_int16x8) + OPENCV_HAL_WRAP_NOT_OP(v_int32x4) + OPENCV_HAL_WRAP_NOT_OP(v_int64x2) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float64x2) + #endif #endif #if CV_SIMD_WIDTH != 32/*256*/ && CV_SIMD256 OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_uint8x32) @@ -849,6 +902,18 @@ namespace CV__SIMD_NAMESPACE { OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int16x16) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int32x8) OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_int64x4) + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float32x8) + OPENCV_HAL_WRAP_NOT_OP(v_uint8x32) + OPENCV_HAL_WRAP_NOT_OP(v_uint16x16) + OPENCV_HAL_WRAP_NOT_OP(v_uint32x8) + OPENCV_HAL_WRAP_NOT_OP(v_uint64x4) + OPENCV_HAL_WRAP_NOT_OP(v_int8x32) + OPENCV_HAL_WRAP_NOT_OP(v_int16x16) + OPENCV_HAL_WRAP_NOT_OP(v_int32x8) + OPENCV_HAL_WRAP_NOT_OP(v_int64x4) + #if CV_SIMD_64F + OPENCV_HAL_WRAP_BIN_OP_LOGIC(v_float64x4) + #endif #endif #define OPENCV_HAL_WRAP_BIN_OP_MUL(_Tpvec) \ diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_compat_overloaded.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_compat_overloaded.hpp index 7dd735f99a..914ad28978 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_compat_overloaded.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_compat_overloaded.hpp @@ -45,6 +45,7 @@ OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vuint8m2_t, u8m2, vuint8m2_t, i8) OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vuint8m4_t, u8m4, vuint8m4_t, i8) OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vuint8m8_t, u8m8, vuint8m8_t, i8) OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vfloat32m1_t, f32m1, vuint32m1_t, i32) +OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vuint32m1_t, u32m1, vuint32m1_t, i32) #if CV_SIMD_SCALABLE_64F OPENCV_HAL_IMPL_RVV_FUN_LOXEI(vfloat64m1_t, f64m1, vuint32mf2_t, i32) #endif diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp index 6c28b44f5b..a45c90cf90 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp @@ -475,6 +475,25 @@ OPENCV_HAL_IMPL_RVV_LUT(v_float32, float, m1) OPENCV_HAL_IMPL_RVV_LUT(v_float64, double, mf2) #endif +#define OPENCV_HAL_IMPL_RVV_LUT_VEC(_Tpvec, _Tp) \ +inline _Tpvec v_lut(const _Tp* tab, const v_int32& vidx) \ +{ \ + v_uint32 vidx_ = vmul(vreinterpret_u32m1(vidx), sizeof(_Tp), VTraits::vlanes()); \ + return vloxei32(tab, vidx_, VTraits<_Tpvec>::vlanes()); \ +} +OPENCV_HAL_IMPL_RVV_LUT_VEC(v_float32, float) +OPENCV_HAL_IMPL_RVV_LUT_VEC(v_int32, int) +OPENCV_HAL_IMPL_RVV_LUT_VEC(v_uint32, unsigned) + +#if CV_SIMD_SCALABLE_64F +inline v_float64 v_lut(const double* tab, const v_int32& vidx) \ +{ \ + vuint32mf2_t vidx_ = vmul(vlmul_trunc_u32mf2(vreinterpret_u32m1(vidx)), sizeof(double), VTraits::vlanes()); \ + return vloxei32(tab, vidx_, VTraits::vlanes()); \ +} +#endif + + inline v_uint8 v_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut((schar*)tab, idx)); } inline v_uint8 v_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_pairs((schar*)tab, idx)); } inline v_uint8 v_lut_quads(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_quads((schar*)tab, idx)); } @@ -690,23 +709,27 @@ inline v_float64 v_not (const v_float64& a) \ ////////////// Bitwise shifts ////////////// +/* Usage +1. v_shl(vec); +2. v_shl(vec, N); // instead of vec << N, when N is non-constant. +*/ #define OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(_Tpvec, vl) \ -template inline _Tpvec v_shl(const _Tpvec& a) \ +template inline _Tpvec v_shl(const _Tpvec& a, int n = s) \ { \ return _Tpvec(vsll(a, uint8_t(n), vl)); \ } \ -template inline _Tpvec v_shr(const _Tpvec& a) \ +template inline _Tpvec v_shr(const _Tpvec& a, int n = s) \ { \ return _Tpvec(vsrl(a, uint8_t(n), vl)); \ } #define OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(_Tpvec, vl) \ -template inline _Tpvec v_shl(const _Tpvec& a) \ +template inline _Tpvec v_shl(const _Tpvec& a, int n = s) \ { \ return _Tpvec(vsll(a, uint8_t(n), vl)); \ } \ -template inline _Tpvec v_shr(const _Tpvec& a) \ +template inline _Tpvec v_shr(const _Tpvec& a, int n = s) \ { \ return _Tpvec(vsra(a, uint8_t(n), vl)); \ } diff --git a/modules/core/include/opencv2/core/hal/intrin_wasm.hpp b/modules/core/include/opencv2/core/hal/intrin_wasm.hpp index b4178af8b7..db3cb2a9ae 100644 --- a/modules/core/include/opencv2/core/hal/intrin_wasm.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_wasm.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include "opencv2/core/saturate.hpp" #define CV_SIMD128 1 diff --git a/modules/core/src/copy.cpp b/modules/core/src/copy.cpp index 930c392c4e..7b7c717d54 100644 --- a/modules/core/src/copy.cpp +++ b/modules/core/src/copy.cpp @@ -898,14 +898,14 @@ void copyMakeBorder_8u( const uchar* src, size_t srcstep, cv::Size srcroi, } dstroi.width *= elemSize; - dst += dststep*top; for( i = 0; i < top; i++ ) { j = cv::borderInterpolate(i - top, srcroi.height, borderType); - memcpy(dst + (i - top)*dststep, dst + j*dststep, dstroi.width); + memcpy(dst + i*dststep, dst + (top+j)*dststep, dstroi.width); } + dst += dststep*top; for( i = 0; i < bottom; i++ ) { j = cv::borderInterpolate(i + srcroi.height, srcroi.height, borderType); diff --git a/modules/core/src/hal_internal.cpp b/modules/core/src/hal_internal.cpp index 53b355a44f..34b5714ee8 100644 --- a/modules/core/src/hal_internal.cpp +++ b/modules/core/src/hal_internal.cpp @@ -66,6 +66,7 @@ #if defined(__clang__) && defined(__has_feature) #if __has_feature(memory_sanitizer) +#include #define CV_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \ __msan_unpoison(address, size) #endif diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index f8ae0e9c27..c206f09fb5 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -1158,11 +1158,11 @@ Mat& Mat::adjustROI( int dtop, int dbottom, int dleft, int dright ) std::swap(col1, col2); if (dims == 1) { - data += (col1 - ofs.x)*esz; + data += (col1 - ofs.x)*(std::ptrdiff_t)esz; cols = col2 - col1; size.p[0] = cols; } else { - data += (row1 - ofs.y)*step + (col1 - ofs.x)*esz; + data += (row1 - ofs.y)*(std::ptrdiff_t)step + (col1 - ofs.x)*(std::ptrdiff_t)esz; rows = row2 - row1; cols = col2 - col1; size.p[0] = rows; size.p[1] = cols; updateContinuityFlag(); diff --git a/modules/core/src/parallel_impl.cpp b/modules/core/src/parallel_impl.cpp index b18204ce84..fc7c4c2b6e 100644 --- a/modules/core/src/parallel_impl.cpp +++ b/modules/core/src/parallel_impl.cpp @@ -580,8 +580,11 @@ void ThreadPool::run(const Range& range, const ParallelLoopBody& body, double ns pthread_mutex_unlock(&mutex); CV_LOG_VERBOSE(NULL, 5, "MainThread: wake worker threads..."); - for (size_t i = 0; i < threads.size(); ++i) + size_t num_threads_to_wake = std::min(static_cast(range.size()), threads.size()); + for (size_t i = 0; i < num_threads_to_wake; ++i) { + if (job->current_task >= job->range.size()) + break; WorkerThread& thread = *(threads[i].get()); if ( #if defined(__clang__) && defined(__has_feature) diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index 509e7f732f..5f14767bd3 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -1369,6 +1369,18 @@ TEST(Core_Mat, copyNx1ToVector) ASSERT_PRED_FORMAT2(cvtest::MatComparator(0, 0), ref_dst16, cv::Mat_(dst16).reshape(1, 5)); } +TEST(Core_Mat, copyMakeBoderUndefinedBehavior) +{ + Mat1b src(4, 4), dst; + randu(src, Scalar(10), Scalar(100)); + // This could trigger a (signed int)*size_t operation which is undefined behavior. + cv::copyMakeBorder(src, dst, 1, 1, 1, 1, cv::BORDER_REFLECT_101); + EXPECT_EQ(0, cv::norm(src.row(1), dst(Rect(1,0,4,1)))); + EXPECT_EQ(0, cv::norm(src.row(2), dst(Rect(1,5,4,1)))); + EXPECT_EQ(0, cv::norm(src.col(1), dst(Rect(0,1,1,4)))); + EXPECT_EQ(0, cv::norm(src.col(2), dst(Rect(5,1,1,4)))); +} + TEST(Core_Matx, fromMat_) { Mat_ a = (Mat_(2,2) << 10, 11, 12, 13); diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp index f508f51ac4..fef8cb839f 100644 --- a/modules/core/test/test_misc.cpp +++ b/modules/core/test/test_misc.cpp @@ -917,5 +917,28 @@ REGISTER_TYPED_TEST_CASE_P(Rect_Test, Overflows); typedef ::testing::Types RectTypes; INSTANTIATE_TYPED_TEST_CASE_P(Negative_Test, Rect_Test, RectTypes); +// Expected that SkipTestException thrown in the constructor should skip test but not fail +struct TestFixtureSkip: public ::testing::Test { + TestFixtureSkip(bool throwEx = true) { + if (throwEx) { + throw SkipTestException("Skip test at constructor"); + } + } +}; + +TEST_F(TestFixtureSkip, NoBodyRun) { + FAIL() << "Unreachable code called"; +} + +// Expected that SkipTestException thrown in SetUp method should skip test but not fail +struct TestSetUpSkip: public ::testing::Test { + virtual void SetUp() CV_OVERRIDE { + throw SkipTestException("Skip test at SetUp"); + } +}; + +TEST_F(TestSetUpSkip, NoBodyRun) { + FAIL() << "Unreachable code called"; +} }} // namespace diff --git a/modules/core/test/test_operations.cpp b/modules/core/test/test_operations.cpp index 2e825afce3..430f285438 100644 --- a/modules/core/test/test_operations.cpp +++ b/modules/core/test/test_operations.cpp @@ -1380,6 +1380,15 @@ TEST(MatTestRoi, adjustRoiOverflow) ASSERT_EQ(roi.rows, m.rows); } +TEST(MatTestRoi, adjustRoiUndefinedBehavior) +{ + Mat m(6, 6, CV_8U); + Mat roi(m, cv::Range(2, 4), cv::Range(2, 4)); + // This could trigger a (negative int)*size_t when updating data, + // which is undefined behavior. + roi.adjustROI(2, 2, 2, 2); + EXPECT_EQ(m.data, roi.data); +} CV_ENUM(SortRowCol, SORT_EVERY_COLUMN, SORT_EVERY_ROW) CV_ENUM(SortOrder, SORT_ASCENDING, SORT_DESCENDING) diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 774e3c7b5a..5963eb68d3 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -9,6 +9,7 @@ ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX LAS ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_block" AVX AVX2) ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_depthwise" AVX AVX2 RVV LASX) ocv_add_dispatched_file_force_all("layers/cpu_kernels/conv_winograd_f63" AVX AVX2) +ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index e133ffea65..b00d3f54a6 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -241,6 +241,39 @@ CV__DNN_INLINE_NS_BEGIN }; + /** @brief This function performs array summation based + * on the Einstein summation convention. The function + * allows for concise expressions of various mathematical + * operations using subscripts. + * + * By default, the labels are placed in alphabetical + * order at the end of the output. + * For example: + * if `c = einsum("i,j", a, b)`, then `c[i,j] == a[i]*b[j]`. + * However, if `c = einsum("j,i", a, b)`, then `c[i,j] = a[j]*b[i]`. + * Alternatively, you can control the output order or prevent + * an axis from being summed/force an axis to be summed + * by providing indices for the output. + * For example: + * `diag(a)` -> `einsum("ii->i", a)` + * `sum(a, axis=0)` -> `einsum("i...->", a)` + * Subscripts at the beginning and end may be specified + * by putting an ellipsis "..." in the middle. + * For instance, the function `einsum("i...i", a)` takes + * the diagonal of the first and last dimensions of + * the operand, and `einsum("ij...,jk...->ik...")` performs + * the matrix product using the first two indices + * of each operand instead of the last two. + * When there is only one operand, no axes being summed, + * and no output parameter, this function returns + * a view into the operand instead of creating a copy. + */ + class CV_EXPORTS EinsumLayer : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS BaseConvolutionLayer : public Layer { public: @@ -1101,6 +1134,22 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS GemmLayer : public Layer { + public: + bool trans_a; + bool trans_b; + float alpha; + float beta; + + static Ptr create(const LayerParams& params); + }; + + class CV_EXPORTS ExpandLayer : public Layer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + //! @} //! @} CV__DNN_INLINE_NS_END diff --git a/modules/dnn/perf/perf_gemm.cpp b/modules/dnn/perf/perf_gemm.cpp new file mode 100644 index 0000000000..8051cc273e --- /dev/null +++ b/modules/dnn/perf/perf_gemm.cpp @@ -0,0 +1,249 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "perf_precomp.hpp" +#include + +namespace opencv_test { + +struct GemmParam_t { + std::vector a_shape; + std::vector b_shape; + std::vector c_shape; + bool trans_a; + bool trans_b; + + GemmParam_t(std::vector a_shape_, std::vector b_shape_, std::vector c_shape_ = {}, bool trans_a_ = false, bool trans_b_ = false) + : a_shape(a_shape_), b_shape(b_shape_), c_shape(c_shape_), trans_a(trans_a_), trans_b(trans_b_) {} +}; + +// TODO: Dsiable most of the test cases except vision transformers to save time +static const GemmParam_t test_gemm_configs[] = { + // vision transformers cases + { { 768, 768 }, { 768, 768 }, { 768 } }, + { { 1024, 1024 }, { 1024, 1024 }, { 1024 } }, + { { 50, 768 }, { 768, 2304 } }, + { { 197, 768 }, { 768, 2304 } }, + { { 50, 1024 }, { 1024, 3072 } }, + { { 197, 1024 }, { 1024, 3072 } }, + +// these cases are commented to save testing time +/* + // square mat + { { 64, 64 }, { 64, 64 } }, + { { 128, 128 }, { 128, 128 } }, + { { 256, 256 }, { 256, 256 } }, + { { 512, 512 }, { 512, 512 } }, + { { 1024, 1024 }, { 1024, 1024 } }, + { { 4096, 4096 }, { 4096, 4096 } }, + + // retangular mat + { { 256, 256 }, { 256, 1024 } }, + { { 256, 1024 }, { 1024, 256 } }, + { { 256, 1024 }, { 1024, 1024 } }, + { { 1024, 1024 }, { 1024, 256 } }, + { { 1024, 256 }, { 256, 1024 } }, + { { 1024, 256 }, { 256, 256 } }, + + // with C + { { 256, 256 }, { 256, 256 }, { 256 } }, + { { 256, 256 }, { 256, 1024 }, { 1024 } }, + { { 256, 1024 }, { 1024, 256 }, { 256 } }, + { { 256, 1024 }, { 1024, 1024 }, { 1024 } }, + { { 1024, 1024 }, { 1024, 256 }, { 256 } }, + { { 1024, 256 }, { 256, 1024 }, { 1024 } }, + { { 1024, 256 }, { 256, 256 }, { 256 } }, + + // with C and trans_b + { { 256, 256 }, { 256, 256 }, { 256 } , false, true}, + { { 256, 1024 }, { 256, 1024 }, { 256 } , false, true}, + { { 256, 1024 }, { 1024, 1024 }, { 1024 } , false, true}, + { { 1024, 1024 }, { 1024, 1024 }, { 1024 } , false, true}, + { { 1024, 256 }, { 1024, 256 }, { 1024 } , false, true}, + { { 1024, 256 }, { 256, 256 }, { 256 } , false, true}, + + // with C and trans_b and trans_a + { { 256, 256 }, { 256, 256 }, { 256 } , true, true}, + { { 1024, 256 }, { 256, 1024 }, { 256 } , true, true}, + { { 256, 1024 }, { 1024, 256 }, { 1024 } , true, true}, + { { 1024, 1024 }, { 1024, 1024 }, { 1024 } , true, true}, +*/ +}; + +struct GemmParamId +{ + enum { + GEMM_0 = 0, + GEMM_LAST = sizeof(test_gemm_configs) / sizeof(test_gemm_configs[0]) + }; + int val_; + GemmParamId(int val = 0) : val_(val) {} + operator int() const { return val_; } + static ::testing::internal::ParamGenerator all() + { + enum { NUM = (int)GEMM_LAST }; + GemmParamId v_[NUM]; for (int i = 0; i < NUM; ++i) { v_[i] = GemmParamId(i); } // reduce generated code size + return ::testing::ValuesIn(v_, v_ + NUM); + } +}; + +static inline void PrintTo(const GemmParamId& v, std::ostream* os) +{ + CV_Assert((int)v >= 0); CV_Assert((int)v < GemmParamId::GEMM_LAST); + const GemmParam_t& p = test_gemm_configs[(int)v]; + + auto print_shape = [os](const std::vector& shape, const std::string tag) { + if (shape.empty()) { + return ; + } + + *os << tag << "=["; + for (size_t i = 0; i < shape.size(); ++i) { + if (i == shape.size() - 1) { + *os << shape[i] << "]"; + break; + } + *os << shape[i] << ", "; + } + }; + + print_shape(p.a_shape, "A"); + print_shape(p.b_shape, ", B"); + print_shape(p.c_shape, ", C"); + *os << ", trans_a=" << p.trans_a << ", trans_b=" << p.trans_b; +} + +typedef tuple > GemmTestParam_t; +typedef TestBaseWithParam Gemm; + +PERF_TEST_P_(Gemm, gemm) +{ + int test_id = (int)get<0>(GetParam()); + ASSERT_GE(test_id, 0); ASSERT_LT(test_id, GemmParamId::GEMM_LAST); + const GemmParam_t& params = test_gemm_configs[test_id]; + auto a_shape = params.a_shape; + auto b_shape = params.b_shape; + auto c_shape = params.c_shape; + auto trans_a = params.trans_a; + auto trans_b = params.trans_b; + float alpha = 1.f; + float beta = 1.f; + + Backend backend_id = get<0>(get<1>(GetParam())); + Target target_id = get<1>(get<1>(GetParam())); + + bool have_bias = c_shape.empty() ? false : true; + + Mat A(static_cast(a_shape.size()), a_shape.data(), CV_32F); + randu(A, -1.0f, 1.0f); + Mat B(static_cast(b_shape.size()), b_shape.data(), CV_32F); + randu(A, -1.0f, 1.0f); + + LayerParams lp; + lp.type = "Gemm"; + lp.name = "testLayer"; + lp.set("transA", trans_a); + lp.set("transB", trans_b); + lp.set("alpha", alpha); + lp.set("beta", beta); + lp.set("real_ndims_C", static_cast(c_shape.size())); + + lp.set("constB", true); + lp.blobs.push_back(B); + if (have_bias) { + Mat C(static_cast(c_shape.size()), c_shape.data(), CV_32F); + randu(C, -1.0f, 1.0f); + lp.set("have_bias", true); + lp.set("constC", true); + lp.blobs.push_back(C); + } + + Net net; + net.addLayerToPrev(lp.name, lp.type, lp); + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + + // warmup + { + net.setInput(A); + Mat out = net.forward(); + } + + TEST_CYCLE() + { + Mat res = net.forward(); + } + + SANITY_CHECK_NOTHING(); +} + +PERF_TEST_P_(Gemm, innerproduct) +{ + int test_id = (int)get<0>(GetParam()); + ASSERT_GE(test_id, 0); ASSERT_LT(test_id, GemmParamId::GEMM_LAST); + const GemmParam_t& params = test_gemm_configs[test_id]; + auto a_shape = params.a_shape; + auto b_shape = params.b_shape; + auto c_shape = params.c_shape; + auto trans_a = params.trans_a; + auto trans_b = params.trans_b; + + Backend backend_id = get<0>(get<1>(GetParam())); + Target target_id = get<1>(get<1>(GetParam())); + + bool have_bias = c_shape.empty() ? false : true; + + Mat A(static_cast(a_shape.size()), a_shape.data(), CV_32F); + randu(A, -1.0f, 1.0f); + Mat B(static_cast(b_shape.size()), b_shape.data(), CV_32F); + randu(A, -1.0f, 1.0f); + + LayerParams lp; + lp.type = "InnerProduct"; + lp.name = "testLayer"; + if (trans_a) { + cv::transpose(A, A); + } + if (!trans_b) { + cv::transpose(B, B); + } + lp.blobs.push_back(B); + lp.set("num_output", B.size[0]); + if (have_bias) { + Mat C(static_cast(c_shape.size()), c_shape.data(), CV_32F); + randu(C, -1.0f, 1.0f); + lp.blobs.push_back(C); + lp.set("bias_term", true); + } else { + lp.set("bias_term", false); + } + + Net net; + net.addLayerToPrev(lp.name, lp.type, lp); + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + + // warmup + { + std::vector input_names(1); + input_names[0] = "A"; + net.setInputsNames(input_names); + net.setInput(A, input_names[0]); + Mat out = net.forward(); + } + + TEST_CYCLE() + { + Mat res = net.forward(); + } + + SANITY_CHECK_NOTHING(); +} + +INSTANTIATE_TEST_CASE_P(/**/, Gemm, Combine( + GemmParamId::all(), + dnnBackendsAndTargets(false, false) // defined in ../test/test_common.hpp +)); + +} // namespace diff --git a/modules/dnn/src/caffe/caffe_importer.cpp b/modules/dnn/src/caffe/caffe_importer.cpp index 3c08b92a75..a5ee074ef8 100644 --- a/modules/dnn/src/caffe/caffe_importer.cpp +++ b/modules/dnn/src/caffe/caffe_importer.cpp @@ -125,6 +125,7 @@ public: { const google::protobuf::UnknownField& field = unknownFields.field(i); CV_Assert(field.type() == google::protobuf::UnknownField::TYPE_GROUP); + CV_CheckGE(field.group().field_count(), 2, "UnknownField should have at least 2 items: name and value"); std::string fieldName = field.group().field(0).length_delimited(); std::string fieldValue = field.group().field(1).length_delimited(); params.set(fieldName, fieldValue); diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 2ce54ac0bb..3183d71f0b 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -101,6 +101,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Reduce, ReduceLayer); CV_DNN_REGISTER_LAYER_CLASS(LRN, LRNLayer); CV_DNN_REGISTER_LAYER_CLASS(InnerProduct, InnerProductLayer); + CV_DNN_REGISTER_LAYER_CLASS(Gemm, GemmLayer); CV_DNN_REGISTER_LAYER_CLASS(Softmax, SoftmaxLayer); CV_DNN_REGISTER_LAYER_CLASS(SoftMax, SoftmaxLayer); // For compatibility. See https://github.com/opencv/opencv/issues/16877 CV_DNN_REGISTER_LAYER_CLASS(MVN, MVNLayer); @@ -157,6 +158,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(Reciprocal, ReciprocalLayer); CV_DNN_REGISTER_LAYER_CLASS(Gather, GatherLayer); CV_DNN_REGISTER_LAYER_CLASS(LayerNormalization, LayerNormLayer); + CV_DNN_REGISTER_LAYER_CLASS(Expand, ExpandLayer); CV_DNN_REGISTER_LAYER_CLASS(Crop, CropLayer); CV_DNN_REGISTER_LAYER_CLASS(Eltwise, EltwiseLayer); @@ -183,6 +185,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer); CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer); CV_DNN_REGISTER_LAYER_CLASS(CumSum, CumSumLayer); + CV_DNN_REGISTER_LAYER_CLASS(Einsum, EinsumLayer); CV_DNN_REGISTER_LAYER_CLASS(Scatter, ScatterLayer); CV_DNN_REGISTER_LAYER_CLASS(ScatterND, ScatterNDLayer); diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp new file mode 100644 index 0000000000..b7aa18d486 --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm.cpp @@ -0,0 +1,262 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// This file is modified from the ficus (https://github.com/vpisarev/ficus/blob/master/runtime/ficus/impl/gemm.impl.h). +// Here is the original license: +/* + This file is a part of ficus language project. + See ficus/LICENSE for the licensing terms +*/ + +#include "../../precomp.hpp" +#include "fast_gemm.hpp" + +#define CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +#include "fast_gemm_kernels.simd.hpp" +#include "layers/cpu_kernels/fast_gemm_kernels.simd_declarations.hpp" +#undef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +#include "fast_gemm_kernels.default.hpp" + +namespace cv { namespace dnn { + +void fastGemmPackB(const Mat &B, std::vector &packed_B, bool trans, FastGemmOpt &opt) { + CV_CheckEQ(B.dims, 2, "fastGemmPackB: input mat should be two-dimensional"); + CV_CheckTypeEQ(B.type(), CV_32F, "fastGemmPackB: only float32 is supported for now"); + + auto B_shape = shape(B); + int K = B_shape[0], N = B_shape[1], ldb0 = N, ldb1 = 1; + if (trans) { + std::swap(K, N); + std::swap(ldb0, ldb1); + } + +#if CV_TRY_NEON + if (opt.use_neon) { + int size_packed_B = opt_NEON::fastGemmPackBSize(N, K); + packed_B.resize(size_packed_B); + opt_NEON::fastGemmPackBKernel(B.ptr(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize()); + } else +#endif +#if CV_TRY_AVX2 + if (opt.use_avx2) { + int size_packed_B = opt_AVX2::fastGemmPackBSize(N, K); + packed_B.resize(size_packed_B); + opt_AVX2::fastGemmPackBKernel(B.ptr(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize()); + } else +#endif +#if CV_TRY_AVX + if (opt.use_avx) { + int size_packed_B = opt_AVX::fastGemmPackBSize(N, K); + packed_B.resize(size_packed_B); + opt_AVX::fastGemmPackBKernel(B.ptr(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize()); + } else +#endif +#if CV_TRY_LASX + if (opt.use_lasx) { + int size_packed_B = opt_LASX::fastGemmPackBSize(N, K); + packed_B.resize(size_packed_B); + opt_LASX::fastGemmPackBKernel(B.ptr(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize()); + } else +#endif + { + int size_packed_B = cpu_baseline::fastGemmPackBSize(N, K); + packed_B.resize(size_packed_B); + cpu_baseline::fastGemmPackBKernel(B.ptr(), (char *)packed_B.data(), N, K, ldb0, ldb1, B.elemSize()); + } +} + +static void fast_gemm_thin(float alpha, float beta, int M, int N, int K, + const char *a_, int lda0, int lda1, + const char *b_, int ldb, + char *c_, int ldc) { + const float* a = (const float*)a_; + + auto fn = [&](const Range &r) { + for(int start = r.start ; start < r.end; start++ ) { + float* c_i = (float*)c_ + start * ldc; + if (beta == 0.f) + for(int j = 0; j < N; j++ ) c_i[j] = 0.f; + else if (beta != 1.f) + for(int j = 0; j < N; j++ ) c_i[j] *= beta; + for(int k = 0; k < K; k++ ) { + const float* b_k = (const float*)b_ + k * ldb; + float aval = alpha * a[start * lda0 + k * lda1]; + for(int j = 0; j < N; j++ ) + c_i[j] += aval * b_k[j]; + } + } + }; + + int total = M; // outer loops + int cost_per_thread = static_cast(K * N); // inner loops + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +void fastGemm(bool trans_a, int M, int N, int K, + float alpha, const float *A, int lda, + const float *packed_B, float beta, + float *C, int ldc, FastGemmOpt &opt) { + int lda0 = lda, lda1 = 1; + if (trans_a) { + std::swap(lda0, lda1); + } + +#if CV_TRY_NEON + if (opt.use_neon) { + opt_NEON::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float)); + } else +#endif +#if CV_TRY_AVX2 + if (opt.use_avx2) { + opt_AVX2::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float)); + } else +#endif +#if CV_TRY_AVX + if (opt.use_avx) { + opt_AVX::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float)); + } else +#endif +#if CV_TRY_LASX + if (opt.use_lasx) { + opt_LASX::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float)); + } else +#endif + { + cpu_baseline::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, (const char *)packed_B, beta, (char *)C, ldc, sizeof(float)); + } +} + +void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb, + float alpha, const float *A, int lda0, int lda1, const float *B, int ldb0, int ldb1, + float beta, float *C, int ldc, FastGemmOpt &opt) { + + const char *a = (const char *)A; + const char *b = (const char *)B; + char *c = (char *)C; + + int M = trans_a ? na : ma; + int N = trans_b ? mb : nb; + int K = trans_a ? ma : na; + + if (trans_a) { + std::swap(lda0, lda1); + } + if (trans_b) { + std::swap(ldb0, ldb1); + } + + if (!trans_b && ldb1 == 1 && (M <= 4 || (uint64_t)M * N * K <= 10000)) { + return fast_gemm_thin(alpha, beta, M, N, K, a, lda0, lda1, b, ldb0, c, ldc); + } + +#if CV_TRY_NEON + if (opt.use_neon) { + opt_NEON::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, + (const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float)); + } else +#endif +#if CV_TRY_AVX2 + if (opt.use_avx2) { + opt_AVX2::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, + (const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float)); + } else +#endif +#if CV_TRY_AVX + if (opt.use_avx) { + opt_AVX::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, + (const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float)); + } else +#endif +#if CV_TRY_LASX + if (opt.use_lasx) { + opt_LASX::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, + (const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float)); + } else +#endif + { + cpu_baseline::fastGemmKernel(M, N, K, alpha, (const char *)A, lda0, lda1, + (const char *)B, ldb0, ldb1, beta, (char *)C, ldc, sizeof(float)); + } +} + +void fastGemm(bool trans_a, bool trans_b, + float alpha, const Mat &A, const Mat &B, + float beta, Mat &C, FastGemmOpt &opt) { + CV_CheckTypeEQ(A.type(), CV_32F, "DNN/fastGemm: only support float32 for now"); + CV_CheckTypeEQ(A.type(), B.type(), "DNN/fastGemm: A and B should have the same type"); + CV_CheckTypeEQ(B.type(), C.type(), "DNN/fastGemm: B and C should have the same type"); + + const auto shape_a = shape(A); + CV_CheckEQ(shape_a.size(), static_cast(2), "DNN/fastGemm: A must be 2-dimensional"); + const auto shape_b = shape(B); + CV_CheckEQ(shape_b.size(), static_cast(2), "DNN/fastGemm: B must be 2-dimensional"); + const auto shape_c = shape(C); + CV_CheckEQ(shape_c.size(), static_cast(2), "DNN/fastGemm: C must be 2-dimensional"); + + int ma = shape_a[0], na = shape_a[1]; + int mb = shape_b[0], nb = shape_b[1]; + + int lda0 = na, lda1 = 1, ldb0 = nb, ldb1 = 1, ldc = shape_c[1]; + + const float *a = A.ptr(); + const float *b = B.ptr(); + float *c = C.ptr(); + + fastGemm(trans_a, trans_b, ma, na, mb, nb, + alpha, a, lda0, lda1, b, ldb0, ldb1, + beta, c, ldc, opt); +} + +void fastGemmBatched(bool trans_a, bool trans_b, + float alpha, const Mat &A, const Mat &B, + float beta, Mat &C, FastGemmOpt &opt) { + CV_CheckTypeEQ(A.type(), B.type(), "DNN/fastGemmBatched: A and B should have the same type"); + CV_CheckTypeEQ(B.type(), C.type(), "DNN/fastGemmBatched: B and C should have the same type"); + CV_CheckTypeEQ(A.type(), CV_32F, "DNN/fastGemmBatched: only support float32 for now"); + + const auto shape_a = shape(A); + size_t dims_A = shape_a.size(); + CV_CheckGE(dims_A, static_cast(2), "DNN/fastGemmBatched: A must be n-dimensional (n >= 2)"); + const auto shape_b = shape(B); + CV_CheckEQ(shape_b.size(), static_cast(2), "DNN/fastGemmBatched: B must be 2-dimensional"); + const auto shape_c = shape(C); + size_t dims_C = shape_c.size(); + CV_CheckGE(dims_C, static_cast(2), "DNN/fastGemmBatched: C must be n-dimensional (n >= 2)"); + + if (trans_a) { + int ma = shape_a[dims_A - 2], na = shape_a[dims_A - 1]; + int mb = shape_b[0], nb = shape_b[1]; + + int lda0 = na, lda1 = 1, ldb0 = nb, ldb1 = 1, ldc = shape_c[1]; + + const float *a = A.ptr(); + const float *b = B.ptr(); + float *c = C.ptr(); + + int batches = std::accumulate(shape_a.begin(), shape_a.end() - 2, 1, std::multiplies()); + int step_a = ma * na, step_c = na * nb; + for (int i = 0; i < batches; i++) { + fastGemm(true, trans_b, ma, na, mb, nb, + alpha, a + i * step_a, lda0, lda1, b, ldb0, ldb1, + beta, c + i * step_c, ldc, opt); + } + } else { + int ma = std::accumulate(shape_a.begin(), shape_a.end() - 1, 1, std::multiplies()), + na = shape_a[dims_A - 1]; + int mb = shape_b[0], nb = shape_b[1]; + + int lda0 = na, lda1 = 1, ldb0 = nb, ldb1 = 1, ldc = shape_c[1]; + + const float *a = A.ptr(); + const float *b = B.ptr(); + float *c = C.ptr(); + + fastGemm(false, trans_b, ma, na, mb, nb, + alpha, a, lda0, lda1, b, ldb0, ldb1, + beta, c, ldc, opt); + } +} + +}} // cv::dnn diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp new file mode 100644 index 0000000000..7f9e5c3017 --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm.hpp @@ -0,0 +1,65 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// This file is modified from the ficus (https://github.com/vpisarev/ficus/blob/master/runtime/ficus/impl/gemm.impl.h). +// Here is the original license: +/* + This file is a part of ficus language project. + See ficus/LICENSE for the licensing terms +*/ + +#ifndef OPENCV_DNN_FAST_GEMM_HPP +#define OPENCV_DNN_FAST_GEMM_HPP + +#include "opencv2/core/hal/intrin.hpp" +#include + +namespace cv { namespace dnn { + +struct FastGemmOpt { + bool use_avx; + bool use_avx2; + bool use_neon; + bool use_lasx; + + FastGemmOpt() { + use_avx = false; + use_avx2 = false; + use_neon = false; + use_lasx = false; + } + + void init() { + use_avx = checkHardwareSupport(CPU_AVX); + use_avx2 = checkHardwareSupport(CPU_AVX2); + use_neon = checkHardwareSupport(CPU_NEON); + use_lasx = checkHardwareSupport(CPU_LASX); + } + + bool all() { + return use_avx || use_avx2 || use_neon || use_lasx; + } +}; + +void fastGemmPackB(const Mat &m, std::vector &packed_B, bool trans, FastGemmOpt &opt); + +void fastGemm(bool trans_a, int M, int N, int K, + float alpha, const float *A, int lda, + const float *packed_B, float beta, + float *C, int ldc, FastGemmOpt &opt); +void fastGemm(bool trans_a, bool trans_b, int ma, int na, int mb, int nb, + float alpha, const float *A, int lda0, int lda1, const float *B, int ldb0, int ldb1, + float beta, float *C, int ldc, FastGemmOpt &opt); +void fastGemm(bool trans_a, bool trans_b, + float alpha, const Mat &A, const Mat &B, + float beta, Mat &C, FastGemmOpt &opt); + +// FIXME: B needs to 2d for now. Support nd (n>=2) B in the future. +void fastGemmBatched(bool trans_a, bool trans_b, + float alpha, const Mat &A, const Mat &B, + float beta, Mat &C, FastGemmOpt &opt); + +}} // cv::dnn + +#endif // OPENCV_DNN_FAST_GEMM_HPP diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.default.hpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.default.hpp new file mode 100644 index 0000000000..6a8ef6b590 --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.default.hpp @@ -0,0 +1,393 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// This file is modified from the ficus (https://github.com/vpisarev/ficus/blob/master/runtime/ficus/impl/gemm.impl.h). +// Here is the original license: +/* + This file is a part of ficus language project. + See ficus/LICENSE for the licensing terms +*/ + +#include +#include // parallel_for_ + +#define FAST_GEMM_DEFAULT_STORAGE (1<<20) // 2^20 +#define FAST_GEMM_DEFAULT_MAX_STACKBUF (1 << 14) + +#define FAST_GEMM_DEFAULT_F32_MC 64 +#define FAST_GEMM_DEFAULT_F32_NC 240 +#define FAST_GEMM_DEFAULT_F32_MR 8 +#define FAST_GEMM_DEFAULT_F32_NR 12 +#define FAST_GEMM_DEFAULT_F32_PACKED_STRIDE_K 256 + +#define FAST_GEMM_DEFAULT_IMPLEMENT_PACK(N, suffix, styp, dtyp) \ +static void fast_gemm_pack##N##suffix( int m, int k, const void* A_, \ + int lda0, int lda1, void* packA_ ) \ +{ \ + const styp* A = (const styp*)A_; \ + dtyp* packA = (dtyp*)packA_; \ + for( int i = 0; i < m; i += N ) { \ + if (i + N-1 < m) { \ + const styp* a_ptr = A + lda0*i; \ + for( int j = 0; j < k*lda1; packA += N, j += lda1 ) \ + { \ + FAST_GEMM_DEFAULT_LOAD_TO_BUF_##N(styp); \ + FAST_GEMM_DEFAULT_PACK##suffix##_##N(buf, packA); \ + } \ + } else { \ + const styp* a_ptr[N]; \ + for (int k = 0; k < N; k++) a_ptr[k] = A + lda0*(i+k < m ? i+k : i); \ + for( int j = 0; j < k*lda1; packA += N, j += lda1 ) \ + { \ + FAST_GEMM_DEFAULT_LOAD_TO_BUF_BORDERS_##N(styp); \ + FAST_GEMM_DEFAULT_PACK##suffix##_##N(buf, packA); \ + } \ + } \ + } \ +} + +#define FAST_GEMM_DEFAULT_LOAD_TO_BUF_8(styp) \ + styp buf[] = { \ + a_ptr[j], a_ptr[j+lda0], a_ptr[j+lda0*2], a_ptr[j+lda0*3], \ + a_ptr[j+lda0*4], a_ptr[j+lda0*5], a_ptr[j+lda0*6], a_ptr[j+lda0*7] } + +#define FAST_GEMM_DEFAULT_LOAD_TO_BUF_BORDERS_8(styp) \ + styp buf[] = { \ + a_ptr[0][j], a_ptr[1][j], a_ptr[2][j], a_ptr[3][j], \ + a_ptr[4][j], a_ptr[5][j], a_ptr[6][j], a_ptr[7][j] } + +#define FAST_GEMM_DEFAULT_LOAD_TO_BUF_12(styp) \ + styp buf[] = { \ + a_ptr[j], a_ptr[j+lda0], a_ptr[j+lda0*2], a_ptr[j+lda0*3], \ + a_ptr[j+lda0*4], a_ptr[j+lda0*5], a_ptr[j+lda0*6], a_ptr[j+lda0*7], \ + a_ptr[j+lda0*8], a_ptr[j+lda0*9], a_ptr[j+lda0*10], a_ptr[j+lda0*11] } + +#define FAST_GEMM_DEFAULT_LOAD_TO_BUF_BORDERS_12(styp) \ + styp buf[] = { \ + a_ptr[0][j], a_ptr[1][j], a_ptr[2][j], a_ptr[3][j], \ + a_ptr[4][j], a_ptr[5][j], a_ptr[6][j], a_ptr[7][j], \ + a_ptr[8][j], a_ptr[9][j], a_ptr[10][j], a_ptr[11][j] } + +#define FAST_GEMM_DEFAULT_PACK_COPY(src, dst, N) \ + memcpy((dst), (src), N*sizeof(src[0])) +#define FAST_GEMM_DEFAULT_PACK_f32_8(src, dst) FAST_GEMM_DEFAULT_PACK_COPY((src), (dst), 8) +#define FAST_GEMM_DEFAULT_PACK_f32_12(src, dst) FAST_GEMM_DEFAULT_PACK_COPY((src), (dst), 12) + +namespace cv { namespace dnn { namespace cpu_baseline { + +int fastGemmPackBSize(int N, int K); + +void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0, int ldb1, int esz); + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *B, int ldb0, int ldb1, + float beta, char *C, int ldc, int esz); +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *packed_B, float beta, char *C, int ldc, int esz); + +FAST_GEMM_DEFAULT_IMPLEMENT_PACK(8, _f32, float, float) +FAST_GEMM_DEFAULT_IMPLEMENT_PACK(12, _f32, float, float) + +int fastGemmPackBSize(int N, int K) { + int GEMM_NC = FAST_GEMM_DEFAULT_F32_NC, GEMM_NR = FAST_GEMM_DEFAULT_F32_NR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + + return static_cast((N + NC - 1) / NC) * NC * K; +} + +void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0, int ldb1, int esz) { + int GEMM_NC = FAST_GEMM_DEFAULT_F32_NC, GEMM_NR = FAST_GEMM_DEFAULT_F32_NR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = std::min(FAST_GEMM_DEFAULT_F32_PACKED_STRIDE_K, K); + + int n_tiles = (N + NC - 1) / NC; + for (int r = 0; r < n_tiles; ++r) { + int j0 = r * NC; + int nc = N - j0 < NC ? N - j0 : NC; + int _nc = static_cast((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz; + for (int k = 0; k < K; k += KC) { + int kc = K - k < KC ? K - k : KC; + fast_gemm_pack12_f32(nc, kc, B + (k * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_B); + packed_B += _nc * kc; + } + } +} + +#if CV_SIMD128 +static void fast_gemm8x12_f32(int k, const char *a_, const char *b_, + char *c_, int ldc, float alpha) { + const float* a = (const float*)a_; + const float* b = (const float*)b_; + float* c = (float*)c_; + + v_float32x4 s00 = v_setzero_f32(), s01 = s00, s02 = s00; + v_float32x4 s10 = s00, s11 = s00, s12 = s00; + v_float32x4 s20 = s00, s21 = s00, s22 = s00; + v_float32x4 s30 = s00, s31 = s00, s32 = s00; + v_float32x4 s40 = s00, s41 = s00, s42 = s00; + v_float32x4 s50 = s00, s51 = s00, s52 = s00; + v_float32x4 s60 = s00, s61 = s00, s62 = s00; + v_float32x4 s70 = s00, s71 = s00, s72 = s00; + + for(int p = 0; p < k; p++, a += FAST_GEMM_DEFAULT_F32_MR, b += FAST_GEMM_DEFAULT_F32_NR) { + v_float32x4 b0 = v_load(b), b1 = v_load(b + 4), b2 = v_load(b + 8); + + v_float32x4 a0 = v_setall_f32(*a); + s00 = v_fma(b0, a0, s00); + s01 = v_fma(b1, a0, s01); + s02 = v_fma(b2, a0, s02); + v_float32x4 a1 = v_setall_f32(*(a + 1)); + s10 = v_fma(b0, a1, s10); + s11 = v_fma(b1, a1, s11); + s12 = v_fma(b2, a1, s12); + + v_float32x4 a2 = v_setall_f32(*(a + 2)); + s20 = v_fma(b0, a2, s20); + s21 = v_fma(b1, a2, s21); + s22 = v_fma(b2, a2, s22); + v_float32x4 a3 = v_setall_f32(*(a + 3)); + s30 = v_fma(b0, a3, s30); + s31 = v_fma(b1, a3, s31); + s32 = v_fma(b2, a3, s32); + + a0 = v_setall_f32(*(a + 4)); + s40 = v_fma(b0, a0, s40); + s41 = v_fma(b1, a0, s41); + s42 = v_fma(b2, a0, s42); + a1 = v_setall_f32(*(a + 5)); + s50 = v_fma(b0, a1, s50); + s51 = v_fma(b1, a1, s51); + s52 = v_fma(b2, a1, s52); + + a2 = v_setall_f32(*(a + 6)); + s60 = v_fma(b0, a2, s60); + s61 = v_fma(b1, a2, s61); + s62 = v_fma(b2, a2, s62); + a3 = v_setall_f32(*(a + 7)); + s70 = v_fma(b0, a3, s70); + s71 = v_fma(b1, a3, s71); + s72 = v_fma(b2, a3, s72); + } + + v_float32x4 c0, c1, c2, c3, c4, c5, v_alpha = v_setall_f32(alpha); +#define FAST_GEMM_FINALE(row0, row1) \ + c0 = v_load(c + row0 * ldc); \ + c1 = v_load(c + row0 * ldc + 4); \ + c2 = v_load(c + row0 * ldc + 8); \ + c3 = v_load(c + row1 * ldc); \ + c4 = v_load(c + row1 * ldc + 4); \ + c5 = v_load(c + row1 * ldc + 8); \ + c0 = v_fma(s##row0##0, v_alpha, c0); \ + c1 = v_fma(s##row0##1, v_alpha, c1); \ + c2 = v_fma(s##row0##2, v_alpha, c2); \ + c3 = v_fma(s##row1##0, v_alpha, c3); \ + c4 = v_fma(s##row1##1, v_alpha, c4); \ + c5 = v_fma(s##row1##2, v_alpha, c5); \ + v_store(c + row0 * ldc, c0); \ + v_store(c + row0 * ldc + 4, c1); \ + v_store(c + row0 * ldc + 8, c2); \ + v_store(c + row1 * ldc, c3); \ + v_store(c + row1 * ldc + 4, c4); \ + v_store(c + row1 * ldc + 8, c5); + + FAST_GEMM_FINALE(0, 1); + FAST_GEMM_FINALE(2, 3); + FAST_GEMM_FINALE(4, 5); + FAST_GEMM_FINALE(6, 7); +#undef FAST_GEMM_FINALE +} + +#else +static void fast_gemm_f32(int k, const char *a_, const char *b_, + char *c_, int ldc, float alpha) { + const float* a = (const float*)a_; + const float* b = (const float*)b_; + float* c = (float*)c_; + + float sbuf[FAST_GEMM_DEFAULT_F32_MR * FAST_GEMM_DEFAULT_F32_NR]; + memset(sbuf, 0, sizeof(sbuf)); + for(int p = 0; p < k; p++) { + for( int i = 0; i < FAST_GEMM_DEFAULT_F32_MR; i++ ) { + float ai = a[FAST_GEMM_DEFAULT_F32_MR * p + i]; + for( int j = 0; j < FAST_GEMM_DEFAULT_F32_NR; j++ ) + sbuf[i * FAST_GEMM_DEFAULT_F32_NR + j] += b[FAST_GEMM_DEFAULT_F32_NR * p + j] * ai; + } + } + for (int i = 0; i < FAST_GEMM_DEFAULT_F32_MR; i++) { + for (int j = 0; j < FAST_GEMM_DEFAULT_F32_NR; j++) + c[i * ldc + j] += alpha * sbuf[i * FAST_GEMM_DEFAULT_F32_NR + j]; + } +} +#endif // CV_SIMD128 + +static void fast_gemm_macro_kernel(int m, int n, int k, + const char *packed_A, const char *packed_B, + float alpha, char *c, int ldc0, int esz) { + int ldc0_esz = ldc0 * esz; + + double tempC[FAST_GEMM_DEFAULT_F32_MR * FAST_GEMM_DEFAULT_F32_NR]; // make sure the buffer is big enough + for(int i = 0; i < m; i += FAST_GEMM_DEFAULT_F32_MR) { + for(int j = 0; j < n; j += FAST_GEMM_DEFAULT_F32_NR) { + char* cptr0 = &c[i * ldc0_esz + j * esz]; + char* cptr = cptr0; + int ldc = ldc0; + int mr = m - i < FAST_GEMM_DEFAULT_F32_MR ? m - i : FAST_GEMM_DEFAULT_F32_MR; + int nr = n - j < FAST_GEMM_DEFAULT_F32_NR ? n - j : FAST_GEMM_DEFAULT_F32_NR; + int nr_esz = nr * esz; + bool partial = (bool)((mr < FAST_GEMM_DEFAULT_F32_MR) | (nr < FAST_GEMM_DEFAULT_F32_NR)); + if (partial) { + memset(tempC, 0, sizeof(tempC)); + cptr = (char *)tempC; + ldc = FAST_GEMM_DEFAULT_F32_NR; + for(int p = 0; p < mr; p++) + memcpy(cptr + p * (ldc * esz), cptr0 + p * ldc0_esz, nr_esz); + } +#if CV_SIMD128 + fast_gemm8x12_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); +#else + fast_gemm_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); +#endif + + if (partial) { + for(int p = 0; p < mr; p++) + memcpy(cptr0 + p * ldc0_esz, cptr + p * (ldc * esz), nr_esz); + } + } + } +} + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *B, int ldb0, int ldb1, + float beta, char *C, int ldc, int esz) { + int GEMM_MC = FAST_GEMM_DEFAULT_F32_MC, + GEMM_NC = FAST_GEMM_DEFAULT_F32_NC, + GEMM_MR = FAST_GEMM_DEFAULT_F32_MR, + GEMM_NR = FAST_GEMM_DEFAULT_F32_NR; + + int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = FAST_GEMM_DEFAULT_STORAGE / ((MC + NC) * esz); + KC = KC > 8 ? KC : 8; + KC = KC < K ? KC : K; + + size_t buff_size = KC * (MC + NC) * esz; + bool use_stackbuff = buff_size <= FAST_GEMM_DEFAULT_MAX_STACKBUF; + int m_tiles = (M + MC - 1) / MC; + int n_tiles = (N + NC - 1) / NC; + int total_tiles = m_tiles * n_tiles; + + auto fn = [&](const Range &r) { + char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size)); + char* packed_b = packed_a + KC * MC * esz; + int start = r.start; + int end = r.end; + + for (int tile_idx = start; tile_idx < end; tile_idx++) { + int i0 = (tile_idx / n_tiles) * MC; + int j0 = (tile_idx % n_tiles) * NC; + int mc = M - i0 < MC ? M - i0 : MC; + int nc = N - j0 < NC ? N - j0 : NC; + int ldc_block = ldc; + char* c_block = C + (i0 * ldc + j0) * esz; + + if (beta == 0.f) { + for(int i = 0; i < mc; i++) + memset(c_block + i * ldc_block * esz, 0, nc * esz); + } else if (beta != 1.f) { + for(int i = 0; i < mc; i++) { + float* c_i = (float*)c_block + i * ldc_block; + for(int j = 0; j < nc; j++) + c_i[j] *= beta; + } + } + + for(int k0 = 0; k0 < K; k0 += KC) + { + int kc = K - k0 < KC ? K - k0 : KC; + fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); + fast_gemm_pack12_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b); + fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b, alpha, c_block, ldc_block, esz); + } + } + + if (!use_stackbuff) { + free(packed_a); + } + }; + + int total = total_tiles; + int cost_per_thread = static_cast((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR)); + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *packed_B, float beta, char *C, int ldc, int esz) { + int GEMM_MC = FAST_GEMM_DEFAULT_F32_MC, + GEMM_NC = FAST_GEMM_DEFAULT_F32_NC, + GEMM_MR = FAST_GEMM_DEFAULT_F32_MR, + GEMM_NR = FAST_GEMM_DEFAULT_F32_NR; + + int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = std::min(FAST_GEMM_DEFAULT_F32_PACKED_STRIDE_K, K); + + size_t buff_size = KC * MC * esz; + bool use_stackbuff = buff_size <= FAST_GEMM_DEFAULT_MAX_STACKBUF; + int m_tiles = (M + MC - 1) / MC; + int n_tiles = (N + NC - 1) / NC; + int total_tiles = m_tiles * n_tiles; + + auto fn = [&](const Range &r) { + char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size)); // TODO: use AutoBuffer + const char *packed_b_ = packed_B; + int start = r.start; + int end = r.end; + + for (int tile_idx = start; tile_idx < end; tile_idx++) { + int i0 = (tile_idx / n_tiles) * MC; + int j0 = (tile_idx % n_tiles) * NC; + int mc = M - i0 < MC ? M - i0 : MC; + int nc = N - j0 < NC ? N - j0 : NC; + int ldc_block = ldc; + char* c_block = C + (i0 * ldc + j0) * esz; + packed_b_ = packed_B + j0 * K * esz; + + if (beta == 0.f) { + for(int i = 0; i < mc; i++) + memset(c_block + i * ldc_block * esz, 0, nc * esz); + } else if (beta != 1.f) { + for(int i = 0; i < mc; i++) { + float* c_i = (float*)c_block + i * ldc_block; + for(int j = 0; j < nc; j++) + c_i[j] *= beta; + } + } + + int _nc = static_cast((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz; + for(int k0 = 0; k0 < K; k0 += KC) + { + int kc = K - k0 < KC ? K - k0 : KC; + fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); + fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b_, alpha, c_block, ldc_block, esz); + packed_b_ += _nc * kc; + } + } + + if (!use_stackbuff) { + free(packed_a); + } + }; + + int total = total_tiles; + int cost_per_thread = static_cast((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR)); + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +}}} // cv::dnn::cpu_baseline diff --git a/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp b/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp new file mode 100644 index 0000000000..99a7d3b2d7 --- /dev/null +++ b/modules/dnn/src/layers/cpu_kernels/fast_gemm_kernels.simd.hpp @@ -0,0 +1,1059 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// This file is modified from the ficus (https://github.com/vpisarev/ficus/blob/master/runtime/ficus/impl/gemm.impl.h). +// Here is the original license: +/* + This file is a part of ficus language project. + See ficus/LICENSE for the licensing terms +*/ + +#include +#include // parallel_for_ + +#define FAST_GEMM_STORAGE (1<<20) // 2^20 +#define FAST_GEMM_MAX_STACKBUF (1 << 14) + +#if CV_NEON +#define FAST_GEMM_F32_MC 64 +#define FAST_GEMM_F32_NC 240 +#elif CV_AVX +#define FAST_GEMM_F32_MC 60 +#define FAST_GEMM_F32_NC 320 +#elif CV_LASX +#define FAST_GEMM_F32_MC 48 +#define FAST_GEMM_F32_NC 128 +#endif + +// micro kernel size +#if CV_NEON && CV_NEON_AARCH64 +#define FAST_GEMM_F32_MR 8 +#define FAST_GEMM_F32_NR 12 +#elif CV_NEON +#define FAST_GEMM_F32_MR 4 +#define FAST_GEMM_F32_NR 12 +#elif CV_AVX +#define FAST_GEMM_F32_MR 12 +#define FAST_GEMM_F32_NR 8 +#elif CV_LASX +#define FAST_GEMM_F32_MR 12 +#define FAST_GEMM_F32_NR 16 +#endif + +#if CV_NEON +#define FAST_GEMM_F32_PACKED_STRIDE_K 64 +#elif CV_AVX +#define FAST_GEMM_F32_PACKED_STRIDE_K 128 +#elif CV_LASX +#define FAST_GEMM_F32_PACKED_STRIDE_K 64 +#endif + +#define FAST_GEMM_IMPLEMENT_PACK(N, suffix, styp, dtyp) \ +static void fast_gemm_pack##N##suffix( int m, int k, const void* A_, \ + int lda0, int lda1, void* packA_ ) \ +{ \ + const styp* A = (const styp*)A_; \ + dtyp* packA = (dtyp*)packA_; \ + for( int i = 0; i < m; i += N ) { \ + if (i + N-1 < m) { \ + const styp* a_ptr = A + lda0*i; \ + for( int j = 0; j < k*lda1; packA += N, j += lda1 ) \ + { \ + FAST_GEMM_LOAD_TO_BUF_##N(styp); \ + FAST_GEMM_PACK##suffix##_##N(buf, packA); \ + } \ + } else { \ + const styp* a_ptr[N]; \ + for (int k = 0; k < N; k++) a_ptr[k] = A + lda0*(i+k < m ? i+k : i); \ + for( int j = 0; j < k*lda1; packA += N, j += lda1 ) \ + { \ + FAST_GEMM_LOAD_TO_BUF_BORDERS_##N(styp); \ + FAST_GEMM_PACK##suffix##_##N(buf, packA); \ + } \ + } \ + } \ +} + +#define FAST_GEMM_LOAD_TO_BUF_4(styp) \ + styp buf[] = { \ + a_ptr[j], a_ptr[j+lda0], a_ptr[j+lda0*2], a_ptr[j+lda0*3] } + +#define FAST_GEMM_LOAD_TO_BUF_BORDERS_4(styp) \ + styp buf[] = { \ + a_ptr[0][j], a_ptr[1][j], a_ptr[2][j], a_ptr[3][j] } + +#define FAST_GEMM_LOAD_TO_BUF_8(styp) \ + styp buf[] = { \ + a_ptr[j], a_ptr[j+lda0], a_ptr[j+lda0*2], a_ptr[j+lda0*3], \ + a_ptr[j+lda0*4], a_ptr[j+lda0*5], a_ptr[j+lda0*6], a_ptr[j+lda0*7] } + +#define FAST_GEMM_LOAD_TO_BUF_BORDERS_8(styp) \ + styp buf[] = { \ + a_ptr[0][j], a_ptr[1][j], a_ptr[2][j], a_ptr[3][j], \ + a_ptr[4][j], a_ptr[5][j], a_ptr[6][j], a_ptr[7][j] } + +#define FAST_GEMM_LOAD_TO_BUF_12(styp) \ + styp buf[] = { \ + a_ptr[j], a_ptr[j+lda0], a_ptr[j+lda0*2], a_ptr[j+lda0*3], \ + a_ptr[j+lda0*4], a_ptr[j+lda0*5], a_ptr[j+lda0*6], a_ptr[j+lda0*7], \ + a_ptr[j+lda0*8], a_ptr[j+lda0*9], a_ptr[j+lda0*10], a_ptr[j+lda0*11] } + +#define FAST_GEMM_LOAD_TO_BUF_BORDERS_12(styp) \ + styp buf[] = { \ + a_ptr[0][j], a_ptr[1][j], a_ptr[2][j], a_ptr[3][j], \ + a_ptr[4][j], a_ptr[5][j], a_ptr[6][j], a_ptr[7][j], \ + a_ptr[8][j], a_ptr[9][j], a_ptr[10][j], a_ptr[11][j] } + +#define FAST_GEMM_LOAD_TO_BUF_16(styp) \ + styp buf[] = { \ + a_ptr[j], a_ptr[j+lda0], a_ptr[j+lda0*2], a_ptr[j+lda0*3], \ + a_ptr[j+lda0*4], a_ptr[j+lda0*5], a_ptr[j+lda0*6], a_ptr[j+lda0*7], \ + a_ptr[j+lda0*8], a_ptr[j+lda0*9], a_ptr[j+lda0*10], a_ptr[j+lda0*11], \ + a_ptr[j+lda0*12], a_ptr[j+lda0*13], a_ptr[j+lda0*14], a_ptr[j+lda0*15] } + +#define FAST_GEMM_LOAD_TO_BUF_BORDERS_16(styp) \ + styp buf[] = { \ + a_ptr[0][j], a_ptr[1][j], a_ptr[2][j], a_ptr[3][j], \ + a_ptr[4][j], a_ptr[5][j], a_ptr[6][j], a_ptr[7][j], \ + a_ptr[8][j], a_ptr[9][j], a_ptr[10][j], a_ptr[11][j], \ + a_ptr[12][j], a_ptr[13][j], a_ptr[14][j], a_ptr[15][j] } + +#define FAST_GEMM_PACK_COPY(src, dst, N) \ + memcpy((dst), (src), N*sizeof(src[0])) +#define FAST_GEMM_PACK_f32_4(src, dst) FAST_GEMM_PACK_COPY((src), (dst), 4) +#define FAST_GEMM_PACK_f32_8(src, dst) FAST_GEMM_PACK_COPY((src), (dst), 8) +#define FAST_GEMM_PACK_f32_12(src, dst) FAST_GEMM_PACK_COPY((src), (dst), 12) +#define FAST_GEMM_PACK_f32_16(src, dst) FAST_GEMM_PACK_COPY((src), (dst), 16) + +namespace cv { namespace dnn { + +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +// TODO: type to size_t +int fastGemmPackBSize(int N, int K); + +void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0, int ldb1, int esz); + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *B, int ldb0, int ldb1, + float beta, char *C, int ldc, int esz); +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *packed_B, float beta, char *C, int ldc, int esz); + +// NEON (AARCH64: 32 x 128-bit registers, armv7: 16 x 128-bit registers) +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_NEON + +#if CV_NEON_AARCH64 +FAST_GEMM_IMPLEMENT_PACK(8, _f32, float, float) +#else +FAST_GEMM_IMPLEMENT_PACK(4, _f32, float, float) +#endif +FAST_GEMM_IMPLEMENT_PACK(12, _f32, float, float) + +int fastGemmPackBSize(int N, int K) { + int GEMM_NC = FAST_GEMM_F32_NC, GEMM_NR = FAST_GEMM_F32_NR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + + return static_cast((N + NC - 1) / NC) * NC * K; +} + +void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0, int ldb1, int esz) { + int GEMM_NC = FAST_GEMM_F32_NC, GEMM_NR = FAST_GEMM_F32_NR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K); + + int n_tiles = (N + NC - 1) / NC; + for (int r = 0; r < n_tiles; ++r) { + int j0 = r * NC; + int nc = N - j0 < NC ? N - j0 : NC; + int _nc = static_cast((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz; + for (int k = 0; k < K; k += KC) { + int kc = K - k < KC ? K - k : KC; + fast_gemm_pack12_f32(nc, kc, B + (k * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_B); + packed_B += _nc * kc; + } + } +} + +#if CV_NEON_AARCH64 +static void fast_gemm8x12_f32(int k, const char *a_, const char *b_, + char *c_, int ldc, float alpha) { + const float* a = (const float*)a_; + const float* b = (const float*)b_; + float* c = (float*)c_; + + float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00; + float32x4_t s10 = s00, s11 = s00, s12 = s00; + float32x4_t s20 = s00, s21 = s00, s22 = s00; + float32x4_t s30 = s00, s31 = s00, s32 = s00; + float32x4_t s40 = s00, s41 = s00, s42 = s00; + float32x4_t s50 = s00, s51 = s00, s52 = s00; + float32x4_t s60 = s00, s61 = s00, s62 = s00; + float32x4_t s70 = s00, s71 = s00, s72 = s00; + + for(int p = 0; p < k; p++, a += FAST_GEMM_F32_MR, b += FAST_GEMM_F32_NR) + { + float32x4_t a0 = vld1q_f32(a); + float32x4_t b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8); + + s00 = vfmaq_laneq_f32(s00, b0, a0, 0); + s01 = vfmaq_laneq_f32(s01, b1, a0, 0); + s02 = vfmaq_laneq_f32(s02, b2, a0, 0); + s10 = vfmaq_laneq_f32(s10, b0, a0, 1); + s11 = vfmaq_laneq_f32(s11, b1, a0, 1); + s12 = vfmaq_laneq_f32(s12, b2, a0, 1); + + s20 = vfmaq_laneq_f32(s20, b0, a0, 2); + s21 = vfmaq_laneq_f32(s21, b1, a0, 2); + s22 = vfmaq_laneq_f32(s22, b2, a0, 2); + s30 = vfmaq_laneq_f32(s30, b0, a0, 3); + s31 = vfmaq_laneq_f32(s31, b1, a0, 3); + s32 = vfmaq_laneq_f32(s32, b2, a0, 3); + + a0 = vld1q_f32(a + 4); + + s40 = vfmaq_laneq_f32(s40, b0, a0, 0); + s41 = vfmaq_laneq_f32(s41, b1, a0, 0); + s42 = vfmaq_laneq_f32(s42, b2, a0, 0); + s50 = vfmaq_laneq_f32(s50, b0, a0, 1); + s51 = vfmaq_laneq_f32(s51, b1, a0, 1); + s52 = vfmaq_laneq_f32(s52, b2, a0, 1); + + s60 = vfmaq_laneq_f32(s60, b0, a0, 2); + s61 = vfmaq_laneq_f32(s61, b1, a0, 2); + s62 = vfmaq_laneq_f32(s62, b2, a0, 2); + s70 = vfmaq_laneq_f32(s70, b0, a0, 3); + s71 = vfmaq_laneq_f32(s71, b1, a0, 3); + s72 = vfmaq_laneq_f32(s72, b2, a0, 3); + } + + float32x4_t c0, c1, c2, c3, c4, c5, v_alpha = vdupq_n_f32(alpha); +#define FAST_GEMM_FINALE(row0, row1) \ + c0 = vld1q_f32(c + row0 * ldc); \ + c1 = vld1q_f32(c + row0 * ldc + 4); \ + c2 = vld1q_f32(c + row0 * ldc + 8); \ + c3 = vld1q_f32(c + row1 * ldc); \ + c4 = vld1q_f32(c + row1 * ldc + 4); \ + c5 = vld1q_f32(c + row1 * ldc + 8); \ + c0 = vfmaq_f32(c0, s##row0##0, v_alpha); \ + c1 = vfmaq_f32(c1, s##row0##1, v_alpha); \ + c2 = vfmaq_f32(c2, s##row0##2, v_alpha); \ + c3 = vfmaq_f32(c3, s##row1##0, v_alpha); \ + c4 = vfmaq_f32(c4, s##row1##1, v_alpha); \ + c5 = vfmaq_f32(c5, s##row1##2, v_alpha); \ + vst1q_f32(c + row0 * ldc, c0); \ + vst1q_f32(c + row0 * ldc + 4, c1); \ + vst1q_f32(c + row0 * ldc + 8, c2); \ + vst1q_f32(c + row1 * ldc, c3); \ + vst1q_f32(c + row1 * ldc + 4, c4); \ + vst1q_f32(c + row1 * ldc + 8, c5); + + FAST_GEMM_FINALE(0, 1); + FAST_GEMM_FINALE(2, 3); + FAST_GEMM_FINALE(4, 5); + FAST_GEMM_FINALE(6, 7); +#undef FAST_GEMM_FINALE +} + +#else // CV_NEON_AARCH64 +static void fast_gemm4x12_f32(int k, const char *a_, const char *b_, + char *c_, int ldc, float alpha) { + const float* a = (const float*)a_; + const float* b = (const float*)b_; + float* c = (float*)c_; + + float32x4_t s00 = vdupq_n_f32(0.f), s01 = s00, s02 = s00, + s10 = s00, s11 = s00, s12 = s00, + s20 = s00, s21 = s00, s22 = s00, + s30 = s00, s31 = s00, s32 = s00; + + for(int p = 0; p < k; p++, a += FAST_GEMM_F32_MR, b += FAST_GEMM_F32_NR) + { + float32x4_t b0 = vld1q_f32(b), b1 = vld1q_f32(b + 4), b2 = vld1q_f32(b + 8); + + float32x4_t a0 = vld1q_dup_f32(a); + s00 = vmlaq_f32(a0, b0, s00); + s01 = vmlaq_f32(a0, b1, s01); + s02 = vmlaq_f32(a0, b2, s02); + + a0 = vld1q_dup_f32(a + 1); + s10 = vmlaq_f32(a0, b0, s10); + s11 = vmlaq_f32(a0, b1, s11); + s12 = vmlaq_f32(a0, b2, s12); + + a0 = vld1q_dup_f32(a + 2); + s20 = vmlaq_f32(a0, b0, s20); + s21 = vmlaq_f32(a0, b1, s21); + s22 = vmlaq_f32(a0, b2, s22); + + a0 = vld1q_dup_f32(a + 3); + s30 = vmlaq_f32(a0, b0, s30); + s31 = vmlaq_f32(a0, b1, s31); + s32 = vmlaq_f32(a0, b2, s32); + } + + float32x4_t c0, c1, c2, v_alpha = vdupq_n_f32(alpha); +#define FAST_GEMM_FINALE(row0) \ + c0 = vld1q_f32(c + row0 * ldc); \ + c1 = vld1q_f32(c + row0 * ldc + 4); \ + c2 = vld1q_f32(c + row0 * ldc + 8); \ + c0 = vmlaq_f32(c0, s##row0##0, v_alpha); \ + c1 = vmlaq_f32(c1, s##row0##1, v_alpha); \ + c2 = vmlaq_f32(c2, s##row0##2, v_alpha); \ + vst1q_f32(c + row0 * ldc, c0); \ + vst1q_f32(c + row0 * ldc + 4, c1); \ + vst1q_f32(c + row0 * ldc + 8, c2); + + FAST_GEMM_FINALE(0); + FAST_GEMM_FINALE(1); + FAST_GEMM_FINALE(2); + FAST_GEMM_FINALE(3); +#undef FAST_GEMM_FINALE +} + +#endif // micro kernel CV_NEON_AARCH64 + +static void fast_gemm_macro_kernel(int m, int n, int k, + const char *packed_A, const char *packed_B, + float alpha, char *c, int ldc0, int esz) { + int ldc0_esz = ldc0 * esz; + + double tempC[FAST_GEMM_F32_MR * FAST_GEMM_F32_NR]; // make sure the buffer is big enough + for(int i = 0; i < m; i += FAST_GEMM_F32_MR) { + for(int j = 0; j < n; j += FAST_GEMM_F32_NR) { + char* cptr0 = &c[i * ldc0_esz + j * esz]; + char* cptr = cptr0; + int ldc = ldc0; + int mr = m - i < FAST_GEMM_F32_MR ? m - i : FAST_GEMM_F32_MR; + int nr = n - j < FAST_GEMM_F32_NR ? n - j : FAST_GEMM_F32_NR; + int nr_esz = nr * esz; + bool partial = (bool)((mr < FAST_GEMM_F32_MR) | (nr < FAST_GEMM_F32_NR)); + if (partial) { + memset(tempC, 0, sizeof(tempC)); + cptr = (char *)tempC; + ldc = FAST_GEMM_F32_NR; + for(int p = 0; p < mr; p++) + memcpy(cptr + p * (ldc * esz), cptr0 + p * ldc0_esz, nr_esz); + } +#if CV_NEON_AARCH64 + fast_gemm8x12_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); +#else + fast_gemm4x12_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); +#endif + + if (partial) { + for(int p = 0; p < mr; p++) + memcpy(cptr0 + p * ldc0_esz, cptr + p * (ldc * esz), nr_esz); + } + } + } +} + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *B, int ldb0, int ldb1, + float beta, char *C, int ldc, int esz) { + int GEMM_MC = FAST_GEMM_F32_MC, + GEMM_NC = FAST_GEMM_F32_NC, + GEMM_MR = FAST_GEMM_F32_MR, + GEMM_NR = FAST_GEMM_F32_NR; + + int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = FAST_GEMM_STORAGE / ((MC + NC) * esz); + KC = KC > 8 ? KC : 8; + KC = KC < K ? KC : K; + + size_t buff_size = KC * (MC + NC) * esz; + bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF; + int m_tiles = (M + MC - 1) / MC; + int n_tiles = (N + NC - 1) / NC; + int total_tiles = m_tiles * n_tiles; + + auto fn = [&](const Range &r) { + char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size)); + char* packed_b = packed_a + KC * MC * esz; + int start = r.start; + int end = r.end; + + for (int tile_idx = start; tile_idx < end; tile_idx++) { + int i0 = (tile_idx / n_tiles) * MC; + int j0 = (tile_idx % n_tiles) * NC; + int mc = M - i0 < MC ? M - i0 : MC; + int nc = N - j0 < NC ? N - j0 : NC; + int ldc_block = ldc; + char* c_block = C + (i0 * ldc + j0) * esz; + + if (beta == 0.f) { + for(int i = 0; i < mc; i++) + memset(c_block + i * ldc_block * esz, 0, nc * esz); + } else if (beta != 1.f) { + for(int i = 0; i < mc; i++) { + float* c_i = (float*)c_block + i * ldc_block; + for(int j = 0; j < nc; j++) + c_i[j] *= beta; + } + } + + for(int k0 = 0; k0 < K; k0 += KC) + { + int kc = K - k0 < KC ? K - k0 : KC; +#if CV_NEON_AARCH64 + fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); +#else + fast_gemm_pack4_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); +#endif + fast_gemm_pack12_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b); + fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b, alpha, c_block, ldc_block, esz); + } + } + + if (!use_stackbuff) { + free(packed_a); + } + }; + + int total = total_tiles; + int cost_per_thread = static_cast((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR)); + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *packed_B, float beta, char *C, int ldc, int esz) { + int GEMM_MC = FAST_GEMM_F32_MC, + GEMM_NC = FAST_GEMM_F32_NC, + GEMM_MR = FAST_GEMM_F32_MR, + GEMM_NR = FAST_GEMM_F32_NR; + + int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K); + + size_t buff_size = KC * MC * esz; + bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF; + int m_tiles = (M + MC - 1) / MC; + int n_tiles = (N + NC - 1) / NC; + int total_tiles = m_tiles * n_tiles; + + auto fn = [&](const Range &r) { + char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size)); // TODO: use AutoBuffer + const char *packed_b_ = packed_B; + int start = r.start; + int end = r.end; + + for (int tile_idx = start; tile_idx < end; tile_idx++) { + int i0 = (tile_idx / n_tiles) * MC; + int j0 = (tile_idx % n_tiles) * NC; + int mc = M - i0 < MC ? M - i0 : MC; + int nc = N - j0 < NC ? N - j0 : NC; + int ldc_block = ldc; + char* c_block = C + (i0 * ldc + j0) * esz; + packed_b_ = packed_B + j0 * K * esz; + + if (beta == 0.f) { + for(int i = 0; i < mc; i++) + memset(c_block + i * ldc_block * esz, 0, nc * esz); + } else if (beta != 1.f) { + for(int i = 0; i < mc; i++) { + float* c_i = (float*)c_block + i * ldc_block; + for(int j = 0; j < nc; j++) + c_i[j] *= beta; + } + } + + int _nc = static_cast((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz; + for(int k0 = 0; k0 < K; k0 += KC) + { + int kc = K - k0 < KC ? K - k0 : KC; +#if CV_NEON_AARCH64 + fast_gemm_pack8_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); +#else + fast_gemm_pack4_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); +#endif + fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b_, alpha, c_block, ldc_block, esz); + packed_b_ += _nc * kc; + } + } + + if (!use_stackbuff) { + free(packed_a); + } + }; + + int total = total_tiles; + int cost_per_thread = static_cast((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR)); + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +#endif // CV_NEON, CV_NEON_AARCH64 + +// AVX and AVX2 (16 x 256-bit registers) +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX + +FAST_GEMM_IMPLEMENT_PACK(8, _f32, float, float) +FAST_GEMM_IMPLEMENT_PACK(12, _f32, float, float) + +int fastGemmPackBSize(int N, int K) { + int GEMM_NC = FAST_GEMM_F32_NC, GEMM_NR = FAST_GEMM_F32_NR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + + return static_cast((N + NC - 1) / NC) * NC * K; +} + +void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0, int ldb1, int esz) { + int GEMM_NC = FAST_GEMM_F32_NC, GEMM_NR = FAST_GEMM_F32_NR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K); + + int n_tiles = (N + NC - 1) / NC; + for (int r = 0; r < n_tiles; ++r) { + int j0 = r * NC; + int nc = N - j0 < NC ? N - j0 : NC; + int _nc = static_cast((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz; + for (int k = 0; k < K; k += KC) { + int kc = K - k < KC ? K - k : KC; + fast_gemm_pack8_f32(nc, kc, B + (k * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_B); + packed_B += _nc * kc; + } + } +} + +#if !CV_FMA3 // AVX workaround for FMA +#undef _mm256_fmadd_ps +#define _mm256_fmadd_ps(a, b, c) _mm256_add_ps(c, _mm256_mul_ps(a, b)) +#endif + +static void fast_gemm12x8_f32(int k, const char *a_, const char *b_, char *c_, int ldc, float alpha) { + const float* a = (const float*)a_; + const float* b = (const float*)b_; + float* c = (float*)c_; + + __m256 s00 = _mm256_setzero_ps(), + s10 = _mm256_setzero_ps(), + s20 = _mm256_setzero_ps(), + s30 = _mm256_setzero_ps(), + s40 = _mm256_setzero_ps(), + s50 = _mm256_setzero_ps(), + s60 = _mm256_setzero_ps(), + s70 = _mm256_setzero_ps(), + s80 = _mm256_setzero_ps(), + s90 = _mm256_setzero_ps(), + s100 = _mm256_setzero_ps(), + s110 = _mm256_setzero_ps(); + for (int p = 0; p < k; p++, a += FAST_GEMM_F32_MR, b += FAST_GEMM_F32_NR) { + __m256 b0 = _mm256_loadu_ps(b); + + __m256 a0 = _mm256_set1_ps(*a); + s00 = _mm256_fmadd_ps(b0, a0, s00); + __m256 a1 = _mm256_set1_ps(*(a + 1)); + s10 = _mm256_fmadd_ps(b0, a1, s10); + __m256 a2 = _mm256_set1_ps(*(a + 2)); + s20 = _mm256_fmadd_ps(b0, a2, s20); + + a0 = _mm256_set1_ps(*(a + 3)); + s30 = _mm256_fmadd_ps(b0, a0, s30); + a1 = _mm256_set1_ps(*(a + 4)); + s40 = _mm256_fmadd_ps(b0, a1, s40); + a2 = _mm256_set1_ps(*(a + 5)); + s50 = _mm256_fmadd_ps(b0, a2, s50); + + a0 = _mm256_set1_ps(*(a + 6)); + s60 = _mm256_fmadd_ps(b0, a0, s60); + a1 = _mm256_set1_ps(*(a + 7)); + s70 = _mm256_fmadd_ps(b0, a1, s70); + a2 = _mm256_set1_ps(*(a + 8)); + s80 = _mm256_fmadd_ps(b0, a2, s80); + + a0 = _mm256_set1_ps(*(a + 9)); + s90 = _mm256_fmadd_ps(b0, a0, s90); + a1 = _mm256_set1_ps(*(a + 10)); + s100 = _mm256_fmadd_ps(b0, a1, s100); + a2 = _mm256_set1_ps(*(a + 11)); + s110 = _mm256_fmadd_ps(b0, a2, s110); + } + + __m256 c0, c1, c2, c3, v_alpha = _mm256_set1_ps(alpha); +#define FAST_GEMM_FINALE(row0, row1, row2, row3) \ + c0 = _mm256_loadu_ps(c + row0 * ldc); \ + c1 = _mm256_loadu_ps(c + row1 * ldc); \ + c2 = _mm256_loadu_ps(c + row2 * ldc); \ + c3 = _mm256_loadu_ps(c + row3 * ldc); \ + c0 = _mm256_fmadd_ps(s##row0##0, v_alpha, c0); \ + c1 = _mm256_fmadd_ps(s##row1##0, v_alpha, c1); \ + c2 = _mm256_fmadd_ps(s##row2##0, v_alpha, c2); \ + c3 = _mm256_fmadd_ps(s##row3##0, v_alpha, c3); \ + _mm256_storeu_ps(c + row0 * ldc, c0); \ + _mm256_storeu_ps(c + row1 * ldc, c1); \ + _mm256_storeu_ps(c + row2 * ldc, c2); \ + _mm256_storeu_ps(c + row3 * ldc, c3); \ + + FAST_GEMM_FINALE(0, 1, 2, 3); + FAST_GEMM_FINALE(4, 5, 6, 7); + FAST_GEMM_FINALE(8, 9, 10, 11); +#undef FAST_GEMM_FINALE +} + +static void fast_gemm_macro_kernel(int m, int n, int k, + const char *packed_A, const char *packed_B, + float alpha, char *c, int ldc0, int esz) { + int ldc0_esz = ldc0 * esz; + + double tempC[FAST_GEMM_F32_MR * FAST_GEMM_F32_NR]; // make sure the buffer is big enough + for(int i = 0; i < m; i += FAST_GEMM_F32_MR) { + for(int j = 0; j < n; j += FAST_GEMM_F32_NR) { + char* cptr0 = &c[i * ldc0_esz + j * esz]; + char* cptr = cptr0; + int ldc = ldc0; + int mr = m - i < FAST_GEMM_F32_MR ? m - i : FAST_GEMM_F32_MR; + int nr = n - j < FAST_GEMM_F32_NR ? n - j : FAST_GEMM_F32_NR; + int nr_esz = nr * esz; + bool partial = (bool)((mr < FAST_GEMM_F32_MR) | (nr < FAST_GEMM_F32_NR)); + if (partial) { + memset(tempC, 0, sizeof(tempC)); + cptr = (char *)tempC; + ldc = FAST_GEMM_F32_NR; + for(int p = 0; p < mr; p++) + memcpy(cptr + p * (ldc * esz), cptr0 + p * ldc0_esz, nr_esz); + } + fast_gemm12x8_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); + + if (partial) { + for(int p = 0; p < mr; p++) + memcpy(cptr0 + p * ldc0_esz, cptr + p * (ldc * esz), nr_esz); + } + } + } +} + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *B, int ldb0, int ldb1, + float beta, char *C, int ldc, int esz) { + int GEMM_MC = FAST_GEMM_F32_MC, + GEMM_NC = FAST_GEMM_F32_NC, + GEMM_MR = FAST_GEMM_F32_MR, + GEMM_NR = FAST_GEMM_F32_NR; + + int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = FAST_GEMM_STORAGE / ((MC + NC) * esz); + KC = KC > 8 ? KC : 8; + KC = KC < K ? KC : K; + + size_t buff_size = KC * (MC + NC) * esz; + bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF; + int m_tiles = (M + MC - 1) / MC; + int n_tiles = (N + NC - 1) / NC; + int total_tiles = m_tiles * n_tiles; + + auto fn = [&](const Range &r) { + char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size)); + char* packed_b = packed_a + KC * MC * esz; + int start = r.start; + int end = r.end; + + for (int tile_idx = start; tile_idx < end; tile_idx++) { + int i0 = (tile_idx / n_tiles) * MC; + int j0 = (tile_idx % n_tiles) * NC; + int mc = M - i0 < MC ? M - i0 : MC; + int nc = N - j0 < NC ? N - j0 : NC; + int ldc_block = ldc; + char* c_block = C + (i0 * ldc + j0) * esz; + + if (beta == 0.f) { + for(int i = 0; i < mc; i++) + memset(c_block + i * ldc_block * esz, 0, nc * esz); + } else if (beta != 1.f) { + for(int i = 0; i < mc; i++) { + float* c_i = (float*)c_block + i * ldc_block; + for(int j = 0; j < nc; j++) + c_i[j] *= beta; + } + } + + for(int k0 = 0; k0 < K; k0 += KC) + { + int kc = K - k0 < KC ? K - k0 : KC; + fast_gemm_pack12_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); + fast_gemm_pack8_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b); + fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b, alpha, c_block, ldc_block, esz); + } + } + + if (!use_stackbuff) { + free(packed_a); + } + }; + + int total = total_tiles; + int cost_per_thread = static_cast((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR)); + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *packed_B, float beta, char *C, int ldc, int esz) { + int GEMM_MC = FAST_GEMM_F32_MC, + GEMM_NC = FAST_GEMM_F32_NC, + GEMM_MR = FAST_GEMM_F32_MR, + GEMM_NR = FAST_GEMM_F32_NR; + + int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K); + + size_t buff_size = KC * MC * esz; + bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF; + int m_tiles = (M + MC - 1) / MC; + int n_tiles = (N + NC - 1) / NC; + int total_tiles = m_tiles * n_tiles; + + auto fn = [&](const Range &r) { + char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size)); // TODO: use AutoBuffer + const char *packed_b_ = packed_B; + int start = r.start; + int end = r.end; + + for (int tile_idx = start; tile_idx < end; tile_idx++) { + int i0 = (tile_idx / n_tiles) * MC; + int j0 = (tile_idx % n_tiles) * NC; + int mc = M - i0 < MC ? M - i0 : MC; + int nc = N - j0 < NC ? N - j0 : NC; + int ldc_block = ldc; + char* c_block = C + (i0 * ldc + j0) * esz; + packed_b_ = packed_B + j0 * K * esz; + + if (beta == 0.f) { + for(int i = 0; i < mc; i++) + memset(c_block + i * ldc_block * esz, 0, nc * esz); + } else if (beta != 1.f) { + for(int i = 0; i < mc; i++) { + float* c_i = (float*)c_block + i * ldc_block; + for(int j = 0; j < nc; j++) + c_i[j] *= beta; + } + } + + int _nc = static_cast((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz; + for(int k0 = 0; k0 < K; k0 += KC) + { + int kc = K - k0 < KC ? K - k0 : KC; + fast_gemm_pack12_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); + fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b_, alpha, c_block, ldc_block, esz); + packed_b_ += _nc * kc; + } + } + + if (!use_stackbuff) { + free(packed_a); + } + }; + + int total = total_tiles; + int cost_per_thread = static_cast((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR)); + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +#endif // CV_AVX, CV_AVX2 + +// LASX (32 x 256-bit registers) +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_LASX + +FAST_GEMM_IMPLEMENT_PACK(12, _f32, float, float) +FAST_GEMM_IMPLEMENT_PACK(16, _f32, float, float) + +int fastGemmPackBSize(int N, int K) { + int GEMM_NC = FAST_GEMM_F32_NC, GEMM_NR = FAST_GEMM_F32_NR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + + return static_cast((N + NC - 1) / NC) * NC * K; +} + +void fastGemmPackBKernel(const char *B, char *packed_B, int N, int K, int ldb0, int ldb1, int esz) { + int GEMM_NC = FAST_GEMM_F32_NC, GEMM_NR = FAST_GEMM_F32_NR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K); + + int n_tiles = (N + NC - 1) / NC; + for (int r = 0; r < n_tiles; ++r) { + int j0 = r * NC; + int nc = N - j0 < NC ? N - j0 : NC; + int _nc = static_cast((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz; + for (int k = 0; k < K; k += KC) { + int kc = K - k < KC ? K - k : KC; + fast_gemm_pack16_f32(nc, kc, B + (k * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_B); + packed_B += _nc * kc; + } + } +} + +static void fast_gemm12x16_f32(int k, const char *a_, const char *b_, char *c_, int ldc, float alpha) { + const float* a = (const float*)a_; + const float* b = (const float*)b_; + float* c = (float*)c_; + + __m256i dummy; + __m256 s00 = (__m256)__lasx_xvxor_v(dummy, dummy), s01 = s00, + s10 = s00, s11 = s00, + s20 = s00, s21 = s00, + s30 = s00, s31 = s00, + s40 = s00, s41 = s00, + s50 = s00, s51 = s00, + s60 = s00, s61 = s00, + s70 = s00, s71 = s00, + s80 = s00, s81 = s00, + s90 = s00, s91 = s00, + s100 = s00, s101 = s00, + s110 = s00, s111 = s00; + for (int p = 0; p < k; p++, a += FAST_GEMM_F32_MR, b += FAST_GEMM_F32_NR) { + __m256 b0 = (__m256)__lasx_xvld(b, 0), b1 = (__m256)__lasx_xvld(b + 8, 0); + + __m256 a0 = _v256_setall_ps(*a); + s00 = __lasx_xvfmadd_s(b0, a0, s00); + s01 = __lasx_xvfmadd_s(b1, a0, s01); + __m256 a1 = _v256_setall_ps(*(a + 1)); + s10 = __lasx_xvfmadd_s(b0, a1, s10); + s11 = __lasx_xvfmadd_s(b1, a1, s11); + __m256 a2 = _v256_setall_ps(*(a + 2)); + s20 = __lasx_xvfmadd_s(b0, a2, s20); + s21 = __lasx_xvfmadd_s(b1, a2, s21); + __m256 a3 = _v256_setall_ps(*(a + 3)); + s30 = __lasx_xvfmadd_s(b0, a3, s30); + s31 = __lasx_xvfmadd_s(b1, a3, s31); + + a0 = _v256_setall_ps(*(a + 4)); + s40 = __lasx_xvfmadd_s(b0, a0, s40); + s41 = __lasx_xvfmadd_s(b1, a0, s41); + a1 = _v256_setall_ps(*(a + 5)); + s50 = __lasx_xvfmadd_s(b0, a1, s50); + s51 = __lasx_xvfmadd_s(b1, a1, s51); + a2 = _v256_setall_ps(*(a + 6)); + s60 = __lasx_xvfmadd_s(b0, a2, s60); + s61 = __lasx_xvfmadd_s(b1, a2, s61); + a3 = _v256_setall_ps(*(a + 7)); + s70 = __lasx_xvfmadd_s(b0, a3, s70); + s71 = __lasx_xvfmadd_s(b1, a3, s71); + + a0 = _v256_setall_ps(*(a + 8)); + s80 = __lasx_xvfmadd_s(b0, a0, s80); + s81 = __lasx_xvfmadd_s(b1, a0, s81); + a1 = _v256_setall_ps(*(a + 9)); + s90 = __lasx_xvfmadd_s(b0, a1, s90); + s91 = __lasx_xvfmadd_s(b1, a1, s91); + a2 = _v256_setall_ps(*(a + 10)); + s100 = __lasx_xvfmadd_s(b0, a2, s100); + s101 = __lasx_xvfmadd_s(b1, a2, s101); + a3 = _v256_setall_ps(*(a + 11)); + s110 = __lasx_xvfmadd_s(b0, a3, s110); + s111 = __lasx_xvfmadd_s(b1, a3, s111); + } + + __m256 c0, c1, c2, c3, c4, c5, c6, c7, v_alpha = _v256_setall_ps(alpha); +#define FAST_GEMM_FINALE(row0, row1, row2, row3) \ + c0 = (__m256)__lasx_xvld(c + row0 * ldc, 0); \ + c1 = (__m256)__lasx_xvld(c + row0 * ldc, 8 * 4); \ + c2 = (__m256)__lasx_xvld(c + row1 * ldc, 0); \ + c3 = (__m256)__lasx_xvld(c + row1 * ldc, 8 * 4); \ + c4 = (__m256)__lasx_xvld(c + row2 * ldc, 0); \ + c5 = (__m256)__lasx_xvld(c + row2 * ldc, 8 * 4); \ + c6 = (__m256)__lasx_xvld(c + row3 * ldc, 0); \ + c7 = (__m256)__lasx_xvld(c + row3 * ldc, 8 * 4); \ + c0 = __lasx_xvfmadd_s(s##row0##0, v_alpha, c0); \ + c1 = __lasx_xvfmadd_s(s##row0##1, v_alpha, c1); \ + c2 = __lasx_xvfmadd_s(s##row1##0, v_alpha, c2); \ + c3 = __lasx_xvfmadd_s(s##row1##1, v_alpha, c3); \ + c4 = __lasx_xvfmadd_s(s##row2##0, v_alpha, c4); \ + c5 = __lasx_xvfmadd_s(s##row2##1, v_alpha, c5); \ + c6 = __lasx_xvfmadd_s(s##row3##0, v_alpha, c6); \ + c7 = __lasx_xvfmadd_s(s##row3##1, v_alpha, c7); \ + __lasx_xvst(c0, c + row0 * ldc, 0); \ + __lasx_xvst(c1, c + row0 * ldc, 8 * 4); \ + __lasx_xvst(c2, c + row1 * ldc, 0); \ + __lasx_xvst(c3, c + row1 * ldc, 8 * 4); \ + __lasx_xvst(c4, c + row2 * ldc, 0); \ + __lasx_xvst(c5, c + row2 * ldc, 8 * 4); \ + __lasx_xvst(c6, c + row3 * ldc, 0); \ + __lasx_xvst(c7, c + row3 * ldc, 8 * 4); + + FAST_GEMM_FINALE(0, 1, 2, 3); + FAST_GEMM_FINALE(4, 5, 6, 7); + FAST_GEMM_FINALE(8, 9, 10, 11); +#undef FAST_GEMM_FINALE +} + +static void fast_gemm_macro_kernel(int m, int n, int k, + const char *packed_A, const char *packed_B, + float alpha, char *c, int ldc0, int esz) { + int ldc0_esz = ldc0 * esz; + + double tempC[FAST_GEMM_F32_MR * FAST_GEMM_F32_NR]; // make sure the buffer is big enough + for(int i = 0; i < m; i += FAST_GEMM_F32_MR) { + for(int j = 0; j < n; j += FAST_GEMM_F32_NR) { + char* cptr0 = &c[i * ldc0_esz + j * esz]; + char* cptr = cptr0; + int ldc = ldc0; + int mr = m - i < FAST_GEMM_F32_MR ? m - i : FAST_GEMM_F32_MR; + int nr = n - j < FAST_GEMM_F32_NR ? n - j : FAST_GEMM_F32_NR; + int nr_esz = nr * esz; + bool partial = (bool)((mr < FAST_GEMM_F32_MR) | (nr < FAST_GEMM_F32_NR)); + if (partial) { + memset(tempC, 0, sizeof(tempC)); + cptr = (char *)tempC; + ldc = FAST_GEMM_F32_NR; + for(int p = 0; p < mr; p++) + memcpy(cptr + p * (ldc * esz), cptr0 + p * ldc0_esz, nr_esz); + } + fast_gemm12x16_f32(k, packed_A + i * k * esz, packed_B + j * k * esz, cptr, ldc, alpha); + + if (partial) { + for(int p = 0; p < mr; p++) + memcpy(cptr0 + p * ldc0_esz, cptr + p * (ldc * esz), nr_esz); + } + } + } +} + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *B, int ldb0, int ldb1, + float beta, char *C, int ldc, int esz) { + int GEMM_MC = FAST_GEMM_F32_MC, + GEMM_NC = FAST_GEMM_F32_NC, + GEMM_MR = FAST_GEMM_F32_MR, + GEMM_NR = FAST_GEMM_F32_NR; + + int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = FAST_GEMM_STORAGE / ((MC + NC) * esz); + KC = KC > 8 ? KC : 8; + KC = KC < K ? KC : K; + + size_t buff_size = KC * (MC + NC) * esz; + bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF; + int m_tiles = (M + MC - 1) / MC; + int n_tiles = (N + NC - 1) / NC; + int total_tiles = m_tiles * n_tiles; + + auto fn = [&](const Range &r) { + char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size)); + char* packed_b = packed_a + KC * MC * esz; + int start = r.start; + int end = r.end; + + for (int tile_idx = start; tile_idx < end; tile_idx++) { + int i0 = (tile_idx / n_tiles) * MC; + int j0 = (tile_idx % n_tiles) * NC; + int mc = M - i0 < MC ? M - i0 : MC; + int nc = N - j0 < NC ? N - j0 : NC; + int ldc_block = ldc; + char* c_block = C + (i0 * ldc + j0) * esz; + + if (beta == 0.f) { + for(int i = 0; i < mc; i++) + memset(c_block + i * ldc_block * esz, 0, nc * esz); + } else if (beta != 1.f) { + for(int i = 0; i < mc; i++) { + float* c_i = (float*)c_block + i * ldc_block; + for(int j = 0; j < nc; j++) + c_i[j] *= beta; + } + } + + for(int k0 = 0; k0 < K; k0 += KC) + { + int kc = K - k0 < KC ? K - k0 : KC; + fast_gemm_pack12_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); + fast_gemm_pack16_f32(nc, kc, B + (k0 * ldb0 + j0 * ldb1) * esz, ldb1, ldb0, packed_b); + fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b, alpha, c_block, ldc_block, esz); + } + } + + if (!use_stackbuff) { + free(packed_a); + } + }; + + int total = total_tiles; + int cost_per_thread = static_cast((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR)); + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +void fastGemmKernel(int M, int N, int K, + float alpha, const char *A, int lda0, int lda1, + const char *packed_B, float beta, char *C, int ldc, int esz) { + int GEMM_MC = FAST_GEMM_F32_MC, + GEMM_NC = FAST_GEMM_F32_NC, + GEMM_MR = FAST_GEMM_F32_MR, + GEMM_NR = FAST_GEMM_F32_NR; + + int MC = (((GEMM_MC < M ? GEMM_MC : M) + GEMM_MR - 1) / GEMM_MR) * GEMM_MR; + int NC = (((GEMM_NC < N ? GEMM_NC : N) + GEMM_NR - 1) / GEMM_NR) * GEMM_NR; + int KC = std::min(FAST_GEMM_F32_PACKED_STRIDE_K, K); + + size_t buff_size = KC * MC * esz; + bool use_stackbuff = buff_size <= FAST_GEMM_MAX_STACKBUF; + int m_tiles = (M + MC - 1) / MC; + int n_tiles = (N + NC - 1) / NC; + int total_tiles = m_tiles * n_tiles; + + auto fn = [&](const Range &r) { + char* packed_a = (char*)(use_stackbuff ? alloca(buff_size) : malloc(buff_size)); // TODO: use AutoBuffer + const char *packed_b_ = packed_B; + int start = r.start; + int end = r.end; + + for (int tile_idx = start; tile_idx < end; tile_idx++) { + int i0 = (tile_idx / n_tiles) * MC; + int j0 = (tile_idx % n_tiles) * NC; + int mc = M - i0 < MC ? M - i0 : MC; + int nc = N - j0 < NC ? N - j0 : NC; + int ldc_block = ldc; + char* c_block = C + (i0 * ldc + j0) * esz; + packed_b_ = packed_B + j0 * K * esz; + + if (beta == 0.f) { + for(int i = 0; i < mc; i++) + memset(c_block + i * ldc_block * esz, 0, nc * esz); + } else if (beta != 1.f) { + for(int i = 0; i < mc; i++) { + float* c_i = (float*)c_block + i * ldc_block; + for(int j = 0; j < nc; j++) + c_i[j] *= beta; + } + } + + int _nc = static_cast((nc + GEMM_NR - 1) / GEMM_NR) * GEMM_NR * esz; + for(int k0 = 0; k0 < K; k0 += KC) + { + int kc = K - k0 < KC ? K - k0 : KC; + fast_gemm_pack12_f32(mc, kc, A + (i0 * lda0 + k0 * lda1) * esz, lda0, lda1, packed_a); + fast_gemm_macro_kernel(mc, nc, kc, packed_a, packed_b_, alpha, c_block, ldc_block, esz); + packed_b_ += _nc * kc; + } + } + + if (!use_stackbuff) { + free(packed_a); + } + }; + + int total = total_tiles; + int cost_per_thread = static_cast((K / KC) * (MC / GEMM_MR) * (NC / GEMM_NR)); + double nstripes = (size_t)total * cost_per_thread * (1 / 1024.0); + parallel_for_(Range(0, total), fn, nstripes); +} + +#endif // CV_LASX + +CV_CPU_OPTIMIZATION_NAMESPACE_END + +}} // cv::dnn diff --git a/modules/dnn/src/layers/einsum_layer.cpp b/modules/dnn/src/layers/einsum_layer.cpp new file mode 100644 index 0000000000..cfa06e375e --- /dev/null +++ b/modules/dnn/src/layers/einsum_layer.cpp @@ -0,0 +1,1114 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include +#include +#include "../precomp.hpp" +#include "layers_common.hpp" + +namespace cv +{ +namespace dnn +{ + +static bool IsTransposeReshapeForEinsum(const std::vector& perm, + std::vector input_dims, + MatShape& new_shape) { + // As long as the dims with values > 1 stay in the same order, it's a reshape. + // Example: Shape=(1,1,1024,4096) -> perm=(2,0,3,1). + size_t last_permuted_axis = 0; + for (size_t i = 0; i < perm.size(); ++i) { + if (input_dims[perm[i]] == 1) + continue; + if (perm[i] < last_permuted_axis) + return false; + last_permuted_axis = perm[i]; + } + new_shape.assign(input_dims.begin(), input_dims.end()); + for (size_t i = 0; i < perm.size(); ++i) { + new_shape[i] = input_dims[perm[i]]; + } + return true; +} + +Mat batchwiseMatMul( + const Mat& input1, + const MatShape& input1ShapeOverride, + const Mat& input2, + const MatShape& input2ShapeOverride) +{ + + // Sanity checks before the actual MatMul + //input_1.DataType() == input_2.DataType(), "Data types of the inputs must match for MatMul"); + + CV_CheckEQ(input1ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul"); + CV_CheckEQ(input2ShapeOverride.size(), (size_t) 3, "Only 1 batch dimension is allowed for MatMul"); + CV_CheckEQ((size_t) input1ShapeOverride[0], (size_t) input2ShapeOverride[0], "Batch dimension should match for MatMul;"); + CV_CheckEQ((size_t) input1ShapeOverride[2], (size_t) input2ShapeOverride[1], "Incompatible matrix dimensions for matMul"); + + size_t batches = input1ShapeOverride[0]; + size_t M = input1ShapeOverride[1]; + size_t K = input1ShapeOverride[2]; + size_t N = input2ShapeOverride[2]; + + //TODO: deal with dynamic shapes + //TODO: deal with reshaping operation (it might not always be needed) + std::vector output; + if (batches > 1) + { + Mat reshapedInput1 = input1; + Mat reshapedInput2 = input2; + + // input1 should of size MxK + // check if input1 needs reshape, if need reshape + if (input1.size[0] != M || input1.size[1] != K) + { + int shape[] = {static_cast(batches), static_cast(M), static_cast(K)}; + reshapedInput1 = input1.reshape(1, 3, shape); + } + + // input2 should be of size KxN + // check if input2 needs reshape, if needs reshape + if (input2.size[0] != K || input2.size[1] != N) + { + int shape[] = {static_cast(batches), static_cast(K), static_cast(N)}; + reshapedInput2 = input2.reshape(1, 3, shape); + } + + for (size_t i=0; i < batches; i++) + { + std::vector ranges1 = {cv::Range(i, i+1)}; + for (int j = 1; j < reshapedInput1.dims; j++) + ranges1.emplace_back(cv::Range::all()); + + Mat part1 = reshapedInput1(ranges1); + int shape[] = {static_cast(M), static_cast(K)}; + part1 = part1.reshape(1, sizeof(shape)/sizeof(shape[0]), shape); + + std::vector ranges2 = {cv::Range(i, i+1)}; + for (int j = 1; j < reshapedInput2.dims; j++) + ranges2.emplace_back(cv::Range::all()); + + Mat part2 = reshapedInput2(ranges2); + int shape2[] = {static_cast(K), static_cast(N)}; + part2 = part2.reshape(1, sizeof(shape2)/sizeof(shape2[0]), shape2); + + Mat tmp_output; + cv::gemm(part1, part2, 1.0, cv::Mat(), 1.0, tmp_output); + int newShape[] = {1, static_cast(M), static_cast(N)}; + tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape); + + output.emplace_back(tmp_output); + } + + } else { + + Mat reshapedInput1 = input1; + Mat reshapedInput2 = input2; + + // input1 should of size MxK + // check if input1 needs reshape, if need reshape + if (input1.dims > 2 || input1.size[0] != M || input1.size[1] != K) + { + int shape[] = {static_cast(M), static_cast(K)}; + reshapedInput1 = input1.reshape(1, 2, shape); + } + + // input2 should be of size KxN + // check if input2 needs reshape, if needs reshape + if (input2.dims > 2 || input2.size[0] != K || input2.size[1] != N) + { + int shape2[] = {static_cast(K), static_cast(N)}; + reshapedInput2 = input2.reshape(1, 2, shape2); + } + + Mat tmp_output; + cv::gemm(reshapedInput1, reshapedInput2, 1.0, cv::Mat(), 1.0, tmp_output); + + int newShape[] = {1, static_cast(M), static_cast(N)}; + tmp_output = tmp_output.reshape(1, sizeof(newShape)/sizeof(newShape[0]), newShape); + output.emplace_back(tmp_output); + + } + + int outputDim[] = {static_cast(output.size()), static_cast(M), static_cast(N)}; + Mat output_buffer = Mat::zeros(3, outputDim, CV_32F); + + for (size_t i = 0; i < output.size(); i++) { + Mat output_slice = output_buffer.row(i); + output[i].copyTo(output_slice); + } + return output_buffer; +}; + +Mat Transpose( + const cv::Mat& input, + const MatShape& input_shape_override, + const std::vector permutation) +{ + + int input_rank = input_shape_override.size(); + + CV_Assert(input_rank == permutation.size()); + + // TODO: ouptimize + bool reshape = false; + if (input.dims != input_shape_override.size()) + { + reshape = true; + } + + Mat input_reshaped; + if(reshape) + { + input_reshaped = input.reshape(1, input_shape_override.size(), input_shape_override.data()); + } + + MatShape outputDims; + outputDims.reserve(input_rank); + for (const auto& dim : permutation) + outputDims.emplace_back(input_shape_override[dim]); + + Mat output; + // TODO: ouptimize + MatShape tmp_perm; + tmp_perm.reserve(permutation.size()); + for (int i = 0; i < permutation.size(); i++) + tmp_perm.emplace_back(static_cast(permutation[i])); + + cv::transposeND((reshape ? input_reshaped : input), tmp_perm, output); + return output; +} + + +bool IsTransposeRequired(size_t input_rank, const std::vector& permutation) { + CV_Assert(input_rank == permutation.size()); + + // No transpose required for scalars + if (input_rank == 0){ + return false; + } + + // Weeds out cases where permutation is something like [0, 1, 2] for a 3D input and so on + bool transpose_required = false; + for (size_t i = 0; i < input_rank; ++i) { + if (permutation[i] != i) { + transpose_required = true; + break; + } + } + + return transpose_required; +} + +Mat Diagonal( + const cv::Mat& input, + int subscriptIndicesToInputIndex, + int dimIndexInIreprocessedInput) +{ + CV_Error(Error::StsNotImplemented, "Diagonal Not Implemented Yet"); +} + +/** + * Returns the index associated with the input character. + * - Returns a value between 0 and 25 for inputs in the range 'a' to 'z'. + * - Returns a value between 26 and 51 for inputs in the range 'A' to 'Z'. + * - Returns -1 for invalid input that is not in the range 'a' to 'z' or 'A' to 'Z' (the caller should handle the returned result accordingly). + */ +int letterToIndex(const char ch) { + if (ch >= 'a' && ch <= 'z') { + return static_cast(ch) - 'a'; + } + + if (ch >= 'A' && ch <= 'Z') { + return static_cast('z') + static_cast(ch) - 'A'; + } + // invalid character - return error value + return -1; +} + +// Implementation of the Einsum layer is heavily influenced by Onnxruntime at the time of writing. +// Main logic is borrowed from onnxrutime: +// https://github.com/microsoft/onnxruntime/blob/eaea34f8e29df9fb21fab675a3a895084407f306/onnxruntime/core/providers/cpu/math/einsum_utils/einsum_compute_preprocessor.cc#L8 +class LayerEinsumImpl CV_FINAL : public EinsumLayer +{ +private: + Ptr reduce; +public: + // Number of inputs and outputs of the layer + int numInputs; + + // inputShapes; + std::vector einsumInpShapes; + + // Preprocessed inputs + std::vector preProcessedInputs; + + // This is container for preporcessed inputs + std::vector homogenizedInputDims; + + // Collect outpus dimentions + MatShape einsumOutDims; // vector to store output dimentions + + // These hold equation subring, left hand side and right it of + String lhs_eq, rhs_eq; + + // Holds token from left hand side of the equation + std::vector lhs_eq_tokens; + + // Idicates if equation substring is defined in explit way such as "ij, jk->ik" + // as opposed to "ij->" + bool explicitEquation = false; + + // Stores the subscript indices for each input in the equation + std::vector> inputSubscriptIndices; + + // Keeps track of the input index of the last input that had the subscript label + // If the value is `-1`, it means the subscript label was never encountered or it appears in the output + std::vector subscriptIndicesToLastInput; + + // Holds the dimension value of the index corresponding to the subscript label + // `-1` indicates that the corresponding label was not encountered at all + std::vector subscriptIndicesToDimValue; + + // Index corresponding to each output dim corresponding to each subscript index + // A value of -1 means the corresponding subscript index is not found in the output + std::vector subscriptIndicesToOutputIndices; + + // Hold max number of alphabetic numbers + static const size_t numOfLetters = 52; + + // Stores the count corresponding to each letter encountered + // A value of `0` indicates that the corresponding letter hasn't been seen at all + std::array letter2count; + + // Hold the assigned index corresponding to the letter seen + // `-1` means the corresponding letter wasn't seen at all + std::array letter2index; + + // Represents the count of unique subscript labels (subscript indices) + // Example 1: For the equation 'ij, jk -> ik', num_subscript_indices_ = 3 (i, j, k) + // Example 2: For the equation '...ij', 'jk' -> '...ik', + // num_subscript_indices_ = 3 (i, j, k) + number of dimensions specified by an ellipsis (across all inputs) + int numLetterIndices = 0; + + // The number of dimensions that are encompassed by an "ellipsis" - "...". + size_t numOfEllipsisDims = 0; + + + void parseEquation(String equation); + void processEquation(const std::vector& inputs); + void processBroadcastedDims(); + void createOutputSubsctipt(); + void calculateOutputShape(); + void preProcessInputs(InputArrayOfArrays& inputs); + Mat reduceSum(Mat& src, MatShape& reduceAxis); + Mat FinalizeOutput(const Mat& candidateOuput, const MatShape& ordered_subscript_indices_in_candidate); + Mat pairwiseOperandProcess( + const Mat& left, + const MatShape& leftShapeOverride, + const Mat& right, + const MatShape& rightShapeOverride, + const MatShape& reduceDims, + bool isFinalPair + ); + + + // constructor + LayerEinsumImpl(const LayerParams& params) + { + setParamsFrom(params); + String equation = params.get("equation"); + int outputSize = params.get("outputSize"); + numInputs = params.get("inputSize"); + + CV_CheckEQ(outputSize, 1, "Einsum layer should only have one output"); + + // get the input shapes from onnx importer + for (int i=0; i < numInputs; i++){ + auto param = params.get("inputShapes" + cv::format("%d", i)); + int inputDims = param.size(); + std::vector shape; + for (int i = 0; i < inputDims; ++i) + shape.emplace_back(param.get(i)); + einsumInpShapes.emplace_back(shape); + } + + + // Maintains a mapping between input indices and their corresponding subscript labels for each input + inputSubscriptIndices.reserve(numInputs); + + // We allocate space for 10 values as a precaution, + // assuming that we won't encounter any input with a rank greater than 10. + // In such cases, the value of num_subscript_indices_ would be greater than 10. + subscriptIndicesToLastInput.reserve(10); + subscriptIndicesToDimValue.reserve(10); + + // fill in vectors to avoid getting random numbers + letter2count.fill(0); + letter2index.fill(-1); + + // parser equation and extract tokens from the equation + // save token to lhs_eq_tokens variable + parseEquation(equation); // TODO: return lhs_eq_tokens + + // Start preprocessing related to equation parsing + // and dimention broadcasting + processEquation(einsumInpShapes); + processBroadcastedDims(); + + // calculate output shape + createOutputSubsctipt(); + calculateOutputShape(); + } + + // getMeoryShapes + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_UNUSED(internals); + + // check if passed and parsed inputs match up in number and dimensions + CV_CheckEQ(static_cast(inputs.size()), numInputs, + "Number of inputs in forward and inputs during graph constructions do not match"); + for (int i = 0; i < numInputs; i++) + { + if (inputs[i] != einsumInpShapes[i]) + CV_Error(Error::StsAssert, "Passed input shapes do not match with parsed input shapes!"); + } + + outputs.clear(); + outputs.emplace_back(einsumOutDims); + return true; + + } // getMemoryShape + + // forward + void forward(InputArrayOfArrays inputs_arr, + OutputArrayOfArrays outputs_arr, + OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + // homogenize inputs + preProcessInputs(inputs_arr); + + std::vector rawInputs, outputs; + inputs_arr.getMatVector(rawInputs); + outputs_arr.getMatVector(outputs); + Mat result; + + // Pre-process the first input so as to reduce any dims that only it has + { + MatShape reducedDims; + MatShape preservedDims; + MatShape preservedShape; + + reducedDims.reserve(numLetterIndices); // num_subscript_labels is the upper bound. No harm in over-reserving. + preservedDims.reserve(numLetterIndices); // num_subscript_labels is the upper bound. No harm in over-reserving. + + for (size_t i = 0; i < numLetterIndices; ++i) { + if (subscriptIndicesToLastInput[i] == 0) { + reducedDims.push_back(i); + } else { + preservedDims.push_back(i); + } + } + + // Reduce the dims that are last seen in the first input alone + if (reducedDims.size() != 0) + { + result = reduceSum((!preProcessedInputs[0].empty() ? preProcessedInputs[0] : rawInputs[0]), reducedDims); + } else { + // Check if there is a pre-processed version of this input + // If so assign it to result + if (!preProcessedInputs[0].empty()) + { + result = preProcessedInputs[0]; + } + } + + // Finalize the output at this stage if num_inputs == 1 + if (numInputs == 1) { + // Finalize the output by applying any transpose required to get + // it to the required output ordering and move it to the op's output + result = FinalizeOutput(!result.empty() ? result : rawInputs[0], preservedDims); + } + } + + + // Process the operands in a pair-wise fashion + { + bool isFinalPair = false; + // Keep processing each input pair-wise + for (int input = 1; input < numInputs; ++input) { + MatShape reducedDims; + reducedDims.reserve(numLetterIndices); // num_subscript_labels is the upper bound. No harm in over-reserving by a small margin. + for (int dim = 0; dim < numLetterIndices; ++dim) + { + if (subscriptIndicesToLastInput[dim] == input) + { + // This is the last input we are seeing this dimension (and it doesn't occur in the output), so reduce along the dimension + reducedDims.push_back(dim); + } + } + + if (input == numInputs - 1) + isFinalPair = true; + + // create temporary variable + MatShape tmpResult; + for (int i = 0; i < result.size.dims(); i++) + tmpResult.emplace_back(result.size[i]); + + + // Use either the preprocessed inputs (if it is available) or the corresponding raw inputs + result = pairwiseOperandProcess(!result.empty() ? result : rawInputs[0], + !result.empty() ? tmpResult : homogenizedInputDims[0], + !preProcessedInputs[input].empty() ? preProcessedInputs[input] : rawInputs[input], + homogenizedInputDims[input], + reducedDims, + isFinalPair); + } + } + + // check of product of output dimentions and computed output dimentions match + size_t reqProd = std::accumulate(einsumOutDims.begin(), einsumOutDims.end(), 1, std::multiplies()); + MatShape realOutputDims = shape(result); + size_t realProd = std::accumulate(realOutputDims.begin(), realOutputDims.end(), 1, std::multiplies()); + + CV_CheckEQ(reqProd, realProd, "Real output can not be shaped in to requred output"); + + // reduce dimentions + result = result.reshape(1, einsumOutDims.size(), einsumOutDims.data()); + result.copyTo(outputs[0]); + } // forward +}; // EinsumClass + +Mat LayerEinsumImpl::reduceSum(Mat& src, MatShape& reduceAxis) +{ + // initialize ReduceLayer + LayerParams lp; + lp.set("reduce", "SUM"); + int num_axes = reduceAxis.size(); + lp.set("axes", DictValue::arrayInt(&reduceAxis[0] , num_axes)); + reduce = ReduceLayer::create(lp); + + // Compute output shapes + std::vector inputShapes{shape(src)}; + std::vector outputShapes, internalShapes; + reduce->getMemoryShapes(inputShapes, 1, outputShapes, internalShapes); + + Mat output(outputShapes[0], CV_32F); + + std::vector inputs; + std::vector outputs; + std::vector internals; + inputs.emplace_back(src); + outputs.emplace_back(output); + + reduce->forward(inputs, outputs, internals); + return outputs[0]; +} + +void LayerEinsumImpl::preProcessInputs(InputArrayOfArrays& inputs_arr) +{ + std::vector inputs; + inputs_arr.getMatVector(inputs); + + preProcessedInputs.reserve(inputs.size()); + homogenizedInputDims.reserve(inputs.size()); + + int inputIter = 0; + for(const Mat& input : inputs) + { + Mat preprocessed; + + // variable to hold processed version of the original input + MatShape input_dims = shape(input); + + const auto& currSubscriptIndices = inputSubscriptIndices[inputIter]; + + // There should be subscript index (subscript label) for each dim of the input + CV_CheckEQ(input_dims.size(), currSubscriptIndices.size(), + "Rank of the input must match number of subscript labels corresponding to the input"); + + std::vector subscriptIndicesToInputIndex(numLetterIndices, -1); + // this will hold input dims after reordering so that all inputs have + // same axes order + MatShape homogenizedInputDims_(numLetterIndices, 1); + + int dimIndexInIreprocessedInput = 0; + int dimIndexInOriginalInput = 0; + + for (const auto& subscriptIndex : currSubscriptIndices) + { + if(subscriptIndicesToInputIndex[subscriptIndex] == -1){ + subscriptIndicesToInputIndex[subscriptIndex] = dimIndexInIreprocessedInput++; + homogenizedInputDims_[subscriptIndex] = input_dims[dimIndexInOriginalInput]; + } else { + // Call diagonal + preprocessed = Diagonal( + !preprocessed.empty() ? preprocessed : inputs[inputIter], + subscriptIndicesToInputIndex[subscriptIndex], + dimIndexInIreprocessedInput); + } + ++dimIndexInOriginalInput; + } + + std::vector permutation; + for(auto& d : subscriptIndicesToInputIndex) + { + if (d != -1) + permutation.emplace_back(d); + } + + if (IsTransposeRequired( + !preprocessed.empty() ? preprocessed.size.dims() : inputs[inputIter].size.dims(), + permutation)) + { + // call transpose + preprocessed = Transpose( + !preprocessed.empty() ? preprocessed : inputs[inputIter], + !preprocessed.empty() ? shape(preprocessed) : shape(inputs[inputIter]), + permutation); + } + + if (!preprocessed.empty()) + { + preprocessed = preprocessed.reshape(1, homogenizedInputDims_.size(), homogenizedInputDims_.data()); + } + + preProcessedInputs.emplace_back(preprocessed); + homogenizedInputDims.emplace_back(homogenizedInputDims_); + ++inputIter; + } +} + +void LayerEinsumImpl::parseEquation(String equation) +{ + // remove white spaces in the copy + equation.erase(std::remove_if(equation.begin(), equation.end(), ::isspace), equation.end()); + + // check if '->' - the output subscript label is present in the equation; + std::size_t arrow_idx = equation.find("->"); + if (arrow_idx != std::string::npos) + { + // split left and righ hand sides of the equation + lhs_eq = equation.substr(0, arrow_idx); + rhs_eq = equation.substr(arrow_idx + 2); + explicitEquation = true; + } else { + lhs_eq = equation; + } + + // split lhs_eq by ',' - comma and put all created token - splits + // into lhs_eq_tokens vector + std::stringstream src(lhs_eq); + for (std::string token; std::getline(src, token, ',');) { + lhs_eq_tokens.emplace_back(token); + } +} + + +void LayerEinsumImpl::calculateOutputShape() +{ + // Traverse through each of the subscript labels within the output subscript. + bool middleOfEllipsis = false; + // int64_t ellipsisCharCount = 0; + + subscriptIndicesToOutputIndices.resize(numLetterIndices, -1); + + std::array outputLetterToCount; + outputLetterToCount.fill(0); + + int outputDimCounter = 0; + for (auto letter : rhs_eq) + { + if(letter == '.') + { + CV_Error(Error::StsNotImplemented, "Ellipsis are not supported yet"); + } else { + CV_CheckEQ(middleOfEllipsis, false, + "Encountered '.' character that is not part of output subscript"); + + auto letterIndex = letterToIndex(letter); + + CV_CheckNE(letterIndex, -1, + "The only permissible subscript labels are lowercase letters (a-z) and uppercase letters (A-Z)."); + CV_CheckEQ(outputLetterToCount[letterIndex], 0, + "Output subscript constains repeated letters"); + + ++outputLetterToCount[letterIndex]; + auto mappedIndex = letter2index[letterIndex]; + + CV_CheckNE(mappedIndex, -1, + "Output subscript has letters that were not encountered in the inputs"); + + // Push output dimention + // Einsum layer only has one output vector + einsumOutDims.emplace_back(subscriptIndicesToDimValue[mappedIndex]); + + // Reset the last input index for this subscript label + // given that it is seen in the output and hence can't be reduced + subscriptIndicesToLastInput[mappedIndex] = -1; + subscriptIndicesToOutputIndices[mappedIndex] = outputDimCounter++; + } + } +} + +void LayerEinsumImpl::createOutputSubsctipt() +{ + // The explicit form requires no operation, as the output + // would have already been parsed during the input parsing process. + if(explicitEquation) + { + // Ensure that the provided explicit equation includes an ellipsis if the input contains ellipses. + if(numOfEllipsisDims > 0) + { + if(rhs_eq.find("...") == std::string::npos) + { + CV_Error(Error::StsError, + "Provided output subscript does not include ellipsis while Inputs subscrits constain ellipsis"); + } else { + CV_Error(Error::StsNotImplemented, "Ellipsis are not yet supported"); + } + } + } +} + +void LayerEinsumImpl::processBroadcastedDims() +{ + // Only compute this function if ellipsis "..." was found in the equation + if (numOfEllipsisDims > 0){ + // add assert inplace of return bool + CV_Error(Error::StsError, "Ellipsis are not supperted currenly"); + } +} + + + +void LayerEinsumImpl::processEquation(const std::vector& inputs) +{ + + // Check if number of tokens in equal to number of inputs. + // For install "ij, jk -> ik" needs to have 2 inputs tensors + int num_input_tensors = inputs.size(); + CV_CheckEQ(static_cast(lhs_eq_tokens.size()), num_input_tensors, + "Number of input tensors does not match the number of subscripts in the input equation"); + + int inputIdx = 0; + for (const auto& token : lhs_eq_tokens) + { + const MatShape shape = inputs[inputIdx]; + size_t rank = shape.size(); + size_t dim_count = 0; + + std::vector currTokenIndices; + currTokenIndices.reserve(rank); + + // Variable to deal with "ellipsis" - '...' in the input + bool middleOfellipsis = false; + for (auto letter : token) + { + // Broadcasting based tokens are not implemented yet + if (letter == '.') + { + CV_Error(Error::StsNotImplemented, + "Broad casting based indices are not supported currently"); + } else + { + + if (middleOfellipsis) + { + CV_Error(Error::StsAssert, + cv::format( + "Encountered '.' character that is not part of an ellipsis in the input: [%d]", + inputIdx)); + } + + int letterIdx = letterToIndex(letter); + CV_CheckNE(letterIdx, -1, + "The only permissible subscript labels are lowercase letters (a-z) and uppercase letters (A-Z)."); + + int dimValue = shape[dim_count]; + + // The subscript label was not found in the global subscript label array + // Therefore, it is added to both the local and global subscript arrays + if(letter2count[letterIdx] == 0) + { + letter2index[letterIdx] = numLetterIndices++; + subscriptIndicesToDimValue.push_back(dimValue); + subscriptIndicesToLastInput.push_back(inputIdx); + + } else { + // This letter has been seen in at least one other operand's subscript + // It must be equal unless one of them is a 1 (Numpy allows this) + auto mappedIndx = letter2index[letterIdx]; + subscriptIndicesToLastInput[mappedIndx] = inputIdx; + + if (subscriptIndicesToDimValue[mappedIndx] != dimValue) + { + if(subscriptIndicesToDimValue[mappedIndx] == 1){ + //TODO: uncomment later on + // subscriptIndicesToDimValue[mappedIndx] == dimValue; + } else + { + if (dimValue != 1) + { + CV_Error(Error::StsError, cv::format("Einsum operands can not be broadcasted." + "Check input shapes/equation passed." + "Input shape of operand [%d]", inputIdx) + + cv::format(" is incompatible in the dimention [%zu].", static_cast(dim_count))); + } + } + } + } + ++letter2count[letterIdx]; + currTokenIndices.push_back(letter2index[letterIdx]); + + CV_CheckLE(++dim_count, rank, + "The Einsum subscripts string has an excessive number of subscript labels compared to the rank of the input."); + } + } + + // When no broadcasting is requested, the number of subscript labels (dim_counter) should match the input's rank. + CV_Assert(!(numOfEllipsisDims == 0 && dim_count != rank) + && "The Einsum subscripts string does not contain required amount of subscript labels and no ellipsis is provided in the input."); + + inputSubscriptIndices.emplace_back(std::move(currTokenIndices)); + ++inputIdx; + } +} + +Mat LayerEinsumImpl::FinalizeOutput( + const Mat& candidateOutput, + const MatShape& ordered_subscript_indices_in_candidate) +{ + const std::vector& subscript_indices_to_output_indices = subscriptIndicesToOutputIndices; + const auto output_dims = einsumOutDims; + + MatShape output_shape = output_dims; + const auto output_rank = output_dims.size(); + + // CV_CheckEQ((int) candidateOutput.dims, (int) output_shape.size(), + // "Einsum op: The candidate output cannot be reshaped into the op's output"); + + const MatShape candidate_output_dims = MatShape(candidateOutput.size.p, candidateOutput.size.p + candidateOutput.dims); + const int candidate_output_rank = candidate_output_dims.size(); + + // This vector holds the shape of the candidate_output after removing the dims that have + // been reduced in the final output + MatShape candidate_output_shape_without_reduced_dims; + candidate_output_shape_without_reduced_dims.reserve(candidate_output_rank); // reserve upper bound + + // Identify the permutation required by the op's output + std::vector output_permutation; + output_permutation.resize(output_rank, 0); + size_t output_iter = 0; + + for (size_t iter = 0, end = ordered_subscript_indices_in_candidate.size(); iter < end; ++iter) + { + auto output_index = subscript_indices_to_output_indices[ordered_subscript_indices_in_candidate[iter]]; + + // If output_index is -1, then this dimension does not show up in the op's output and has been reduced along the way + if (output_index != -1) + { + output_permutation[output_index] = output_iter++; + candidate_output_shape_without_reduced_dims.push_back(candidate_output_dims[iter]); + } else { + // This dim doesn't show up in the op's output and hence we check if the dim has been reduced in the candidate output + CV_CheckEQ(candidate_output_dims[iter], 1, + "Not all dimensions to be reduced have been reduced in the candidate output. Candidate output dims: "); //%d", candidateOutput.size)); + } + } + + // Transpose to the required final output order + // (Identify no-op transposes and prevent triggering the transpose) + + if (IsTransposeRequired(candidate_output_shape_without_reduced_dims.size(), output_permutation)) + { + auto candidate_output_transposed = Transpose( + candidateOutput, + candidate_output_shape_without_reduced_dims, + output_permutation); + return candidate_output_transposed; + } + return candidateOutput; +} + +Mat LayerEinsumImpl::pairwiseOperandProcess( + const Mat& left, + const MatShape& leftShapeOverride, + const Mat& right, + const MatShape& rightShapeOverride, + const MatShape& reduceDims, + bool isFinalPair +) +{ + size_t matDimSize = left.total(); + size_t overrideDimSize = total(leftShapeOverride); + + CV_CheckEQ(matDimSize, overrideDimSize, "Override dims are not compatible with left tensor shape"); + + matDimSize = right.total(); + overrideDimSize = total(rightShapeOverride); + + CV_CheckEQ(matDimSize, overrideDimSize, "Override dims are not compatible with right tensor shape"); + + // Make copy as this may be overridden downstream + const auto& leftDims = leftShapeOverride; + const auto& rightDims = rightShapeOverride; + + int leftRank = static_cast(leftDims.size()); + int rightRank = static_cast(rightDims.size()); + + Mat currentLeft; + Mat currentRight; + + CV_CheckEQ(leftRank, rightRank, "Raks of pair-wise operands must be equal"); + + // Following vectors hold: + // lro: dim indices that are present in left, right, and reduce_dims + // lo: dim indices that are present in left and reduce_dims + // ro: dim indices that are present in right and reduce_dims + std::vector lro; + lro.reserve(5); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > kTensorShapeSmallBufferElementsSize) + + std::vector lo; + lo.reserve(5); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > kTensorShapeSmallBufferElementsSize) + + std::vector ro; + ro.reserve(5); // Reserve an arbitrary amount of space for this vector (not bound to see a tensor of rank > kTensorShapeSmallBufferElementsSize) + + // Maintain sizes to create reshaped "views" + int lro_size = 1; + int lo_size = 1; + int ro_size = 1; + int reduced_size = 1; + + size_t reduceDimsIter = 0; + size_t reduceDimsSize = reduceDims.size(); + + for (int i = 0; i < leftRank; ++i) + { + int leftDim = leftDims[i]; + int rightDim = rightDims[i]; + + bool hasLeftDim = leftDim > 1; // non-trivial dimension (dim_value != 1) + bool hasRightDim = rightDim > 1; // non-trivial dimension (dim_value != 1) + + if (reduceDimsIter < reduceDimsSize && reduceDims[reduceDimsIter] == i) + { + // This dimension is to be reduced after this pair-wise operation + ++reduceDimsIter; + if (hasLeftDim && hasRightDim){ + // Both inputs have non-trivial dim values along this dimension + // Both the left and right operands have non-trivial dimension value along this axis + CV_CheckEQ(leftDim, rightDim, "Einsum op: Input dimensions must be equal along an axis to be reduced across all inputs"); + reduced_size *= leftDim; + + } else if (hasLeftDim){ + // if the dim to be reduced is only in one of left and right, we can reduce right away + Mat tensorToReduce = !currentLeft.empty() ? currentLeft : left; + MatShape shapeToReduce = !currentLeft.empty() ? shape(currentLeft) : leftDims; + currentLeft = reduceSum(tensorToReduce, shapeToReduce); + + } else if (hasRightDim){ + Mat tensorToReduce = !currentRight.empty() ? currentRight : right; + MatShape shapeToReduce = !currentRight.empty() ? shape(currentRight) : rightDims; + currentLeft = reduceSum(tensorToReduce, shapeToReduce); + } + + } else { + // This dimension is not reduced (i.e.) it appears in the output after processing these 2 operands + // Both the left and right operands have non-trivial dimension value along this axis + // They must be equal + if (hasLeftDim && hasRightDim){ + CV_CheckEQ(leftDim, rightDim, "Input shapes do not align"); + lro.push_back(i); + lro_size *= leftDim; + + } else if (hasLeftDim) { + // The left operand has non-trivial dimension value + lo.push_back(i); + lo_size *= leftDim; + + } else { + // The right operand may or may not have non-trivial dim value + // If it has trivial dim value (1), + // it will just form a trailing dimension for the right operand + ro.push_back(i); + ro_size *= rightDim; + } + } + } + + + // Permutate the left operand so that the axes order go like this: [lro, lo, reduce_dims, ro] + MatShape reshaped_dims; + std::vector left_permutation; + left_permutation.reserve(lro.size() + lo.size() + reduceDims.size() + ro.size()); + left_permutation.insert(left_permutation.end(), lro.begin(), lro.end()); + left_permutation.insert(left_permutation.end(), lo.begin(), lo.end()); + // left_permutation.insert(left_permutation.end(), reduce_dims.begin(), reduce_dims.end()); + + for (auto& a : reduceDims) + { + left_permutation.push_back(a); + } + left_permutation.insert(left_permutation.end(), ro.begin(), ro.end()); + + if (IsTransposeRequired(!currentLeft.empty() ? currentLeft.dims : leftDims.size(), + left_permutation)) + { + if (!currentLeft.empty() && IsTransposeReshapeForEinsum(left_permutation, + shape(currentLeft), + reshaped_dims)) + { + // This can be done because curent_* tensors (if they exist) and output tensors are + // intermediate tensors and cannot be input tensors to the Einsum node itself + // (which are immutable). + currentLeft = currentLeft.reshape(1, reshaped_dims.size(), reshaped_dims.data()); + } else { + // Covered by ExplicitEinsumAsTensorContraction, DiagonalWithMatmul, ... + currentLeft = Transpose(!currentLeft.empty() ? currentLeft: left, + !currentLeft.empty() ? shape(currentLeft) : leftDims, + left_permutation); + } + } + + // Permutate the right operand so that the axes order go like this: [lro, reduce_dims, ro, lo] + std::vector right_permutation; + right_permutation.reserve(lro.size() + lo.size() + reduceDims.size() + ro.size()); + right_permutation.insert(right_permutation.end(), lro.begin(), lro.end()); + // right_permutation.insert(right_permutation.end(), reduce_dims.begin(), reduce_dims.end()); + for (auto& a : reduceDims) { + right_permutation.push_back(a); + } + right_permutation.insert(right_permutation.end(), ro.begin(), ro.end()); + right_permutation.insert(right_permutation.end(), lo.begin(), lo.end()); + + if (IsTransposeRequired(!currentRight.empty() ? currentRight.dims: rightDims.size(), + right_permutation)) + { + if (!currentRight.empty() && IsTransposeReshapeForEinsum(right_permutation, + shape(currentRight), + reshaped_dims)) + { + currentRight = currentRight.reshape(1, reshaped_dims.size(), reshaped_dims.data()); + } else { + currentRight = Transpose(!currentRight.empty() ? currentRight : right, + !currentRight.empty() ? shape(currentRight) : rightDims, + right_permutation); + } + } + + // Calculate output size + // Output shape will be determined by rules of MatMul: + // because we are multiplying two tensors of shapes [lro, lo, reduce_dims] , [lro, reduce_dims, ro] + // [dim_value of `lro` dims, + // dim_value of `lo` dims, + // `1` for each of the `reduce_dims`, + // dim_value of `ro` dims] + MatShape outputDims; + outputDims.reserve(lro.size() + lo.size() + reduceDims.size() + ro.size()); + for (size_t i = 0; i < lro.size(); ++i) + { + outputDims.emplace_back(leftDims[lro[i]]); + } + + for (size_t i = 0; i < lo.size(); ++i) + { + outputDims.emplace_back(leftDims[lo[i]]); + } + + for (size_t i = 0; i < reduceDims.size(); ++i) + { + outputDims.emplace_back(1); // reduced dimensions will have a value 1 in it + } + + for (size_t i = 0; i < ro.size(); ++i) { + outputDims.emplace_back(rightDims[ro[i]]); + } + + MatShape currentSubscriptOrder; + // Calculate output permutation + // After the MatMul op, the because the two operands have been permutated, + // the output is permutated as well with respect to the original ordering of the axes. + // The permutated order will be the dims in: [lro, lo, reduced_dims, ro] + // Hence invert the permutation by a permutation that puts the axes in the same ordering + std::vector outputPermutation; + if (!isFinalPair) { // If this is not the final pair, we need to permutate the result to match the pre-fixed order for the next iteration + outputPermutation.resize(lro.size() + lo.size() + reduceDims.size() + ro.size(), 0); + size_t iter = 0; + for (size_t i = 0; i < lro.size(); ++i) + { + outputPermutation[lro[i]] = iter++; + } + + for (size_t i = 0; i < lo.size(); ++i) + { + outputPermutation[lo[i]] = iter++; + } + + for (size_t i = 0; i < reduceDims.size(); ++i) + { + outputPermutation[reduceDims[i]] = iter++; + } + + for (size_t i = 0; i < ro.size(); ++i) + { + outputPermutation[ro[i]] = iter++; + } + + } else { + currentSubscriptOrder.reserve(lro.size() + lo.size() + reduceDims.size() + ro.size()); + currentSubscriptOrder.insert(currentSubscriptOrder.end(), lro.begin(), lro.end()); + currentSubscriptOrder.insert(currentSubscriptOrder.end(), lo.begin(), lo.end()); + currentSubscriptOrder.insert(currentSubscriptOrder.end(), reduceDims.begin(), reduceDims.end()); + currentSubscriptOrder.insert(currentSubscriptOrder.end(), ro.begin(), ro.end()); + } + + Mat output = batchwiseMatMul( + !currentLeft.empty() ? currentLeft : left, + MatShape({static_cast(lro_size), static_cast(lo_size), static_cast(reduced_size)}), + !currentRight.empty() ? currentRight : right, + MatShape({static_cast(lro_size), static_cast(reduced_size), static_cast(ro_size)}) + ); + + //reshape + output = output.reshape(1, outputDims.size(), outputDims.data()); + + if (!isFinalPair) + { // This is not the final pair - so bring the axes order to what the inputs conformed to + if (IsTransposeRequired(outputDims.size(), outputPermutation)) + { + if (IsTransposeReshapeForEinsum(outputPermutation, + outputDims, + reshaped_dims)) + { + // See note following the previous call of function IsTransposeReshapeForEinsum. + // Covered by ExplicitEinsumAsTensorContractionReshapeFinal. + output = output.reshape(1, reshaped_dims.size(), reshaped_dims.data()); + } + } else { + output = Transpose( + output, + outputDims, + outputPermutation); + } + } else { // This is the final pair - Transpose directly to the output ordering required and copy the contents to the op's output + // not sure if this finalize shape is needed at all + output = FinalizeOutput(output, currentSubscriptOrder); + } + return output; +}; + +Ptr EinsumLayer::create(const LayerParams& params) +{ + return makePtr(params); +} + +}} // namespace cv::dnn diff --git a/modules/dnn/src/layers/expand_layer.cpp b/modules/dnn/src/layers/expand_layer.cpp new file mode 100644 index 0000000000..c31a932ae1 --- /dev/null +++ b/modules/dnn/src/layers/expand_layer.cpp @@ -0,0 +1,149 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include + +namespace cv { namespace dnn { + +class ExpandLayerImpl CV_FINAL : public ExpandLayer +{ +public: + ExpandLayerImpl(const LayerParams ¶ms) { + setParamsFrom(params); + + // shape as param + CV_CheckTrue(params.has("shape"), "DNN/Expand: shape is required in Expand layer initialization"); + DictValue param_shape = params.get("shape"); + int ndims_shape = param_shape.size(); + CV_CheckGT(ndims_shape, 0, "DNN/Expand: ndims of shape must be > 0"); + target_shape.resize(ndims_shape); + for (int i = 0; i < ndims_shape; i++) { + target_shape[i] = param_shape.get(i); + } + + // FIXME: remove when 0d/1d mat is available + const_input_1d = params.get("const_input_1d", false); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + CV_CheckGE(inputs.size(), static_cast(1), "DNN/Expand: one input at least"); + CV_CheckLE(inputs.size(), static_cast(2), "DNN/Expand: two input at most"); + CV_CheckFalse(target_shape.empty(), "DNN/Expand: shape must known before memory is set"); + + MatShape input_shape = inputs[0]; // 1d tensor is represented as 2d mat, e.g. [3] -> [3, 1] + if (const_input_1d) { + input_shape = {inputs[0][0]}; + } + + auto& moreDimension = input_shape.size() > target_shape.size() ? input_shape : target_shape; + auto& lessDimension = input_shape.size() <= target_shape.size() ? input_shape : target_shape; + + /* Example: + i = 3 + | + moreDimension: 1 2 3 4 5, assign non-aligned dimensions to output shape + lessDimension: 1 1 5, when dimension is aligned, check valid dimension (either equal or one of them is 1) and assign bigger one + | + j = 0 = i - (moreDimension.size() - lessDimension.size()); + */ + MatShape outputShape(moreDimension.size(), 1); + for (int i = 0; i < moreDimension.size(); i++) { + int d = moreDimension[i]; + int j = i - (moreDimension.size() - lessDimension.size()); + if (j >= 0) { + if (d == 1 || lessDimension[j] == 1 || // broadcast + d == lessDimension[j]) { // plain copy + outputShape[i] = std::max(d, lessDimension[j]); + } else { + CV_Error(Error::StsBadSize, cv::format("DNN/Expand: invalid dimension, d (%d) != d (%d)", moreDimension[i], lessDimension[j])); + } + } else { + outputShape[i] = d; + } + } + outputs.assign(1, outputShape); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + const auto &input = inputs[0]; + auto input_shape = shape(input); + if (const_input_1d) { + input_shape = {input_shape[0]}; + } + + auto& moreDimension = input_shape.size() > target_shape.size() ? input_shape : target_shape; + auto& lessDimension = input_shape.size() <= target_shape.size() ? input_shape : target_shape; + + MatShape final_target_shape(moreDimension.size(), 1); + for (int i = 0; i < moreDimension.size(); i++) { + int d = moreDimension[i]; + int j = i - (moreDimension.size() - lessDimension.size()); + if (j >= 0) { + final_target_shape[i] = std::max(lessDimension[j], d); + } else { + final_target_shape[i] = d; + } + } + target_shape.clear(); + target_shape = std::move(final_target_shape); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + int target_shape_total = std::accumulate(target_shape.begin(), target_shape.end(), 1, std::multiplies()); + if (target_shape_total == inputs[0].total()) { + const char *data = inputs[0].ptr(); + char *output = outputs[0].ptr(); + int step = target_shape_total * outputs[0].elemSize(); + std::memcpy(output, data, step); + return; + } + + if (const_input_1d) { + const char *data = inputs[0].ptr(); + char *output = outputs[0].ptr(); + int step = target_shape.back() * outputs[0].elemSize(); + int total = std::accumulate(target_shape.begin(), target_shape.end() - 1, 1, std::multiplies()); + for (int i = 0; i < total; i++) { + std::memcpy(output + i * step, data, step); + } + } else { + cv::broadcast(inputs[0], target_shape, outputs[0]); + } + } + +private: + MatShape target_shape; + bool const_input_1d; +}; + +Ptr ExpandLayer::create(const LayerParams ¶ms) { + return makePtr(params); +} + +}} // cv::dnn diff --git a/modules/dnn/src/layers/gemm_layer.cpp b/modules/dnn/src/layers/gemm_layer.cpp new file mode 100644 index 0000000000..0a58abce5d --- /dev/null +++ b/modules/dnn/src/layers/gemm_layer.cpp @@ -0,0 +1,373 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +// backends +#include "../op_cuda.hpp" +#ifdef HAVE_CUDA +// #include "../cuda4dnn/primitives/matmul.hpp" +#include "../cuda4dnn/primitives/inner_product.hpp" +using namespace cv::dnn::cuda4dnn; +#endif +#include "../op_cann.hpp" +#include "../ie_ngraph.hpp" +#include "../op_vkcom.hpp" + +#include +#include "cpu_kernels/fast_gemm.hpp" + +namespace cv { namespace dnn { + +class GemmLayerImpl CV_FINAL : public GemmLayer { +public: + GemmLayerImpl(const LayerParams& params) { + setParamsFrom(params); + + trans_a = params.get("transA", false); + trans_b = params.get("transB", false); + alpha = params.get("alpha", 1.0f); + beta = params.get("beta", 1.0f); + + const_B = params.get("constB", false); // true means blobs[0] is B + const_C = params.get("constC", false); // true means blobs.back() is C + have_bias = params.get("have_bias", false); // NOTE: have_bias being true does not mean bias is constant + + real_ndims_C = params.get("real_ndims_C", -1); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE { + return backendId == DNN_BACKEND_OPENCV || + (backendId == DNN_BACKEND_CUDA && const_B && !trans_a) || + backendId == DNN_BACKEND_CANN || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH || + (backendId == DNN_BACKEND_VKCOM && haveVulkan() && !have_bias && !trans_a); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + int num_inputs = static_cast(inputs.size() + blobs.size()); + CV_CheckGE(num_inputs, 2, "DNN/Gemm: Gemm takes at least two inputs"); + CV_CheckLE(num_inputs, 3, "DNN/Gemm: Gemm takes at most three inputs"); + + // Check whether A and B are two dimensional + const auto shape_A = inputs[0]; + const auto shape_B = const_B ? shape(blobs[0]) : inputs[1]; + CV_CheckGE(shape_A.size(), static_cast(2), "DNN/Gemm: Tensor A must be n-dimensional (n >= 2)"); + CV_CheckEQ(shape_B.size(), static_cast(2), "DNN/Gemm: Tensor B must be two dimensional"); + + // Check legal matrix multiplication + size_t dims_A = shape_A.size(); + int ma = shape_A[dims_A - 2], na = shape_A[dims_A - 1]; + int mb = shape_B[0], nb = shape_B[1]; + int M = trans_a ? na : ma; + int N = trans_b ? mb : nb; + int K_a = trans_a ? ma : na; + int K_b = trans_b ? nb : mb; + CV_CheckEQ(K_a, K_b, "DNN/Gemm: Invalid dimension of dim K"); + + // Check whether C can be unidirectional broadcast to (M, N). Handle carefully with 1D Mat. + if (have_bias) { + const auto shape_C = const_C ? shape(blobs.back()) : inputs.back(); + + auto ndims_C = shape_C.size(); + CV_CheckLE(ndims_C, static_cast(2), "DNN/Gemm: C can only be 0d (scalar) / 1d / 2d tensor"); + + if (real_ndims_C == 1) { // (1,) or (N,) + CV_Check(shape_C[0], shape_C[0] == 1 || shape_C[0] == N, "DNN/Gemm: invalid dimension of C"); + } else if (real_ndims_C == 2) { // (1, 1) or (1, N) or (M, 1) or (M, N) + // printf("shape_C=[%d, %d]\n", shape_C[0], shape_C[1]); + CV_Check(shape_C[0], (shape_C[0] == 1 && shape_C[1] == 1) || + (shape_C[0] == 1 && shape_C[1] == N) || + (shape_C[0] == M && shape_C[1] == 1) || + (shape_C[0] == M && shape_C[1] == N), + "DNN/Gemm: C must be of shape (1, 1) or (1, N) or (M, 1) or (M, N)"); + if (shape_C[0] == 1) { + CV_Check(shape_C[1], shape_C[1] == 1 || shape_C[1] == N, "DNN/Gemm: invalid dimension of C"); + } else if (shape_C[0] == M) { + CV_Check(shape_C[1], shape_C[1] == 1 || shape_C[1] == N, "DNN/Gemm: invalid dimension of C"); + } else { + CV_Error(Error::StsBadSize, "DNN/Gemm: invalid dimension of C"); + } + } + } + + int batches = std::accumulate(shape_A.begin(), shape_A.end() - 2, 1, std::multiplies()); + MatShape shape_y{M * batches, N}; + outputs.assign(1, shape_y); + return false; + } + + // TODO: replace with cv::broadcast() once 1d mat is supported + // FIXME: fix if conditions if 1d mat is supported properly + void broadcastCWtihBeta(int M, int N, const Mat &C) { + if (beta != 0 && !C.empty()) { + broadcast_C.clear(); + broadcast_C.resize(M * N, 0.f); + + const float *ptr_c = C.ptr(); + const auto shape_C = shape(C); + if ((real_ndims_C == 0) || (real_ndims_C == 1 && shape_C[0] == 1) || + (real_ndims_C == 2 && shape_C[0] == 1 && shape_C[1] == 1)) { + // (), (1,), (1, 1) + float c = *ptr_c; + int total = M * N; + for (int i = 0; i < total; ++i) { + broadcast_C[i] = beta * c; + } + } else if ((real_ndims_C == 1 && shape_C[0] == N) || + (real_ndims_C == 2 && shape_C[0] == 1 && shape_C[1] == N)) { + // (N,), (1, N) + for (int i = 0; i < M; ++i) { + int step = i * N; + for (int j = 0; j < N; ++j) { + broadcast_C[step + j] = beta * ptr_c[j]; + } + } + } else if (real_ndims_C == 2 && shape_C[0] == M && shape_C[1] == 1) { + // (M, 1) + for (int i = 0; i < M; ++i) { + int step = i * N; + for (int j = 0; j < N; ++j) { + broadcast_C[step + j] = beta * ptr_c[i]; + } + } + } else { + // (M, N) + std::transform(ptr_c, ptr_c + M * N, broadcast_C.begin(), [this] (const float &c) { + return this->beta * c; }); + } + } + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE { + opt.init(); + + // pack B if it is const + if (const_B) { + fastGemmPackB(blobs[0], packed_B, trans_b, opt); + } + + // also pre-broadcast bias + if (const_C) { + const auto &C = blobs.back(); + + std::vector outputs; + outputs_arr.getMatVector(outputs); + const auto &Y = outputs[0]; + const auto shape_Y = shape(Y); + size_t dims_Y = shape_Y.size(); + int M = shape_Y[dims_Y - 2], N = shape_Y[dims_Y - 1]; + + // broadcast + broadcastCWtihBeta(M, N, C); + } + } + + // Y = A * B + C, note that C is unidirectionaly broadcastable to (A * B). + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + const auto &A = inputs[0]; + auto &Y = outputs[0]; + + const auto shape_A = shape(A), shape_Y = shape(Y); + size_t dims_A = shape_A.size(); + int ma = shape_A[dims_A - 2], na = shape_A[dims_A - 1]; + size_t dims_Y = shape_Y.size(); + int M = shape_Y[dims_Y - 2], N = shape_Y[dims_Y - 1]; + int K = trans_a ? ma : na; + int batches = std::accumulate(shape_A.begin(), shape_A.end() - 2, 1, std::multiplies()); + + // broadcast C and copy C to output + if (have_bias) { + if (!const_C) { + broadcastCWtihBeta(M, N, inputs.back()); + } + int step = M * N; + CV_CheckEQ(broadcast_C.size(), static_cast(step), "DNN/Gemm: C is not broadcast properly"); + float *ptr_y = Y.ptr(); + for (int i = 0; i < batches; i++) { + std::memcpy(ptr_y + i * step, broadcast_C.data(), step * sizeof(float)); + } + } else { // initialization + float *ptr_y = Y.ptr(); + size_t total = Y.total(); + std::memset(ptr_y, 0, total * sizeof(float)); + } + + if (const_B) { + CV_CheckGT(packed_B.size(), static_cast(0), "DNN/Gemm: constant B is not pre-packed"); + M *= batches; + fastGemm(trans_a, M, N, K, alpha, A.ptr(), na, packed_B.data(), 1.f, Y.ptr(), N, opt); + } else { + fastGemmBatched(trans_a, trans_b, alpha, A, inputs[1], 1.f, Y, opt); + } + } + +#ifdef HAVE_CUDA + // Y = A * B + C. B should be guaranteed as two dimensional. + Ptr initCUDA(void *context_, + const std::vector>& inputs, + const std::vector>& outputs) CV_OVERRIDE { + CV_CheckFalse(trans_a, "DNN/Gemm/Cuda: does not support transA"); + CV_CheckTrue(const_B, "DNN/Gemm/Cuda: input B (weight) is required to be constant"); + auto context = reinterpret_cast(context_); + auto wrapper_A = inputs[0].dynamicCast(); + auto B = blobs[0]; + auto C = have_bias && const_C ? blobs[1] : Mat(); // in most cases C is constant + + if (!trans_b) + cv::transpose(B, B); + auto flatten_start_axis = normalize_axis(1, wrapper_A->getRank()); + return make_cuda_node(preferableTarget, std::move(context->stream), std::move(context->cublas_handle), flatten_start_axis, B, C); + } +#endif // HAVE_CUDA + +#ifdef HAVE_CANN + // Y = A * B + C. + virtual Ptr initCann(const std::vector > &inputs, + const std::vector > &outputs, + const std::vector >& nodes) CV_OVERRIDE { + auto x1 = inputs[0].dynamicCast(); + auto desc_x1 = x1->getTensorDesc(); + auto op_x1 = nodes[0].dynamicCast()->getOp(); + + auto op = std::make_shared(name); + + // set attributes + op->set_attr_transpose_x1(trans_a); + op->set_attr_transpose_x2(trans_b); + + // set inputs + // set inputs : x1 + op->set_input_x1_by_name(*op_x1, x1->name.c_str()); + op->update_input_desc_x1(*desc_x1); + // set inputs : x2 + if (const_B) { + auto B = blobs[0]; + auto op_const_B = std::make_shared(B.data, B.type(), shape(B), cv::format("%s_w", name.c_str())); + op->set_input_x2_by_name(*(op_const_B->getOp()), "y"); + op->update_input_desc_x2(*(op_const_B->getTensorDesc())); + } else { + CV_CheckGE(inputs.size(), static_cast(2), "DNN/Gemm/CANN: input B is required since it is not constant"); + CV_CheckGE(nodes.size(), static_cast(2), "DNN/Gemm/CANN: input B is required since it is not constant"); + auto op_x2 = nodes[1].dynamicCast()->getOp(); + auto desc_x2 = inputs[1].dynamicCast()->getTensorDesc(); + op->set_input_x2_by_name(*op_x2, "y"); + op->update_input_desc_x2(*desc_x2); + } + // set inputs : bias + auto mat_C = have_bias && const_C ? blobs.back() : Mat::zeros(1, 1, CV_32F); + auto op_const_C = std::make_shared(mat_C.data, mat_C.type(), shape(mat_C), cv::format("%s_b", name.c_str())); + op->set_input_bias(*(op_const_C->getOp())); + op->update_input_desc_bias(*(op_const_C->getTensorDesc())); + + // set outputs + op->update_output_desc_y(*output_desc); + return Ptr(new CannBackendNode(op)); + } +#endif // HAVE_CANN + +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector >& inputs, + const std::vector >& nodes) CV_OVERRIDE + { + auto ieInpNode = nodes[0].dynamicCast()->node; + std::shared_ptr matmul; + + if (nodes.size() == 2) + { + auto& inp2 = nodes[1].dynamicCast()->node; + matmul = std::make_shared(ieInpNode, inp2, trans_a, trans_b); + } + else + { + std::shared_ptr ieWeights = std::make_shared(ngraph::element::f32, getShape(blobs[0]), blobs[0].data); + + int flatten_axis = ieInpNode.get_shape().size() - ieWeights->get_shape().size(); + if (flatten_axis > 0) { + std::vector shape(1 + flatten_axis, 0); + shape[shape.size() - 1] = -1; + ieInpNode = std::make_shared( + ieInpNode, + std::make_shared(ngraph::element::i32, ngraph::Shape{shape.size()}, shape.data()), + true + ); + } + matmul = std::make_shared(ieInpNode, ieWeights, trans_a, trans_b); + } + if (alpha != 1.0f) { + matmul = std::make_shared(matmul, + std::make_shared(ngraph::element::f32, ngraph::Shape{1}, &alpha) + ); + } + + if (have_bias && const_C) { + Mat bias = blobs.back(); + auto shape = bias.total() == bias.size[0] ? ngraph::Shape{bias.total()} : getShape(bias); + std::shared_ptr bias_node = std::make_shared(ngraph::element::f32, shape, bias.data); + if (beta != 1.0f) { + bias_node = std::make_shared(bias_node, + std::make_shared(ngraph::element::f32, ngraph::Shape{1}, &beta) + ); + } + matmul = std::make_shared(matmul, bias_node, ngraph::op::AutoBroadcastType::NUMPY); + } + return Ptr(new InfEngineNgraphNode(matmul)); + } +#endif // HAVE_DNN_NGRAPH + +#ifdef HAVE_VULKAN + // Y = A * B + C. Currently support 2d matrix multiplication without bias. + virtual Ptr initVkCom(const std::vector > &inputs, + std::vector > &outputs) CV_OVERRIDE + { + // does not support with bias; only 2d matmul + auto wrapper_Y = outputs[0].dynamicCast(); + auto shape_Y = shape(*(wrapper_Y->getMat())); + if (have_bias || shape_Y.size() > static_cast(2)) { + return Ptr(); + } + + std::vector vkBlobs; + if (const_B) { + vkBlobs.push_back(blobs[0]); + } + + auto wrapper_A = inputs[0].dynamicCast(); + auto shape_A = shape(*wrapper_A->getMat()); + Ptr op = (new vkcom::OpMatMul(vkBlobs, shape_A[0], shape_A[1], shape_Y[1])); + return Ptr(new VkComBackendNode(inputs, op, outputs)); + } +#endif + +private: + bool const_B; + bool const_C; + bool have_bias; + std::vector packed_B; + std::vector broadcast_C; + int real_ndims_C; + FastGemmOpt opt; +}; + +Ptr GemmLayer::create(const LayerParams& params) { + return makePtr(params); +} + +}} // namespace cv::dnn diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 8f9be6e961..a43815dbe4 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -821,6 +821,16 @@ public: } }; +/* Constant folding shape for Expand. + + Before fusion: + +--------------------------------------------------------------+ (X) + | | + ConstantOfShape[input=[4]] -> Mul[B=-1] -> Equal[A=[2, -1, -1, -1]] -> Where[Y=[2, -1, -1, -1]] -> Expand + \ \ + value=[1] (condition) + +*/ class ExpandSubgraph : public Subgraph { public: @@ -837,6 +847,128 @@ public: addNodeToMatch("Expand", input, where); setFusedNode("Expand", input, shape); } + + static int extractValue(const Ptr& net, int node_id, int64_t &val) { + Ptr node_wrapper = net->getNode(node_id); + opencv_onnx::NodeProto* node = node_wrapper.dynamicCast()->node; + + if (node->attribute_size() == 0) { + val = 0; + return 1; + } else if (node->attribute_size() == 1) { + opencv_onnx::AttributeProto attr = node->attribute(0); + if (attr.name() != "value") { + return 0; + } + Mat mat_value = getMatFromTensor(attr.t()); + switch (mat_value.type()) { + case CV_32S: { + val = static_cast(mat_value.at()); + } break; + default: return 0; + } + return 1; + } + return 0; + } + + static std::vector extractConstant(const Ptr& net, int node_id, int input_id) + { + auto onnx_net = net.dynamicCast(); + int initializer_id = onnx_net->getInputInitializerId(node_id, input_id); + Mat mat_constant; + if (initializer_id != -1) // initializer + { + mat_constant = onnx_net->getMatFromInitializer(initializer_id); + } + else + { + const Ptr node = net->getNode(node_id); + int constant_id = getInputNodeId(net, node, input_id); + Ptr constant_ptr = net->getNode(constant_id); + opencv_onnx::NodeProto* constant_node = constant_ptr.dynamicCast()->node; + opencv_onnx::TensorProto constant_proto = constant_node->attribute(0).t(); + mat_constant = getMatFromTensor(constant_proto); + } + + std::vector retvals{mat_constant.begin(), mat_constant.end()}; + return retvals; + } + + virtual bool match(const Ptr& net, int nodeId, + std::vector& matchedNodesIds, + std::vector& targetNodesIds) CV_OVERRIDE { + if (Subgraph::match(net, nodeId, matchedNodesIds, targetNodesIds)) { + int64_t value_ConstantOfShape; + if (!extractValue(net, matchedNodesIds[0], value_ConstantOfShape)) { + return false; + } + std::vector input_ConstantOfShape = extractConstant(net, matchedNodesIds[0], 0); + if (input_ConstantOfShape.size() != static_cast(1)) { + return false; + } + + auto B_Mul = extractConstant(net, matchedNodesIds[1], 1); + if (B_Mul.size() != static_cast(1)) { + return false; + } + + auto A_Equal = extractConstant(net, matchedNodesIds[2], 0); + if (A_Equal.size() != static_cast(input_ConstantOfShape[0])) { + return false; + } + + auto Y_Where = extractConstant(net, matchedNodesIds[3], 2); + if (Y_Where.size() != A_Equal.size()) { + return false; + } + + // run ConstantOfShape + std::vector output_ConstantOfShape(std::accumulate(input_ConstantOfShape.begin(), input_ConstantOfShape.end(), static_cast(1), std::multiplies()), value_ConstantOfShape); + // run Mul + std::vector output_Mul = output_ConstantOfShape; + for (size_t i = 0; i < output_Mul.size(); i++) { + int64_t b = B_Mul[0]; + output_Mul[i] *= b; + } + // run Equal + std::vector output_Equal(output_Mul.size()); + for (int i = 0; i < output_Equal.size(); i++) { + if (A_Equal[i] == output_Mul[i]) { + output_Equal[i] = true; + } else { + output_Equal[i] = false; + } + } + // run Where + std::vector output_Where(output_Equal.size()); + for (int i = 0; i < output_Where.size(); i++) { + if (output_Equal[i]) { + output_Where[i] = output_ConstantOfShape[i]; + } else { + output_Where[i] = Y_Where[i]; + } + } + shape = output_Where; + + return true; + } + return false; + } + + virtual void finalize(const Ptr& graph, + const Ptr& fusedNode, + std::vector >& inputs) CV_OVERRIDE { + // replace values + opencv_onnx::NodeProto* node_shape = inputs[1].dynamicCast()->node; + auto attr = node_shape->mutable_attribute()->Mutable(0); + auto tensor = attr->mutable_t(); + tensor->clear_raw_data(); + tensor->set_raw_data(std::string((const char*)(shape.data()), shape.size() * sizeof(int64_t))); + } + +protected: + std::vector shape; }; class MishSubgraph : public Subgraph diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index bd80b0a6d7..1dc4d056c7 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -93,8 +93,6 @@ class ONNXImporter const opencv_onnx::NodeProto& node_proto); void setParamsDtype(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - void expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, - const std::string& input, size_t n); void lstm_extractConsts(LayerParams& layerParams, const opencv_onnx::NodeProto& lstm_proto, size_t idx, int* blobShape_, int size); void lstm_add_reshape(const std::string& input_name, const std::string& output_name, int* layerShape, size_t n); std::string lstm_add_slice(int index, const std::string& input_name, int* begin, int* end, size_t n); @@ -194,6 +192,7 @@ private: void parseTile (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseLayerNorm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseSimpleLayers (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseEinsum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); // Domain: com.microsoft // URL: https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md @@ -656,37 +655,6 @@ void ONNXImporter::addLayer(LayerParams& layerParams, } } -/** @brief Make N copies of input layer and set them as input to node_proto. - * @param prefix prefix of new layers' names - * @param node_proto node which will contain all copies as inputs - * @param input name of the node to copy - * @param n number of copies - */ -void ONNXImporter::expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, - const std::string& input, size_t n) -{ - std::vector input_names; - input_names.reserve(n); - for (size_t j = 0; j < n; j++) - { - LayerParams copyLP; - copyLP.name = format("%s/copy_%zu", prefix.c_str(), j); - copyLP.type = "Identity"; - CV_Assert((layer_id.find(copyLP.name) == layer_id.end()) && - "Couldn't copy the node: generated name already exists in the graph."); - input_names.push_back(copyLP.name); - - node_proto.set_input(0, input); - node_proto.set_output(0, copyLP.name); - addLayer(copyLP, node_proto); - } - node_proto.clear_input(); - for (size_t i = 0; i < input_names.size(); i++) - { - node_proto.add_input(input_names[i]); - } -} - void ONNXImporter::addConstant(const std::string& name, const Mat& blob) { CV_LOG_DEBUG(NULL, "DNN/ONNX: add constant '" << name << "' shape=" << toString(shape(blob)) << ": " << toString(blob)); @@ -1951,73 +1919,45 @@ void ONNXImporter::parseBatchNormalization(LayerParams& layerParams, const openc addLayer(layerParams, node_proto); } -// A * B + C = Y, we require that the dimension of A is [m, k], and the dimension of B is [n, k]. -// And the dim of output Y is [m, n] -void ONNXImporter::parseGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +void ONNXImporter::parseGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "InnerProduct"; - int transA = layerParams.get("transA", 0); - layerParams.set("transA", transA == 1); + auto node_proto = node_proto_; + layerParams.type = "Gemm"; + CV_CheckGE(node_proto.input_size(), 2, "DNN/ONNXImporter: Gemm requires at least two inputs"); + CV_CheckLE(node_proto.input_size(), 3, "DNN/ONNXImporter: Gemm have at most three inputs."); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inputBuf = getBlob(node_proto, 0); - - LayerParams constParams; - constParams.name = node_proto.input(0); - constParams.type = "Const"; - constParams.blobs.push_back(inputBuf); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - - int transB = layerParams.get("transB", 0); - int secondInpDims; - if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) - { - Mat weights = getBlob(node_proto, 1); - secondInpDims = weights.dims; - - if (transA == 0) // optimized barnch, for now, we can only optimize the Gemm when transA = 0. - { - if (transB == 0) - { - transpose(weights, weights); - } - layerParams.set("transB", false); - layerParams.blobs.push_back(weights); - layerParams.set("num_output", layerParams.blobs[0].size[0]); + for (int i = 0; i < node_proto.input_size(); ++i) { + if (i == 2) { + layerParams.set("have_bias", true); + } + if (constBlobs.find(node_proto.input(i)) == constBlobs.end()) { + continue; } - else // no optimized branch, TODO! optimize when the transA==1. - { - LayerParams constParams; - constParams.name = node_proto.input(1); - constParams.type = "Const"; - constParams.blobs.push_back(weights); - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - layerParams.set("transB", transB == 1); + if (i == 2 && constBlobsExtraInfo.find(node_proto.input(2)) != constBlobsExtraInfo.end()) { + layerParams.set("real_ndims_C", getBlobExtraInfo(node_proto, 2).real_ndims); + } + + Mat blob = getBlob(node_proto, i); + + if (i == 0) { // A, always as inputs without prepacking + LayerParams const_A_params; + const_A_params.name = layerParams.name + "/const_A"; + const_A_params.type = "Const"; + const_A_params.blobs.push_back(blob); + + opencv_onnx::NodeProto const_node_proto; + const_node_proto.add_output(const_A_params.name); + addLayer(const_A_params, const_node_proto); + node_proto.set_input(0, const_A_params.name); + } else { // B or C + std::string const_params_name = i == 1 ? "B" : "C"; + + layerParams.blobs.push_back(blob); + layerParams.set(cv::format("const%s", const_params_name.c_str()), true); } } - else - { - layerParams.set("transB", transB == 1); - secondInpDims = outShapes[node_proto.input(1)].size(); - } - if (node_proto.input_size() == 3) - { - Mat bias = getBlob(node_proto, 2); - layerParams.blobs.push_back(bias); - } - - layerParams.set("bias_term", node_proto.input_size() == 3); - layerParams.set("is_matmul", secondInpDims > 2); addLayer(layerParams, node_proto); } @@ -2372,137 +2312,38 @@ void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::N addLayer(layerParams, node_proto); } -void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - opencv_onnx::NodeProto node_proto = node_proto_; - CV_CheckEQ(node_proto.input_size(), 2, ""); - const std::string& input0 = node_proto.input(0); - const std::string& input1 = node_proto.input(1); - const std::string output_name = node_proto.output(0); - Mat newShapeMat = getBlob(input1); - MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); + CV_CheckEQ(node_proto.input_size(), 2, "DNN/ONNXImporter-Expand: two inputs are required"); + // input shape must be constant and it is passed as param to the layer + CV_CheckTrue(constBlobs.find(node_proto.input(1)) != constBlobs.end(), + "DNN/ONNXImporter-Expand: input shape must be constant"); - MatShape inpShape; - bool haveVariables = constBlobs.find(input0) == constBlobs.end(); - if (haveVariables) - { - IterShape_t shapeIt = outShapes.find(input0); - CV_Assert(shapeIt != outShapes.end()); - inpShape = shapeIt->second; - } - else - { - Mat blob = getBlob(input0); - if (constBlobsExtraInfo.find(node_proto.input(0)) != constBlobsExtraInfo.end() && - getBlobExtraInfo(node_proto, 0).real_ndims == 1) { - inpShape = {(int)blob.total()}; - } else { - inpShape = shape(blob); - } + Mat mat_input_shape = getBlob(node_proto, 1); + CV_CheckTypeEQ(mat_input_shape.depth(), CV_32S, "DNN/ONNXImporter-Expand: data type of input shape must be CV_32S"); + for (int i = 0; i < mat_input_shape.total(); ++i) { + CV_Check(i, *(mat_input_shape.ptr() + i) >= 0, "DNN/ONNXImporter-Expand: invalid shape dimension"); } + layerParams.set("shape", DictValue::arrayInt(mat_input_shape.ptr(), mat_input_shape.total())); - String srcName = input0; - // Unsqueeze and repeat along new axis - if (targetShape.size() > inpShape.size()) - { - inpShape.insert(inpShape.begin(), targetShape.size() - inpShape.size(), 1); - for (int i = 0; i < targetShape.size(); i++) - { - if (abs(targetShape[i]) == 1) - targetShape[i] = inpShape[i]; - } - if (haveVariables) - { - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - srcName = reshapeLp.name; - } - } - CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); - - std::vector broadcast_axes; - // shapes aren't right-aligned here because targetShape.size() == inpShape.size() - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] != inpShape[i]) - { - if (inpShape[i] == 1) - { - broadcast_axes.push_back(i); - } - else if (targetShape[i] != 1) - { - CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { + bool const_input_1d = false; + if (constBlobsExtraInfo.find(node_proto.input(0)) != constBlobsExtraInfo.end()) { + if (getBlobExtraInfo(node_proto, 0).real_ndims == 1) { + const_input_1d = true; } } - } - - if (!haveVariables) - { - if (broadcast_axes.empty()) - { - addConstant(output_name, getBlob(node_proto, 0).reshape(1, targetShape)); - return; - } + layerParams.set("const_input_1d", const_input_1d); Mat input = getBlob(node_proto, 0); - MatShape subTargetShape = inpShape; - for (auto broadcast_axis : broadcast_axes) - { - subTargetShape[broadcast_axis] = targetShape[broadcast_axis]; - input = input.reshape(0, total(inpShape, 0, broadcast_axis)); - Mat output = cv::repeat(input, 1, subTargetShape[broadcast_axis]); - input = output.reshape(0, subTargetShape); - } - addConstant(output_name, input); + std::vector inputs, expanded; + inputs.push_back(input); + runLayer(layerParams, inputs, expanded); + CV_CheckEQ(expanded.size(), static_cast(1), "DNN/Expand: only one output is expected when folding constant"); + addConstant(node_proto.output(0), expanded[0]); return; } - if (broadcast_axes.size() == 2 && - broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - CV_Assert(layer_id.find(constParams.name) == layer_id.end()); - constParams.type = "Const"; - - Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); - constParams.blobs.push_back(inp); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - - layerParams.type = "Scale"; - layerParams.set("bias_term", false); - node_proto.set_input(0, constParams.name); - node_proto.set_input(1, srcName); - } - else if (broadcast_axes.size() == 1) - { - // FIXME: this will end up creating massive amount of Identity nodes for broadcasting, - // for example, broadcast 1 to 256 needs 256 Identity nodes and 1 Concat node. - // Possible improvement is to use "Scale". - expandMid(layerParams.name, node_proto, srcName, targetShape[broadcast_axes[0]]); - - layerParams.set("axis", broadcast_axes[0]); - layerParams.type = "Concat"; - node_proto.set_output(0, output_name); - } - else if (broadcast_axes.empty()) - { - layerParams.type = "Identity"; - } - else - CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); addLayer(layerParams, node_proto); } @@ -2666,11 +2507,11 @@ void ONNXImporter::parseGather(LayerParams& layerParams, const opencv_onnx::Node CV_CheckEQ(node_proto.input_size(), 2, ""); // TODO: get rid of the type conversions and 1-d/0-d special-casing when the time comes - if (layer_id.find(node_proto.input(1)) == layer_id.end()) + if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) { int real_ndims = getBlobExtraInfo(node_proto.input(1)).real_ndims; layerParams.set("real_ndims", real_ndims); - if (layer_id.find(node_proto.input(0)) == layer_id.end()) + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) { std::vector inputs, output; @@ -3339,6 +3180,40 @@ void ONNXImporter::parseSimpleLayers(LayerParams& layerParams, const opencv_onnx addLayer(layerParams, node_proto); } + +void ONNXImporter::parseEinsum(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + std::vector einsumInpShapes; + for (int j = 0; j < node_proto.input_size(); j++) + { + const auto& inputLayerName = node_proto.input(j); + auto it = outShapes.find(inputLayerName); + if (it != outShapes.end()) + { + einsumInpShapes.emplace_back(it->second); + } else { + CV_Error(Error::StsAssert, "ERROR input shape not found"); + } + } + + CV_CheckFalse(einsumInpShapes.empty(), "ERROR no inputs shapes"); + for (int i = 0; i < einsumInpShapes.size(); i++) { + layerParams.set("inputShapes" + cv::format("%d", i), DictValue::arrayInt(einsumInpShapes[i].begin(), einsumInpShapes[i].size())); + } + + // Check if of eqution is valid + std::string equation = layerParams.get("equation"); + CV_CheckFalse(equation.empty(), "Equation is empty"); + + // Save number of inputs. We need it in layer initialization + layerParams.set("inputSize", node_proto.input_size()); + + // Save number of outputs. We need it in layer initialization + layerParams.set("outputSize", node_proto.output_size()); + + addLayer(layerParams, node_proto); +} + void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { const std::string& name = layerParams.name; @@ -4045,6 +3920,7 @@ void ONNXImporter::buildDispatchMap_ONNX_AI(int opset_version) dispatch["Sum"] = dispatch["Min"] = dispatch["Max"] = &ONNXImporter::parseElementWise; dispatch["Where"] = &ONNXImporter::parseElementWise; dispatch["Range"] = &ONNXImporter::parseRange; + dispatch["Einsum"] = &ONNXImporter::parseEinsum; std::vector simpleLayers{"Acos", "Acosh", "Asin", "Asinh", "Atan", "Atanh", "Ceil", "Celu", "Cos", "Cosh", "Dropout", "Erf", "Exp", "Floor", "HardSigmoid", "HardSwish", diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp index 2e8cde2799..5531b28111 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp @@ -1120,15 +1120,16 @@ void removePhaseSwitches(tensorflow::GraphDef& net) inpName = inpName.substr(1 + (int)inpName.find('^'), inpName.rfind(':')); nodesMapIt = nodesMap.find(inpName); CV_Assert(nodesMapIt != nodesMap.end()); - int inpNodeId = nodesMapIt->second; + + CV_CheckGT(numConsumers[inpNodeId], 0, + "Input node of the current node should have at least one output node"); if (numConsumers[inpNodeId] == 1) { mergeOpSubgraphNodes.push(inpNodeId); nodesToRemove.push_back(inpNodeId); } - else if (numConsumers[inpNodeId] > 0) - numConsumers[inpNodeId] -= 1; + numConsumers[inpNodeId] -= 1; } } std::sort(nodesToRemove.begin(), nodesToRemove.end()); diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index a23fac187b..756bdc949c 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -2665,14 +2665,20 @@ void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::No connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); } +// ArgMin or ArgMax node void TFImporter::parseArg(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { const std::string& name = layer.name(); const std::string& type = layer.op(); - Mat dimension = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(dimension.total() == 1 && dimension.type() == CV_32SC1); - layerParams.set("axis", *dimension.ptr()); + if (layer.input_size() < 2) + layerParams.set("axis", 0); // default dimension is 0 + else + { + Mat dimension = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(dimension.total() == 1 && dimension.type() == CV_32SC1); + layerParams.set("axis", dimension.at(0)); + } layerParams.set("op", type == "ArgMax" ? "max" : "min"); layerParams.set("keepdims", false); //tensorflow doesn't have this atrr, the output's dims minus one(default); @@ -2866,6 +2872,7 @@ const tensorflow::TensorProto& TFImporter::getConstBlob(const tensorflow::NodeDe if (input_blob_index == -1) CV_Error(Error::StsError, "Const input blob for weights not found"); + CV_CheckLT(input_blob_index, layer.input_size(), "Input index is out of range"); Pin kernel_inp = parsePin(layer.input(input_blob_index)); if (const_layers.find(kernel_inp.name) == const_layers.end()) diff --git a/modules/dnn/src/torch/THDiskFile.cpp b/modules/dnn/src/torch/THDiskFile.cpp index 84b6b23e81..bede95e021 100644 --- a/modules/dnn/src/torch/THDiskFile.cpp +++ b/modules/dnn/src/torch/THDiskFile.cpp @@ -375,15 +375,21 @@ static long THDiskFile_readString(THFile *self, const char *format, char **str_) long total = TBRS_BSZ; long pos = 0L; + if (p == NULL) + THError("read error: failed to allocate buffer"); for (;;) { if(total-pos == 0) /* we need more space! */ { total += TBRS_BSZ; - p = (char*)THRealloc(p, total); + char *new_p = (char*)THRealloc(p, total); + if (new_p == NULL) + { + THFree(p); + THError("read error: failed to reallocate buffer"); + } + p = new_p; } - if (p == NULL) - THError("read error: failed to allocate buffer"); pos += fread(p+pos, 1, total-pos, dfself->handle); if (pos < total) /* eof? */ { @@ -409,15 +415,21 @@ static long THDiskFile_readString(THFile *self, const char *format, char **str_) long pos = 0L; long size; + if (p == NULL) + THError("read error: failed to allocate buffer"); for (;;) { if(total-pos <= 1) /* we can only write '\0' in there! */ { total += TBRS_BSZ; - p = (char*)THRealloc(p, total); + char *new_p = (char*)THRealloc(p, total); + if (new_p == NULL) + { + THFree(p); + THError("read error: failed to reallocate buffer"); + } + p = new_p; } - if (p == NULL) - THError("read error: failed to allocate buffer"); if (fgets(p+pos, total-pos, dfself->handle) == NULL) /* eof? */ { if(pos == 0L) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 9570355b4f..03cc52f7de 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -573,4 +573,937 @@ TEST_P(DNNTestNetwork, FastNeuralStyle_eccv16) INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, dnnBackendsAndTargets(true, true, false, true, true)); +/* + Backend tests of layers +*/ + +static void testLayer(Mat& input, Net& net, Backend backendId, Target targetId, bool skipCheck = false, bool randInput = true, double l1 = 0.0, double lInf = 0.0) +{ + DNNTestLayer::checkBackend(backendId, targetId); + if (randInput) + randu(input, -1.0f, 1.0f); + + net.setInput(input); + net.setPreferableBackend(DNN_BACKEND_OPENCV); + Mat outputDefault = net.forward().clone(); + + net.setPreferableBackend(backendId); + net.setPreferableTarget(targetId); + Mat output = net.forward().clone(); + + if (skipCheck) + return; + + double default_l1, default_lInf; + DNNTestLayer::getDefaultThresholds(backendId, targetId, &default_l1, &default_lInf); + if (l1 == 0.0) + l1 = default_l1; + if (lInf == 0.0) + lInf = default_lInf; + normAssert(outputDefault, output, "", l1, lInf); + if (cvtest::debugLevel > 0 || testing::Test::HasFailure()) + { + std::cout << "l1=" << l1 << " lInf=" << lInf << std::endl; + std::cout << outputDefault.reshape(1, outputDefault.total()).t() << std::endl; + std::cout << output.reshape(1, outputDefault.total()).t() << std::endl; + } +} + +static void testLayer(LayerParams& params, Mat& input, Backend backendId, Target targetId, bool skipCheck = false, double l1 = 0.0, double lInf = 0.0) +{ + Net net; + net.addLayerToPrev(params.name, params.type, params); + testLayer(input, net, backendId, targetId, skipCheck, true, l1, lInf); +} + +class Test_layers_backends : public DNNTestLayer {}; + +//////////////////////////////////////////////////////////////////////////////// +// Padding +//////////////////////////////////////////////////////////////////////////////// +TEST_P(Test_layers_backends, Padding) +{ + static const int kNumRuns = 10; + std::vector paddings(8); + cv::RNG& rng = cv::theRNG(); + for (int t = 0; t < kNumRuns; ++t) + { + for (int i = 0; i < paddings.size(); ++i) + paddings[i] = rng(5); + + LayerParams lp; + lp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); + lp.type = "Padding"; + lp.name = "testLayer"; + + int sz[] = {1 + (int)rng(10), 1 + (int)rng(10), 1 + (int)rng(10), 1 + (int)rng(10)}; + Mat input(4, &sz[0], CV_32F); + testLayer(lp, input, backend, target); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Convolution +//////////////////////////////////////////////////////////////////////////////// +typedef TestWithParam > > Convolution; +TEST_P(Convolution, Accuracy) +{ + int inChannels = get<0>(GetParam())[0]; + int outChannels = get<0>(GetParam())[1]; + int group = get<0>(GetParam())[2]; + Size inSize = get<1>(GetParam()); + Size kernel = get<2>(GetParam()); + Size stride = get<3>(GetParam()); + Size pad = get<4>(GetParam()); + Size dilation = get<5>(GetParam()); + bool hasBias = get<6>(GetParam()); + Backend backendId = get<0>(get<7>(GetParam())); + Target targetId = get<1>(get<7>(GetParam())); + + bool skipCheck = false; + + int sz[] = {outChannels, inChannels / group, kernel.height, kernel.width}; + Mat weights(4, &sz[0], CV_32F); + randu(weights, -1.0f, 1.0f); + + LayerParams lp; + lp.set("kernel_w", kernel.width); + lp.set("kernel_h", kernel.height); + lp.set("pad_w", pad.width); + lp.set("pad_h", pad.height); + lp.set("stride_w", stride.width); + lp.set("stride_h", stride.height); + lp.set("dilation_w", dilation.width); + lp.set("dilation_h", dilation.height); + lp.set("num_output", outChannels); + lp.set("group", group); + lp.set("bias_term", hasBias); + lp.type = "Convolution"; + lp.name = "testLayer"; + lp.blobs.push_back(weights); + if (hasBias) + { + Mat bias(1, outChannels, CV_32F); + randu(bias, -1.0f, 1.0f); + lp.blobs.push_back(bias); + } + int inpSz[] = {1, inChannels, inSize.height, inSize.width}; + Mat input(4, &inpSz[0], CV_32F); + testLayer(lp, input, backendId, targetId, skipCheck); + if (skipCheck) + throw SkipTestException("Skip checks in unstable test"); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, Convolution, testing::Combine( +/*in channels, out channels, group*/ + testing::Values(Vec3i(6, 4, 1), Vec3i(6, 9, 1), + Vec3i(6, 4, 2), Vec3i(6, 9, 3)), +/*in size*/ testing::Values(Size(5, 6)), +/*kernel*/ testing::Values(Size(3, 1), Size(1, 3)), +/*stride*/ testing::Values(Size(1, 1), Size(2, 2)), +/*pad*/ testing::Values(Size(1, 0), Size(0, 1)), +/*dilation*/ testing::Values(Size(1, 1), Size(2, 2)), +/*has bias*/ testing::Bool(), + dnnBackendsAndTargets() +)); + +//////////////////////////////////////////////////////////////////////////////// +// Deconvolution +//////////////////////////////////////////////////////////////////////////////// +typedef TestWithParam > > Deconvolution; +TEST_P(Deconvolution, Accuracy) +{ + int inChannels = get<0>(GetParam())[0]; + int outChannels = get<0>(GetParam())[1]; + int group = get<0>(GetParam())[2]; + Size inSize = get<1>(GetParam()); + Size kernel = get<2>(GetParam()); + Size pad = get<3>(GetParam()); + Size dilation = get<4>(GetParam()); + Size stride = Size(get<5>(GetParam())[0], get<5>(GetParam())[1]); + Size adjPad = Size(get<5>(GetParam())[2], get<5>(GetParam())[3]); + bool hasBias = get<6>(GetParam()); + Backend backendId = get<0>(get<7>(GetParam())); + Target targetId = get<1>(get<7>(GetParam())); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) + && inChannels == 6 && outChannels == 4 && group == 1 + && kernel == Size(3, 1) && pad == Size(0, 1) + && stride == Size(1, 1) && dilation == Size(1, 1)) + applyTestTag(targetId == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) + && inChannels == 6 && outChannels == 4 && group == 1 + && kernel == Size(1, 3) && pad == Size(1, 0) + && stride == Size(1, 1) && dilation == Size(1, 1)) + applyTestTag(targetId == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION + ); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + && inChannels == 6 && outChannels == 4 && group == 1 + && kernel == Size(1, 3) && pad == Size(1, 0) + && stride == Size(1, 1) && dilation == Size(1, 1)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); +#endif + + if (targetId == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + int sz[] = {inChannels, outChannels / group, kernel.height, kernel.width}; + Mat weights(4, &sz[0], CV_32F); + randu(weights, -1.0f, 1.0f); + + LayerParams lp; + lp.set("kernel_w", kernel.width); + lp.set("kernel_h", kernel.height); + lp.set("pad_w", pad.width); + lp.set("pad_h", pad.height); + lp.set("stride_w", stride.width); + lp.set("stride_h", stride.height); + lp.set("dilation_w", dilation.width); + lp.set("dilation_h", dilation.height); + lp.set("adj_w", adjPad.width); + lp.set("adj_h", adjPad.height); + lp.set("num_output", outChannels); + lp.set("group", group); + lp.set("bias_term", hasBias); + lp.type = "Deconvolution"; + lp.name = "testLayer"; + lp.blobs.push_back(weights); + if (hasBias) + { + Mat bias(1, outChannels, CV_32F); + randu(bias, -1.0f, 1.0f); + lp.blobs.push_back(bias); + } + int inpSz[] = {1, inChannels, inSize.height, inSize.width}; + Mat input(4, &inpSz[0], CV_32F); + testLayer(lp, input, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, Deconvolution, testing::Combine( +/*in channels, out channels, group*/ + testing::Values(Vec3i(6, 4, 1), Vec3i(6, 9, 3)), +/*in size*/ testing::Values(Size(5, 6)), +/*kernel*/ testing::Values(Size(3, 1), Size(1, 3)), +/*pad*/ testing::Values(Size(1, 0), Size(0, 1)), +/*dilation*/ testing::Values(Size(1, 1)), +/*stride, adj. pad*/ testing::Values(Vec4i(1,1, 0,0), Vec4i(2,2, 1,0), Vec4i(1,2, 0,1)), +/*has bias*/ testing::Bool(), + dnnBackendsAndTargets() +)); + +//////////////////////////////////////////////////////////////////////////////// +// LRN +//////////////////////////////////////////////////////////////////////////////// +typedef TestWithParam > > LRN; +TEST_P(LRN, Accuracy) +{ + int inChannels = get<0>(GetParam())[0]; + Size inSize = Size(get<0>(GetParam())[1], get<0>(GetParam())[2]); + int localSize = get<1>(GetParam()); + float alpha = get<2>(GetParam())[0]; + float beta = get<2>(GetParam())[1]; + float bias = get<2>(GetParam())[2]; + bool normBySize = get<3>(GetParam()); + std::string nrmType = get<4>(GetParam()); + Backend backendId = get<0>(get<5>(GetParam())); + Target targetId = get<1>(get<5>(GetParam())); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if ((inSize.width == 5 || inSize.height == 5) && targetId == DNN_TARGET_MYRIAD && + nrmType == "ACROSS_CHANNELS") + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); +#endif + + LayerParams lp; + lp.set("norm_region", nrmType); + lp.set("local_size", localSize); + lp.set("alpha", alpha); + lp.set("beta", beta); + lp.set("bias", bias); + lp.set("norm_by_size", normBySize); + lp.type = "LRN"; + lp.name = "testLayer"; + + int sz[] = {1, inChannels, inSize.height, inSize.width}; + Mat input(4, &sz[0], CV_32F); + + double l1 = 0.0, lInf = 0.0; + // The OpenCL kernels use the native_ math functions which have + // implementation defined accuracy, so we use relaxed thresholds. See + // https://github.com/opencv/opencv/issues/9821 for more details. + if (targetId == DNN_TARGET_OPENCL) + { + l1 = 0.01; + lInf = 0.01; + } + testLayer(lp, input, backendId, targetId, false, l1, lInf); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, LRN, testing::Combine( +/*input ch,w,h*/ testing::Values(Vec3i(6, 5, 8), Vec3i(7, 11, 6)), +/*local size*/ testing::Values(3, 5), + testing::Values(Vec3f(0.9f, 1.0f, 1.1f), Vec3f(0.9f, 1.1f, 1.0f), +/*alpha, beta, bias*/ Vec3f(1.0f, 0.9f, 1.1f), Vec3f(1.0f, 1.1f, 0.9f), + Vec3f(1.1f, 0.9f, 1.0f), Vec3f(1.1f, 1.0f, 0.9f)), +/*norm_by_size*/ testing::Bool(), +/*norm_type*/ testing::Values("ACROSS_CHANNELS", "WITHIN_CHANNEL"), + dnnBackendsAndTargets() +)); + +//////////////////////////////////////////////////////////////////////////////// +// Average pooling +//////////////////////////////////////////////////////////////////////////////// +typedef TestWithParam > > AvePooling; +TEST_P(AvePooling, Accuracy) +{ + int inChannels = get<0>(GetParam()); + Size outSize = get<1>(GetParam());; // Input size will be computed from parameters. + Size kernel = get<2>(GetParam()); + Size stride = get<3>(GetParam()); + Backend backendId = get<0>(get<4>(GetParam())); + Target targetId = get<1>(get<4>(GetParam())); + +#if defined(INF_ENGINE_RELEASE) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + && kernel == Size(1, 1) && (stride == Size(1, 1) || stride == Size(2, 2))) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); +#endif + + const int inWidth = (outSize.width - 1) * stride.width + kernel.width; + const int inHeight = (outSize.height - 1) * stride.height + kernel.height; + + LayerParams lp; + lp.set("pool", "ave"); + lp.set("kernel_w", kernel.width); + lp.set("kernel_h", kernel.height); + lp.set("stride_w", stride.width); + lp.set("stride_h", stride.height); + lp.type = "Pooling"; + lp.name = "testLayer"; + + int sz[] = {1, inChannels, inHeight, inWidth}; + Mat input(4, &sz[0], CV_32F); + testLayer(lp, input, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, AvePooling, testing::Combine( +/*in channels*/ testing::Values(3, 4), +/*out size*/ testing::Values(Size(1, 1), Size(2, 2), Size(3, 2), Size(4, 7)), +/*kernel*/ testing::Values(Size(1, 1), Size(2, 2), Size(3, 3), Size(3, 2)), +/*stride*/ testing::Values(Size(1, 1), Size(2, 2), Size(3, 2)), + dnnBackendsAndTargets() +)); + +//////////////////////////////////////////////////////////////////////////////// +// Maximum pooling +//////////////////////////////////////////////////////////////////////////////// +typedef TestWithParam > > MaxPooling; +TEST_P(MaxPooling, Accuracy) +{ + int inChannels = get<0>(GetParam()); + Size inSize = get<1>(GetParam()); + Size kernel = get<2>(GetParam()); + Size stride = get<3>(GetParam()); + Size pad = get<4>(GetParam()); + Backend backendId = get<0>(get<5>(GetParam())); + Target targetId = get<1>(get<5>(GetParam())); + + // https://github.com/openvinotoolkit/openvino/issues/18731 + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && stride != Size(1, 1)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + && (stride == Size(1, 1) || stride == Size(2, 2)) + && (pad == Size(0, 1) || pad == Size(1, 1)) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020020000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + LayerParams lp; + lp.set("pool", "max"); + lp.set("kernel_w", kernel.width); + lp.set("kernel_h", kernel.height); + lp.set("stride_w", stride.width); + lp.set("stride_h", stride.height); + lp.set("pad_w", pad.width); + lp.set("pad_h", pad.height); + lp.type = "Pooling"; + lp.name = "testLayer"; + + int sz[] = {1, inChannels, inSize.height, inSize.width}; + Mat input(4, &sz[0], CV_32F); + testLayer(lp, input, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, MaxPooling, testing::Combine( +/*in channels*/ testing::Values(3, 4), +/*in size*/ testing::Values(Size(5, 5), Size(7, 6)), +/*kernel*/ testing::Values(Size(2, 2), Size(3, 3), Size(3, 2)), +/*stride*/ testing::Values(Size(1, 1), Size(2, 2), Size(3, 2)), +/*pad*/ testing::Values(Size(0, 0), Size(1, 1), Size(0, 1)), + dnnBackendsAndTargets() +)); + +//////////////////////////////////////////////////////////////////////////////// +// Fully-connected +//////////////////////////////////////////////////////////////////////////////// +typedef TestWithParam > > FullyConnected; +TEST_P(FullyConnected, Accuracy) +{ + int batch = get<0>(GetParam()); + int inChannels = get<1>(GetParam()); + Size inSize = get<2>(GetParam()); + int outChannels = get<3>(GetParam()); + bool hasBias = get<4>(GetParam()); + Backend backendId = get<0>(get<5>(GetParam())); + Target targetId = get<1>(get<5>(GetParam())); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (targetId == DNN_TARGET_OPENCL_FP16 || + (targetId == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X))) { + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + // https://github.com/openvinotoolkit/openvino/issues/19436 + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL_FP16 && batch == 16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2023000000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL && batch == 16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL); +#endif + + Mat weights(outChannels, inChannels * inSize.height * inSize.width, CV_32F); + randu(weights, -1.0f, 1.0f); + + Mat bias(1, outChannels, CV_32F); + randu(bias, -1.0f, 1.0f); + + LayerParams lp; + lp.set("num_output", outChannels); + lp.set("bias_term", hasBias); + lp.blobs.push_back(weights); + lp.blobs.push_back(bias); + lp.type = "InnerProduct"; + lp.name = "testLayer"; + + int sz[] = {batch, inChannels, inSize.height, inSize.width}; + Mat input(4, &sz[0], CV_32F); + + double l1 = 0.0; + double lInf = 0.0; +#if defined(INF_ENGINE_RELEASE) + if (targetId == DNN_TARGET_MYRIAD) + { + l1 = 0.015; + lInf = 0.025; + } + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL_FP16) + { + l1 = 0.01; + if (INF_ENGINE_VER_MAJOR_GE(2023000000)) + lInf = 0.016; + } + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL) + { + l1 = 5e-3; + lInf = INF_ENGINE_VER_MAJOR_GE(2023000000) ? 0.016 : 7e-3; + } +#endif + if (targetId == DNN_TARGET_CUDA_FP16) + l1 = 0.015; + + testLayer(lp, input, backendId, targetId, false, l1, lInf); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, FullyConnected, testing::Combine( +/*batch*/ testing::Values(1, 2, 4, 8, 16), +/*in channels*/ testing::Values(3, 4), +/*in size*/ testing::Values(Size(5, 4), Size(4, 5), Size(1, 1)), +/*out channels*/ testing::Values(3, 4), +/*has bias*/ testing::Bool(), + dnnBackendsAndTargets() +)); + +//////////////////////////////////////////////////////////////////////////////// +// SoftMax +//////////////////////////////////////////////////////////////////////////////// +typedef TestWithParam > > SoftMax; +TEST_P(SoftMax, Accuracy) +{ + int inChannels = get<0>(GetParam()); + Backend backendId = get<0>(get<1>(GetParam())); + Target targetId = get<1>(get<1>(GetParam())); + LayerParams lp; + lp.type = "Softmax"; + lp.name = "testLayer"; + + int sz[] = {1, inChannels, 1, 1}; + Mat input(4, &sz[0], CV_32F); + testLayer(lp, input, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, SoftMax, testing::Combine( + testing::Values(3, 4, 5, 1024), + dnnBackendsAndTargets() +)); + +////////////////////////////////////////////////////////////////////////////// +// Max pooling - unpooling +////////////////////////////////////////////////////////////////////////////// +TEST_P(Test_layers_backends, MaxPoolUnpool) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + LayerParams pool; + pool.set("pool", "max"); + pool.set("kernel_w", 2); + pool.set("kernel_h", 2); + pool.set("stride_w", 2); + pool.set("stride_h", 2); + pool.set("pad_w", 0); + pool.set("pad_h", 0); + pool.type = "Pooling"; + pool.name = "testPool"; + + LayerParams unpool; + unpool.set("pool_k_w", 2); + unpool.set("pool_k_h", 2); + unpool.set("pool_stride_w", 2); + unpool.set("pool_stride_h", 2); + unpool.set("pool_pad_w", 0); + unpool.set("pool_pad_h", 0); + unpool.type = "MaxUnpool"; + unpool.name = "testUnpool"; + + Net net; + int poolId = net.addLayer(pool.name, pool.type, pool); + net.connect(0, 0, poolId, 0); + + int unpoolId = net.addLayer(unpool.name, unpool.type, unpool); + net.connect(poolId, 0, unpoolId, 0); + net.connect(poolId, 1, unpoolId, 1); + + int sz[] = {1, 1, 4, 4}; + Mat input(4, &sz[0], CV_32F); + testLayer(input, net, backend, target); +} + +//////////////////////////////////////////////////////////////////////////////// +// AvePooling + in-place layers +//////////////////////////////////////////////////////////////////////////////// +static const int kNumChannels = 3; + +void testInPlaceActivation(LayerParams& lp, Backend backendId, Target targetId, double l1 = 0.0, double lInf = 0.0) +{ + EXPECT_FALSE(lp.name.empty()); + + LayerParams pool; + pool.set("pool", "ave"); + pool.set("kernel_w", 2); + pool.set("kernel_h", 2); + pool.set("stride_w", 2); + pool.set("stride_h", 2); + pool.type = "Pooling"; + pool.name = "ave_pool"; + + Net net; + int poolId = net.addLayer(pool.name, pool.type, pool); + net.connect(0, 0, poolId, 0); + net.addLayerToPrev(lp.name, lp.type, lp); + + int sz[] = {1, kNumChannels, 10, 10}; + Mat input(4, &sz[0], CV_32F); + testLayer(input, net, backendId, targetId, false, true, l1, lInf); +} + +typedef TestWithParam > > BatchNorm; +TEST_P(BatchNorm, Accuracy) +{ + bool hasWeights = get<0>(GetParam()); + bool hasBias = get<1>(GetParam()); + float epsilon = get<2>(GetParam()); + Backend backendId = get<0>(get<3>(GetParam())); + Target targetId = get<1>(get<3>(GetParam())); + + LayerParams lp; + lp.set("has_weight", hasWeights); + lp.set("has_bias", hasBias); + lp.set("eps", epsilon); + lp.type = "BatchNorm"; + lp.name = "testLayer"; + + lp.blobs.reserve(4); + for (int i = 0; i < 3; ++i) + lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); + if (hasBias || hasWeights) + lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); + + for (int i = 0; i < lp.blobs.size(); ++i) + randu(lp.blobs[i], 0.0f, 1.0f); + + testInPlaceActivation(lp, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, BatchNorm, testing::Combine( +/*has weights*/ testing::Bool(), +/*has bias*/ testing::Bool(), +/*epsilon*/ testing::Values(1e-3f, 1e-5f), + dnnBackendsAndTargets() +)); + +typedef TestWithParam > > ReLU; +TEST_P(ReLU, Accuracy) +{ + float negativeSlope = get<0>(GetParam()); + Backend backendId = get<0>(get<1>(GetParam())); + Target targetId = get<1>(get<1>(GetParam())); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD && negativeSlope < 0) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + LayerParams lp; + lp.set("negative_slope", negativeSlope); + lp.type = "ReLU"; + lp.name = "testLayer"; + testInPlaceActivation(lp, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, ReLU, testing::Combine( +/*negative slope*/ testing::Values(2.0f, 0.3f, -0.1f, 0.0f), + dnnBackendsAndTargets() +)); + +typedef TestWithParam > > NoParamActivation; +TEST_P(NoParamActivation, Accuracy) +{ + Backend backendId = get<0>(get<1>(GetParam())); + Target targetId = get<1>(get<1>(GetParam())); + std::string layer_type = get<0>(GetParam()); + + LayerParams lp; + lp.type = layer_type; + lp.name = "testLayer"; + testInPlaceActivation(lp, backendId, targetId); +} +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, NoParamActivation, testing::Combine( +/*type*/ testing::Values("TanH", "Sigmoid", "AbsVal", "BNLL", "Swish", "Mish"), + dnnBackendsAndTargets() +)); + +typedef TestWithParam > > Power; +TEST_P(Power, Accuracy) +{ + float power = get<0>(GetParam())[0]; + float scale = get<0>(GetParam())[1]; + float shift = get<0>(GetParam())[2]; + Backend backendId = get<0>(get<1>(GetParam())); + Target targetId = get<1>(get<1>(GetParam())); + + LayerParams lp; + lp.set("power", power); + lp.set("scale", scale); + lp.set("shift", shift); + lp.type = "Power"; + lp.name = "testLayer"; + testInPlaceActivation(lp, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, Power, testing::Combine( +/*power, scale, shift*/ testing::Values(Vec3f(0.9f, 1.0f, 1.1f), Vec3f(0.9f, 1.1f, 1.0f), + Vec3f(1.0f, 0.9f, 1.1f), Vec3f(1.0f, 1.1f, 0.9f), + Vec3f(1.1f, 0.9f, 1.0f), Vec3f(1.1f, 1.0f, 0.9f)), + dnnBackendsAndTargets() +)); + +typedef TestWithParam > > Exp; +TEST_P(Exp, Accuracy) +{ + float base = get<0>(GetParam())[0]; + float scale = get<0>(GetParam())[1]; + float shift = get<0>(GetParam())[2]; + Backend backendId = get<0>(get<1>(GetParam())); + Target targetId = get<1>(get<1>(GetParam())); + + LayerParams lp; + lp.set("base", base); + lp.set("scale", scale); + lp.set("shift", shift); + lp.type = "Exp"; + lp.name = "testLayer"; + testInPlaceActivation(lp, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, Exp, testing::Combine( +/*base, scale, shift*/ testing::Values(Vec3f(0.9f, -1.0f, 1.1f), Vec3f(0.9f, 1.1f, -1.0f), + Vec3f(-1.0f, 0.9f, 1.1f), Vec3f(-1.0f, 1.1f, 0.9f), + Vec3f(1.1f, 0.9f, -1.0f), Vec3f(1.1f, -1.0f, 0.9f)), + dnnBackendsAndTargets() +)); + +TEST_P(Test_layers_backends, ChannelsPReLU) +{ + LayerParams lp; + lp.type = "ChannelsPReLU"; + lp.name = "testLayer"; + lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); + randu(lp.blobs[0], -1.0f, 1.0f); + + testInPlaceActivation(lp, backend, target); +} + +typedef TestWithParam > > Scale; +TEST_P(Scale, Accuracy) +{ + bool hasBias = get<0>(GetParam()); + Backend backendId = get<0>(get<1>(GetParam())); + Target targetId = get<1>(get<1>(GetParam())); + + LayerParams lp; + lp.set("bias_term", hasBias); + lp.type = "Scale"; + lp.name = "testLayer"; + lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); + randu(lp.blobs[0], -1.0f, 1.0f); + if (hasBias) + { + lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); + randu(lp.blobs[1], -1.0f, 1.0f); + } + testInPlaceActivation(lp, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, Scale, testing::Combine( + testing::Bool(), + dnnBackendsAndTargets() +)); + +//////////////////////////////////////////////////////////////////////////////// +// Concat layer +//////////////////////////////////////////////////////////////////////////////// +// +// input --- conv --- concat --- output +// `--- conv ----^ ^ ^ +// `---- ... ------' ' +// `-----------------' +typedef TestWithParam > > Concat; +TEST_P(Concat, Accuracy) +{ + Vec3i inSize = get<0>(GetParam()); + Vec3i numChannels = get<1>(GetParam()); + Backend backendId = get<0>(get<2>(GetParam())); + Target targetId = get<1>(get<2>(GetParam())); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD + && inSize == Vec3i(1, 4, 5) && numChannels == Vec3i(1, 6, 2) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); // crash +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_CPU + && inSize == Vec3i(1, 4, 5) && numChannels == Vec3i(1, 6, 2) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); // TODO: IE_CPU +#endif + + Net net; + + std::vector convLayerIds; + convLayerIds.reserve(numChannels.channels); + for (int i = 0, n = numChannels.channels; i < n; ++i) + { + if (!numChannels[i]) + break; + + int sz[] = {numChannels[i], inSize[0], 1, 1}; + Mat weights(4, &sz[0], CV_32F); + randu(weights, -1.0f, 1.0f); + + LayerParams convParam; + convParam.set("kernel_w", 1); + convParam.set("kernel_h", 1); + convParam.set("num_output", numChannels[i]); + convParam.set("bias_term", false); + convParam.type = "Convolution"; + std::ostringstream ss; + ss << "convLayer" << i; + convParam.name = ss.str(); + convParam.blobs.push_back(weights); + + int layerId = net.addLayer(convParam.name, convParam.type, convParam); + convLayerIds.push_back(layerId); + net.connect(0, 0, layerId, 0); + } + + LayerParams concatParam; + concatParam.type = "Concat"; + concatParam.name = "testLayer"; + int concatId = net.addLayer(concatParam.name, concatParam.type, concatParam); + net.connect(0, 0, concatId, 0); + for (int i = 0; i < convLayerIds.size(); ++i) + { + net.connect(convLayerIds[i], 0, concatId, i + 1); + } + + int sz[] = {1, inSize[0], inSize[1], inSize[2]}; + Mat input(4, &sz[0], CV_32F); + testLayer(input, net, backendId, targetId); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, Concat, testing::Combine( +/*input size*/ testing::Values(Vec3i(1, 4, 5), Vec3i(2, 8, 6)), +/*channels*/ testing::Values(Vec3i(2, 0, 0), Vec3i(3, 4, 0), Vec3i(1, 6, 2)), + dnnBackendsAndTargets() +)); + +//////////////////////////////////////////////////////////////////////////////// +// Element-wise layers +//////////////////////////////////////////////////////////////////////////////// +// +// input --- conv --- eltwise --- output +// `--- conv ----^ ^ ^ +// `---- ... ------' ' +// `-----------------' +typedef TestWithParam > > Eltwise; +TEST_P(Eltwise, Accuracy) +{ + Vec3i inSize = get<0>(GetParam()); + std::string op = get<1>(GetParam()); + int numConv = get<2>(GetParam()); + bool weighted = get<3>(GetParam()); + Backend backendId = get<0>(get<4>(GetParam())); + Target targetId = get<1>(get<4>(GetParam())); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) + // accuracy + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL && + inSize == Vec3i(1, 4, 5) && op == "sum" && numConv == 1 && !weighted) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL && + inSize == Vec3i(2, 8, 6) && op == "sum" && numConv == 1 && !weighted) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD && + inSize == Vec3i(1, 4, 5)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && numConv > 1) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_OPENCL && + op == "sum" && numConv == 1 && !weighted) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); +#endif + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && numConv > 1) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + bool convInputShift = 1; + int numEltwiseInputs = numConv; + if (op == "div") + { + numConv = 1; + convInputShift = 0; // first input is convolution + } + + Net net; + + std::vector convLayerIds(numConv); + for (int i = 0; i < numConv; ++i) + { + int sz[] = {inSize[0], inSize[0], 1, 1}; + Mat weights(4, &sz[0], CV_32F); + randu(weights, -1.0f, 1.0f); + + LayerParams convParam; + convParam.set("kernel_w", 1); + convParam.set("kernel_h", 1); + convParam.set("num_output", inSize[0]); + convParam.set("bias_term", false); + convParam.type = "Convolution"; + std::ostringstream ss; + ss << "convLayer" << i; + convParam.name = ss.str(); + convParam.blobs.push_back(weights); + + convLayerIds[i] = net.addLayer(convParam.name, convParam.type, convParam); + net.connect(0, 0, convLayerIds[i], 0); + } + + LayerParams eltwiseParam; + eltwiseParam.set("operation", op); + if (op == "sum" && weighted) + { + RNG& rng = cv::theRNG(); + std::vector coeff(1 + numConv); + for (int i = 0; i < coeff.size(); ++i) + { + coeff[i] = rng.uniform(-2.0f, 2.0f); + } + eltwiseParam.set("coeff", DictValue::arrayReal(&coeff[0], coeff.size())); + } + eltwiseParam.type = "Eltwise"; + eltwiseParam.name = "testLayer"; + int eltwiseId = net.addLayer(eltwiseParam.name, eltwiseParam.type, eltwiseParam); + if (convInputShift == 1) + net.connect(0, 0, eltwiseId, 0); + for (int i = 0; i < numConv; ++i) + { + net.connect(convLayerIds[i], 0, eltwiseId, i + convInputShift); + } + if (convInputShift == 0) + net.connect(0, 0, eltwiseId, numConv); + for (int i = numConv; i < numEltwiseInputs; ++i) + { + net.connect(0, 0, eltwiseId, i + 1); + } + + int sz[] = {1, inSize[0], inSize[1], inSize[2]}; + Mat input(4, &sz[0], CV_32F); + if (op == "div") + randu(input, 1.0f, 1.0f); // ensure no divisor value has absouluate value of less than 0.5 + testLayer(input, net, backendId, targetId, /*skipCheck*/false, (op == "div") ? false : true); +} + +INSTANTIATE_TEST_CASE_P(Layer_Test_Backends, Eltwise, testing::Combine( +/*input size*/ testing::Values(Vec3i(1, 4, 5), Vec3i(2, 8, 6)), +/*operation*/ testing::Values("prod", "sum", "div", "max", "min"), +/*num convs*/ testing::Values(1, 2, 3), +/*weighted(for sum only)*/ testing::Bool(), + dnnBackendsAndTargets() +)); + +INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_layers_backends, dnnBackendsAndTargets()); + }} // namespace diff --git a/modules/dnn/test/test_caffe_importer.cpp b/modules/dnn/test/test_caffe_importer.cpp index 66eff49979..14c3ffc6d4 100644 --- a/modules/dnn/test/test_caffe_importer.cpp +++ b/modules/dnn/test/test_caffe_importer.cpp @@ -731,7 +731,7 @@ TEST_P(Test_Caffe_nets, FasterRCNN_vgg16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif - double scoreDiff = 0.001, iouDiff = 0.03; + double scoreDiff = 0.0012, iouDiff = 0.03; #if defined(INF_ENGINE_RELEASE) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); diff --git a/modules/dnn/test/test_halide_layers.cpp b/modules/dnn/test/test_halide_layers.cpp deleted file mode 100644 index 12e62c754a..0000000000 --- a/modules/dnn/test/test_halide_layers.cpp +++ /dev/null @@ -1,999 +0,0 @@ -// This file is part of OpenCV project. -// It is subject to the license terms in the LICENSE file found in the top-level directory -// of this distribution and at http://opencv.org/license.html. -// -// Copyright (C) 2017-2019, Intel Corporation, all rights reserved. -// Third party copyrights are property of their respective owners. - -// This tests doesn't require any external data. They just compare outputs of -// layers using different computation backends. Input and parameters are random. - -#include "test_precomp.hpp" - -namespace opencv_test { namespace { - -using namespace cv; -using namespace cv::dnn; -using namespace testing; - -static void test(Mat& input, Net& net, Backend backendId, Target targetId, bool skipCheck = false, bool randInput = true, double l1 = 0.0, double lInf = 0.0) -{ - DNNTestLayer::checkBackend(backendId, targetId); - if (randInput) - randu(input, -1.0f, 1.0f); - - net.setInput(input); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - Mat outputDefault = net.forward().clone(); - - net.setPreferableBackend(backendId); - net.setPreferableTarget(targetId); - Mat outputHalide = net.forward().clone(); - - if (skipCheck) - return; - - double default_l1, default_lInf; - DNNTestLayer::getDefaultThresholds(backendId, targetId, &default_l1, &default_lInf); - if (l1 == 0.0) - l1 = default_l1; - if (lInf == 0.0) - lInf = default_lInf; - normAssert(outputDefault, outputHalide, "", l1, lInf); - if (cvtest::debugLevel > 0 || testing::Test::HasFailure()) - { - std::cout << "l1=" << l1 << " lInf=" << lInf << std::endl; - std::cout << outputDefault.reshape(1, outputDefault.total()).t() << std::endl; - std::cout << outputHalide.reshape(1, outputDefault.total()).t() << std::endl; - } -} - -static void test(LayerParams& params, Mat& input, Backend backendId, Target targetId, bool skipCheck = false, double l1 = 0.0, double lInf = 0.0) -{ - Net net; - net.addLayerToPrev(params.name, params.type, params); - test(input, net, backendId, targetId, skipCheck, true, l1, lInf); -} - -static inline testing::internal::ParamGenerator > dnnBackendsAndTargetsWithHalide() -{ - return dnnBackendsAndTargets(true, true, false); // OpenCV/CPU is used as reference -} - -class Test_Halide_layers : public DNNTestLayer {}; - -//////////////////////////////////////////////////////////////////////////////// -// Padding -//////////////////////////////////////////////////////////////////////////////// -TEST_P(Test_Halide_layers, Padding) -{ - static const int kNumRuns = 10; - std::vector paddings(8); - cv::RNG& rng = cv::theRNG(); - for (int t = 0; t < kNumRuns; ++t) - { - for (int i = 0; i < paddings.size(); ++i) - paddings[i] = rng(5); - - LayerParams lp; - lp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); - lp.type = "Padding"; - lp.name = "testLayer"; - - int sz[] = {1 + (int)rng(10), 1 + (int)rng(10), 1 + (int)rng(10), 1 + (int)rng(10)}; - Mat input(4, &sz[0], CV_32F); - test(lp, input, backend, target); - } -} - -//////////////////////////////////////////////////////////////////////////////// -// Convolution -//////////////////////////////////////////////////////////////////////////////// -typedef TestWithParam > > Convolution; -TEST_P(Convolution, Accuracy) -{ - int inChannels = get<0>(GetParam())[0]; - int outChannels = get<0>(GetParam())[1]; - int group = get<0>(GetParam())[2]; - Size inSize = get<1>(GetParam()); - Size kernel = get<2>(GetParam()); - Size stride = get<3>(GetParam()); - Size pad = get<4>(GetParam()); - Size dilation = get<5>(GetParam()); - bool hasBias = get<6>(GetParam()); - Backend backendId = get<0>(get<7>(GetParam())); - Target targetId = get<1>(get<7>(GetParam())); - - bool skipCheck = false; - - int sz[] = {outChannels, inChannels / group, kernel.height, kernel.width}; - Mat weights(4, &sz[0], CV_32F); - randu(weights, -1.0f, 1.0f); - - LayerParams lp; - lp.set("kernel_w", kernel.width); - lp.set("kernel_h", kernel.height); - lp.set("pad_w", pad.width); - lp.set("pad_h", pad.height); - lp.set("stride_w", stride.width); - lp.set("stride_h", stride.height); - lp.set("dilation_w", dilation.width); - lp.set("dilation_h", dilation.height); - lp.set("num_output", outChannels); - lp.set("group", group); - lp.set("bias_term", hasBias); - lp.type = "Convolution"; - lp.name = "testLayer"; - lp.blobs.push_back(weights); - if (hasBias) - { - Mat bias(1, outChannels, CV_32F); - randu(bias, -1.0f, 1.0f); - lp.blobs.push_back(bias); - } - int inpSz[] = {1, inChannels, inSize.height, inSize.width}; - Mat input(4, &inpSz[0], CV_32F); - test(lp, input, backendId, targetId, skipCheck); - if (skipCheck) - throw SkipTestException("Skip checks in unstable test"); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Convolution, Combine( -/*in channels, out channels, group*/ - Values(Vec3i(6, 4, 1), Vec3i(6, 9, 1), - Vec3i(6, 4, 2), Vec3i(6, 9, 3)), -/*in size*/ Values(Size(5, 6)), -/*kernel*/ Values(Size(3, 1), Size(1, 3)), -/*stride*/ Values(Size(1, 1), Size(2, 2)), -/*pad*/ Values(Size(1, 0), Size(0, 1)), -/*dilation*/ Values(Size(1, 1), Size(2, 2)), -/*has bias*/ Bool(), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////////// -// Deconvolution -//////////////////////////////////////////////////////////////////////////////// -typedef TestWithParam > > Deconvolution; -TEST_P(Deconvolution, Accuracy) -{ - int inChannels = get<0>(GetParam())[0]; - int outChannels = get<0>(GetParam())[1]; - int group = get<0>(GetParam())[2]; - Size inSize = get<1>(GetParam()); - Size kernel = get<2>(GetParam()); - Size pad = get<3>(GetParam()); - Size dilation = get<4>(GetParam()); - Size stride = Size(get<5>(GetParam())[0], get<5>(GetParam())[1]); - Size adjPad = Size(get<5>(GetParam())[2], get<5>(GetParam())[3]); - bool hasBias = get<6>(GetParam()); - Backend backendId = get<0>(get<7>(GetParam())); - Target targetId = get<1>(get<7>(GetParam())); - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) - && inChannels == 6 && outChannels == 4 && group == 1 - && kernel == Size(3, 1) && pad == Size(0, 1) - && stride == Size(1, 1) && dilation == Size(1, 1)) - applyTestTag(targetId == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION - ); - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16) - && inChannels == 6 && outChannels == 4 && group == 1 - && kernel == Size(1, 3) && pad == Size(1, 0) - && stride == Size(1, 1) && dilation == Size(1, 1)) - applyTestTag(targetId == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, - CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION - ); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD - && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X - && inChannels == 6 && outChannels == 4 && group == 1 - && kernel == Size(1, 3) && pad == Size(1, 0) - && stride == Size(1, 1) && dilation == Size(1, 1)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); -#endif - - if (targetId == DNN_TARGET_CUDA_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); - - int sz[] = {inChannels, outChannels / group, kernel.height, kernel.width}; - Mat weights(4, &sz[0], CV_32F); - randu(weights, -1.0f, 1.0f); - - LayerParams lp; - lp.set("kernel_w", kernel.width); - lp.set("kernel_h", kernel.height); - lp.set("pad_w", pad.width); - lp.set("pad_h", pad.height); - lp.set("stride_w", stride.width); - lp.set("stride_h", stride.height); - lp.set("dilation_w", dilation.width); - lp.set("dilation_h", dilation.height); - lp.set("adj_w", adjPad.width); - lp.set("adj_h", adjPad.height); - lp.set("num_output", outChannels); - lp.set("group", group); - lp.set("bias_term", hasBias); - lp.type = "Deconvolution"; - lp.name = "testLayer"; - lp.blobs.push_back(weights); - if (hasBias) - { - Mat bias(1, outChannels, CV_32F); - randu(bias, -1.0f, 1.0f); - lp.blobs.push_back(bias); - } - int inpSz[] = {1, inChannels, inSize.height, inSize.width}; - Mat input(4, &inpSz[0], CV_32F); - test(lp, input, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Deconvolution, Combine( -/*in channels, out channels, group*/ - Values(Vec3i(6, 4, 1), Vec3i(6, 9, 3)), -/*in size*/ Values(Size(5, 6)), -/*kernel*/ Values(Size(3, 1), Size(1, 3)), -/*pad*/ Values(Size(1, 0), Size(0, 1)), -/*dilation*/ Values(Size(1, 1)), -/*stride, adj. pad*/ Values(Vec4i(1,1, 0,0), Vec4i(2,2, 1,0), Vec4i(1,2, 0,1)), -/*has bias*/ Bool(), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////////// -// LRN -//////////////////////////////////////////////////////////////////////////////// -typedef TestWithParam > > LRN; -TEST_P(LRN, Accuracy) -{ - int inChannels = get<0>(GetParam())[0]; - Size inSize = Size(get<0>(GetParam())[1], get<0>(GetParam())[2]); - int localSize = get<1>(GetParam()); - float alpha = get<2>(GetParam())[0]; - float beta = get<2>(GetParam())[1]; - float bias = get<2>(GetParam())[2]; - bool normBySize = get<3>(GetParam()); - std::string nrmType = get<4>(GetParam()); - Backend backendId = get<0>(get<5>(GetParam())); - Target targetId = get<1>(get<5>(GetParam())); - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if ((inSize.width == 5 || inSize.height == 5) && targetId == DNN_TARGET_MYRIAD && - nrmType == "ACROSS_CHANNELS") - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); -#endif - - LayerParams lp; - lp.set("norm_region", nrmType); - lp.set("local_size", localSize); - lp.set("alpha", alpha); - lp.set("beta", beta); - lp.set("bias", bias); - lp.set("norm_by_size", normBySize); - lp.type = "LRN"; - lp.name = "testLayer"; - - int sz[] = {1, inChannels, inSize.height, inSize.width}; - Mat input(4, &sz[0], CV_32F); - - double l1 = 0.0, lInf = 0.0; - // The OpenCL kernels use the native_ math functions which have - // implementation defined accuracy, so we use relaxed thresholds. See - // https://github.com/opencv/opencv/issues/9821 for more details. - if (targetId == DNN_TARGET_OPENCL) - { - l1 = 0.01; - lInf = 0.01; - } - test(lp, input, backendId, targetId, false, l1, lInf); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, LRN, Combine( -/*input ch,w,h*/ Values(Vec3i(6, 5, 8), Vec3i(7, 11, 6)), -/*local size*/ Values(3, 5), - Values(Vec3f(0.9f, 1.0f, 1.1f), Vec3f(0.9f, 1.1f, 1.0f), -/*alpha, beta, bias*/ Vec3f(1.0f, 0.9f, 1.1f), Vec3f(1.0f, 1.1f, 0.9f), - Vec3f(1.1f, 0.9f, 1.0f), Vec3f(1.1f, 1.0f, 0.9f)), -/*norm_by_size*/ Bool(), -/*norm_type*/ Values("ACROSS_CHANNELS", "WITHIN_CHANNEL"), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////////// -// Average pooling -//////////////////////////////////////////////////////////////////////////////// -typedef TestWithParam > > AvePooling; -TEST_P(AvePooling, Accuracy) -{ - int inChannels = get<0>(GetParam()); - Size outSize = get<1>(GetParam());; // Input size will be computed from parameters. - Size kernel = get<2>(GetParam()); - Size stride = get<3>(GetParam()); - Backend backendId = get<0>(get<4>(GetParam())); - Target targetId = get<1>(get<4>(GetParam())); - -#if defined(INF_ENGINE_RELEASE) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD - && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X - && kernel == Size(1, 1) && (stride == Size(1, 1) || stride == Size(2, 2))) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); -#endif - - const int inWidth = (outSize.width - 1) * stride.width + kernel.width; - const int inHeight = (outSize.height - 1) * stride.height + kernel.height; - - LayerParams lp; - lp.set("pool", "ave"); - lp.set("kernel_w", kernel.width); - lp.set("kernel_h", kernel.height); - lp.set("stride_w", stride.width); - lp.set("stride_h", stride.height); - lp.type = "Pooling"; - lp.name = "testLayer"; - - int sz[] = {1, inChannels, inHeight, inWidth}; - Mat input(4, &sz[0], CV_32F); - test(lp, input, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, AvePooling, Combine( -/*in channels*/ Values(3, 4), -/*out size*/ Values(Size(1, 1), Size(2, 2), Size(3, 2), Size(4, 7)), -/*kernel*/ Values(Size(1, 1), Size(2, 2), Size(3, 3), Size(3, 2)), -/*stride*/ Values(Size(1, 1), Size(2, 2), Size(3, 2)), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////////// -// Maximum pooling -//////////////////////////////////////////////////////////////////////////////// -typedef TestWithParam > > MaxPooling; -TEST_P(MaxPooling, Accuracy) -{ - int inChannels = get<0>(GetParam()); - Size inSize = get<1>(GetParam()); - Size kernel = get<2>(GetParam()); - Size stride = get<3>(GetParam()); - Size pad = get<4>(GetParam()); - Backend backendId = get<0>(get<5>(GetParam())); - Target targetId = get<1>(get<5>(GetParam())); - - // https://github.com/openvinotoolkit/openvino/issues/18731 - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && stride != Size(1, 1)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD - && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X - && (stride == Size(1, 1) || stride == Size(2, 2)) - && (pad == Size(0, 1) || pad == Size(1, 1)) - ) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020020000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - - LayerParams lp; - lp.set("pool", "max"); - lp.set("kernel_w", kernel.width); - lp.set("kernel_h", kernel.height); - lp.set("stride_w", stride.width); - lp.set("stride_h", stride.height); - lp.set("pad_w", pad.width); - lp.set("pad_h", pad.height); - lp.type = "Pooling"; - lp.name = "testLayer"; - - int sz[] = {1, inChannels, inSize.height, inSize.width}; - Mat input(4, &sz[0], CV_32F); - test(lp, input, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, MaxPooling, Combine( -/*in channels*/ Values(3, 4), -/*in size*/ Values(Size(5, 5), Size(7, 6)), -/*kernel*/ Values(Size(2, 2), Size(3, 3), Size(3, 2)), -/*stride*/ Values(Size(1, 1), Size(2, 2), Size(3, 2)), -/*pad*/ Values(Size(0, 0), Size(1, 1), Size(0, 1)), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////////// -// Fully-connected -//////////////////////////////////////////////////////////////////////////////// -typedef TestWithParam > > FullyConnected; -TEST_P(FullyConnected, Accuracy) -{ - int batch = get<0>(GetParam()); - int inChannels = get<1>(GetParam()); - Size inSize = get<2>(GetParam()); - int outChannels = get<3>(GetParam()); - bool hasBias = get<4>(GetParam()); - Backend backendId = get<0>(get<5>(GetParam())); - Target targetId = get<1>(get<5>(GetParam())); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if ((backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || - backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (targetId == DNN_TARGET_OPENCL_FP16 || - (targetId == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X))) { - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); - } -#endif - // https://github.com/openvinotoolkit/openvino/issues/19436 - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL_FP16 && batch == 16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2023000000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL && batch == 16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL); -#endif - - Mat weights(outChannels, inChannels * inSize.height * inSize.width, CV_32F); - randu(weights, -1.0f, 1.0f); - - Mat bias(1, outChannels, CV_32F); - randu(bias, -1.0f, 1.0f); - - LayerParams lp; - lp.set("num_output", outChannels); - lp.set("bias_term", hasBias); - lp.blobs.push_back(weights); - lp.blobs.push_back(bias); - lp.type = "InnerProduct"; - lp.name = "testLayer"; - - int sz[] = {batch, inChannels, inSize.height, inSize.width}; - Mat input(4, &sz[0], CV_32F); - - double l1 = 0.0; - double lInf = 0.0; -#if defined(INF_ENGINE_RELEASE) - if (targetId == DNN_TARGET_MYRIAD) - { - l1 = 0.015; - lInf = 0.025; - } - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL_FP16) - { - l1 = 0.01; - if (INF_ENGINE_VER_MAJOR_GE(2023000000)) - lInf = 0.016; - } - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL) - { - l1 = 5e-3; - lInf = INF_ENGINE_VER_MAJOR_GE(2023000000) ? 0.016 : 7e-3; - } -#endif - if (targetId == DNN_TARGET_CUDA_FP16) - l1 = 0.015; - - test(lp, input, backendId, targetId, false, l1, lInf); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, FullyConnected, Combine( -/*batch*/ Values(1, 2, 4, 8, 16), -/*in channels*/ Values(3, 4), -/*in size*/ Values(Size(5, 4), Size(4, 5), Size(1, 1)), -/*out channels*/ Values(3, 4), -/*has bias*/ Bool(), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////////// -// SoftMax -//////////////////////////////////////////////////////////////////////////////// -typedef TestWithParam > > SoftMax; -TEST_P(SoftMax, Accuracy) -{ - int inChannels = get<0>(GetParam()); - Backend backendId = get<0>(get<1>(GetParam())); - Target targetId = get<1>(get<1>(GetParam())); - LayerParams lp; - lp.type = "Softmax"; - lp.name = "testLayer"; - - int sz[] = {1, inChannels, 1, 1}; - Mat input(4, &sz[0], CV_32F); - test(lp, input, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, SoftMax, Combine( - Values(3, 4, 5, 1024), - dnnBackendsAndTargetsWithHalide() -)); - -////////////////////////////////////////////////////////////////////////////// -// Max pooling - unpooling -////////////////////////////////////////////////////////////////////////////// -TEST_P(Test_Halide_layers, MaxPoolUnpool) -{ - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); - - LayerParams pool; - pool.set("pool", "max"); - pool.set("kernel_w", 2); - pool.set("kernel_h", 2); - pool.set("stride_w", 2); - pool.set("stride_h", 2); - pool.set("pad_w", 0); - pool.set("pad_h", 0); - pool.type = "Pooling"; - pool.name = "testPool"; - - LayerParams unpool; - unpool.set("pool_k_w", 2); - unpool.set("pool_k_h", 2); - unpool.set("pool_stride_w", 2); - unpool.set("pool_stride_h", 2); - unpool.set("pool_pad_w", 0); - unpool.set("pool_pad_h", 0); - unpool.type = "MaxUnpool"; - unpool.name = "testUnpool"; - - Net net; - int poolId = net.addLayer(pool.name, pool.type, pool); - net.connect(0, 0, poolId, 0); - - int unpoolId = net.addLayer(unpool.name, unpool.type, unpool); - net.connect(poolId, 0, unpoolId, 0); - net.connect(poolId, 1, unpoolId, 1); - - int sz[] = {1, 1, 4, 4}; - Mat input(4, &sz[0], CV_32F); - test(input, net, backend, target); -} - -//////////////////////////////////////////////////////////////////////////////// -// AvePooling + in-place layers -//////////////////////////////////////////////////////////////////////////////// -static const int kNumChannels = 3; - -void testInPlaceActivation(LayerParams& lp, Backend backendId, Target targetId, double l1 = 0.0, double lInf = 0.0) -{ - EXPECT_FALSE(lp.name.empty()); - - LayerParams pool; - pool.set("pool", "ave"); - pool.set("kernel_w", 2); - pool.set("kernel_h", 2); - pool.set("stride_w", 2); - pool.set("stride_h", 2); - pool.type = "Pooling"; - pool.name = "ave_pool"; - - Net net; - int poolId = net.addLayer(pool.name, pool.type, pool); - net.connect(0, 0, poolId, 0); - net.addLayerToPrev(lp.name, lp.type, lp); - - int sz[] = {1, kNumChannels, 10, 10}; - Mat input(4, &sz[0], CV_32F); - test(input, net, backendId, targetId, false, true, l1, lInf); -} - -typedef TestWithParam > > BatchNorm; -TEST_P(BatchNorm, Accuracy) -{ - bool hasWeights = get<0>(GetParam()); - bool hasBias = get<1>(GetParam()); - float epsilon = get<2>(GetParam()); - Backend backendId = get<0>(get<3>(GetParam())); - Target targetId = get<1>(get<3>(GetParam())); - - LayerParams lp; - lp.set("has_weight", hasWeights); - lp.set("has_bias", hasBias); - lp.set("eps", epsilon); - lp.type = "BatchNorm"; - lp.name = "testLayer"; - - lp.blobs.reserve(4); - for (int i = 0; i < 3; ++i) - lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); - if (hasBias || hasWeights) - lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); - - for (int i = 0; i < lp.blobs.size(); ++i) - randu(lp.blobs[i], 0.0f, 1.0f); - - testInPlaceActivation(lp, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, BatchNorm, Combine( -/*has weights*/ Bool(), -/*has bias*/ Bool(), -/*epsilon*/ Values(1e-3f, 1e-5f), - dnnBackendsAndTargetsWithHalide() -)); - -typedef TestWithParam > > ReLU; -TEST_P(ReLU, Accuracy) -{ - float negativeSlope = get<0>(GetParam()); - Backend backendId = get<0>(get<1>(GetParam())); - Target targetId = get<1>(get<1>(GetParam())); - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD && negativeSlope < 0) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - - LayerParams lp; - lp.set("negative_slope", negativeSlope); - lp.type = "ReLU"; - lp.name = "testLayer"; - testInPlaceActivation(lp, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, ReLU, Combine( -/*negative slope*/ Values(2.0f, 0.3f, -0.1f, 0.0f), - dnnBackendsAndTargetsWithHalide() -)); - -typedef TestWithParam > > NoParamActivation; -TEST_P(NoParamActivation, Accuracy) -{ - Backend backendId = get<0>(get<1>(GetParam())); - Target targetId = get<1>(get<1>(GetParam())); - std::string layer_type = get<0>(GetParam()); - - LayerParams lp; - lp.type = layer_type; - lp.name = "testLayer"; - testInPlaceActivation(lp, backendId, targetId); -} -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, NoParamActivation, Combine( -/*type*/ Values("TanH", "Sigmoid", "AbsVal", "BNLL", "Swish", "Mish"), - dnnBackendsAndTargetsWithHalide() -)); - -typedef TestWithParam > > Power; -TEST_P(Power, Accuracy) -{ - float power = get<0>(GetParam())[0]; - float scale = get<0>(GetParam())[1]; - float shift = get<0>(GetParam())[2]; - Backend backendId = get<0>(get<1>(GetParam())); - Target targetId = get<1>(get<1>(GetParam())); - - LayerParams lp; - lp.set("power", power); - lp.set("scale", scale); - lp.set("shift", shift); - lp.type = "Power"; - lp.name = "testLayer"; - testInPlaceActivation(lp, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Power, Combine( -/*power, scale, shift*/ Values(Vec3f(0.9f, 1.0f, 1.1f), Vec3f(0.9f, 1.1f, 1.0f), - Vec3f(1.0f, 0.9f, 1.1f), Vec3f(1.0f, 1.1f, 0.9f), - Vec3f(1.1f, 0.9f, 1.0f), Vec3f(1.1f, 1.0f, 0.9f)), - dnnBackendsAndTargetsWithHalide() -)); - -typedef TestWithParam > > Exp; -TEST_P(Exp, Accuracy) -{ - float base = get<0>(GetParam())[0]; - float scale = get<0>(GetParam())[1]; - float shift = get<0>(GetParam())[2]; - Backend backendId = get<0>(get<1>(GetParam())); - Target targetId = get<1>(get<1>(GetParam())); - - LayerParams lp; - lp.set("base", base); - lp.set("scale", scale); - lp.set("shift", shift); - lp.type = "Exp"; - lp.name = "testLayer"; - testInPlaceActivation(lp, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Exp, Combine( -/*base, scale, shift*/ Values(Vec3f(0.9f, -1.0f, 1.1f), Vec3f(0.9f, 1.1f, -1.0f), - Vec3f(-1.0f, 0.9f, 1.1f), Vec3f(-1.0f, 1.1f, 0.9f), - Vec3f(1.1f, 0.9f, -1.0f), Vec3f(1.1f, -1.0f, 0.9f)), - dnnBackendsAndTargetsWithHalide() -)); - -TEST_P(Test_Halide_layers, ChannelsPReLU) -{ - LayerParams lp; - lp.type = "ChannelsPReLU"; - lp.name = "testLayer"; - lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); - randu(lp.blobs[0], -1.0f, 1.0f); - - testInPlaceActivation(lp, backend, target); -} - -typedef TestWithParam > > Scale; -TEST_P(Scale, Accuracy) -{ - bool hasBias = get<0>(GetParam()); - Backend backendId = get<0>(get<1>(GetParam())); - Target targetId = get<1>(get<1>(GetParam())); - - LayerParams lp; - lp.set("bias_term", hasBias); - lp.type = "Scale"; - lp.name = "testLayer"; - lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); - randu(lp.blobs[0], -1.0f, 1.0f); - if (hasBias) - { - lp.blobs.push_back(Mat(1, kNumChannels, CV_32F)); - randu(lp.blobs[1], -1.0f, 1.0f); - } - testInPlaceActivation(lp, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Scale, Combine( - Bool(), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////////// -// Concat layer -//////////////////////////////////////////////////////////////////////////////// -// -// input --- conv --- concat --- output -// `--- conv ----^ ^ ^ -// `---- ... ------' ' -// `-----------------' -typedef TestWithParam > > Concat; -TEST_P(Concat, Accuracy) -{ - Vec3i inSize = get<0>(GetParam()); - Vec3i numChannels = get<1>(GetParam()); - Backend backendId = get<0>(get<2>(GetParam())); - Target targetId = get<1>(get<2>(GetParam())); - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD - && inSize == Vec3i(1, 4, 5) && numChannels == Vec3i(1, 6, 2) - ) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); // crash -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_CPU - && inSize == Vec3i(1, 4, 5) && numChannels == Vec3i(1, 6, 2) - ) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); // TODO: IE_CPU -#endif - - Net net; - - std::vector convLayerIds; - convLayerIds.reserve(numChannels.channels); - for (int i = 0, n = numChannels.channels; i < n; ++i) - { - if (!numChannels[i]) - break; - - int sz[] = {numChannels[i], inSize[0], 1, 1}; - Mat weights(4, &sz[0], CV_32F); - randu(weights, -1.0f, 1.0f); - - LayerParams convParam; - convParam.set("kernel_w", 1); - convParam.set("kernel_h", 1); - convParam.set("num_output", numChannels[i]); - convParam.set("bias_term", false); - convParam.type = "Convolution"; - std::ostringstream ss; - ss << "convLayer" << i; - convParam.name = ss.str(); - convParam.blobs.push_back(weights); - - int layerId = net.addLayer(convParam.name, convParam.type, convParam); - convLayerIds.push_back(layerId); - net.connect(0, 0, layerId, 0); - } - - LayerParams concatParam; - concatParam.type = "Concat"; - concatParam.name = "testLayer"; - int concatId = net.addLayer(concatParam.name, concatParam.type, concatParam); - net.connect(0, 0, concatId, 0); - for (int i = 0; i < convLayerIds.size(); ++i) - { - net.connect(convLayerIds[i], 0, concatId, i + 1); - } - - int sz[] = {1, inSize[0], inSize[1], inSize[2]}; - Mat input(4, &sz[0], CV_32F); - test(input, net, backendId, targetId); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Concat, Combine( -/*input size*/ Values(Vec3i(1, 4, 5), Vec3i(2, 8, 6)), -/*channels*/ Values(Vec3i(2, 0, 0), Vec3i(3, 4, 0), Vec3i(1, 6, 2)), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////////// -// Element-wise layers -//////////////////////////////////////////////////////////////////////////////// -// -// input --- conv --- eltwise --- output -// `--- conv ----^ ^ ^ -// `---- ... ------' ' -// `-----------------' -typedef TestWithParam > > Eltwise; -TEST_P(Eltwise, Accuracy) -{ - Vec3i inSize = get<0>(GetParam()); - std::string op = get<1>(GetParam()); - int numConv = get<2>(GetParam()); - bool weighted = get<3>(GetParam()); - Backend backendId = get<0>(get<4>(GetParam())); - Target targetId = get<1>(get<4>(GetParam())); - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) - // accuracy - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL && - inSize == Vec3i(1, 4, 5) && op == "sum" && numConv == 1 && !weighted) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && targetId == DNN_TARGET_OPENCL && - inSize == Vec3i(2, 8, 6) && op == "sum" && numConv == 1 && !weighted) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_MYRIAD && - inSize == Vec3i(1, 4, 5)) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && numConv > 1) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && targetId == DNN_TARGET_OPENCL && - op == "sum" && numConv == 1 && !weighted) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); -#endif - -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000) - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && numConv > 1) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); -#endif - - bool convInputShift = 1; - int numEltwiseInputs = numConv; - if (op == "div") - { - numConv = 1; - convInputShift = 0; // first input is convolution - } - - Net net; - - std::vector convLayerIds(numConv); - for (int i = 0; i < numConv; ++i) - { - int sz[] = {inSize[0], inSize[0], 1, 1}; - Mat weights(4, &sz[0], CV_32F); - randu(weights, -1.0f, 1.0f); - - LayerParams convParam; - convParam.set("kernel_w", 1); - convParam.set("kernel_h", 1); - convParam.set("num_output", inSize[0]); - convParam.set("bias_term", false); - convParam.type = "Convolution"; - std::ostringstream ss; - ss << "convLayer" << i; - convParam.name = ss.str(); - convParam.blobs.push_back(weights); - - convLayerIds[i] = net.addLayer(convParam.name, convParam.type, convParam); - net.connect(0, 0, convLayerIds[i], 0); - } - - LayerParams eltwiseParam; - eltwiseParam.set("operation", op); - if (op == "sum" && weighted) - { - RNG& rng = cv::theRNG(); - std::vector coeff(1 + numConv); - for (int i = 0; i < coeff.size(); ++i) - { - coeff[i] = rng.uniform(-2.0f, 2.0f); - } - eltwiseParam.set("coeff", DictValue::arrayReal(&coeff[0], coeff.size())); - } - eltwiseParam.type = "Eltwise"; - eltwiseParam.name = "testLayer"; - int eltwiseId = net.addLayer(eltwiseParam.name, eltwiseParam.type, eltwiseParam); - if (convInputShift == 1) - net.connect(0, 0, eltwiseId, 0); - for (int i = 0; i < numConv; ++i) - { - net.connect(convLayerIds[i], 0, eltwiseId, i + convInputShift); - } - if (convInputShift == 0) - net.connect(0, 0, eltwiseId, numConv); - for (int i = numConv; i < numEltwiseInputs; ++i) - { - net.connect(0, 0, eltwiseId, i + 1); - } - - int sz[] = {1, inSize[0], inSize[1], inSize[2]}; - Mat input(4, &sz[0], CV_32F); - if (op == "div") - randu(input, 1.0f, 1.0f); // ensure no divisor value has absouluate value of less than 0.5 - test(input, net, backendId, targetId, /*skipCheck*/false, (op == "div") ? false : true); -} - -INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Eltwise, Combine( -/*input size*/ Values(Vec3i(1, 4, 5), Vec3i(2, 8, 6)), -/*operation*/ Values("prod", "sum", "div", "max", "min"), -/*num convs*/ Values(1, 2, 3), -/*weighted(for sum only)*/ Bool(), - dnnBackendsAndTargetsWithHalide() -)); - -//////////////////////////////////////////////////////////////////////////// -// Mixed backends -//////////////////////////////////////////////////////////////////////////// -#ifdef HAVE_HALIDE -TEST(MixedBackends_Halide_Default_Halide, Accuracy) -{ - // Just a layer that supports Halide backend. - LayerParams lrn; - lrn.type = "LRN"; - lrn.name = "testLRN"; - - // Some of layers that doesn't supports Halide backend yet. - LayerParams mvn; - mvn.type = "MVN"; - mvn.name = "testMVN"; - - // Halide layer again. - LayerParams lrn2; - lrn2.type = "LRN"; - lrn2.name = "testLRN2"; - - Net net; - int lrnId = net.addLayer(lrn.name, lrn.type, lrn); - net.connect(0, 0, lrnId, 0); - net.addLayerToPrev(mvn.name, mvn.type, mvn); - net.addLayerToPrev(lrn2.name, lrn2.type, lrn2); - - int sz[] = {4, 3, 5, 6}; - Mat input(4, &sz[0], CV_32F); - randu(input, -1.0f, 1.0f); - net.setInput(input); - net.setPreferableBackend(DNN_BACKEND_OPENCV); - Mat outputDefault = net.forward().clone(); - - net.setPreferableBackend(DNN_BACKEND_HALIDE); - net.setInput(input); - Mat outputHalide = net.forward().clone(); - normAssert(outputDefault, outputHalide); - - net.setPreferableTarget(DNN_TARGET_OPENCL); - net.setInput(input); - outputHalide = net.forward().clone(); - normAssert(outputDefault, outputHalide); -} -#endif // HAVE_HALIDE - -INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_Halide_layers, dnnBackendsAndTargetsWithHalide()); - -}} // namespace diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp index bfa9dbe03b..97fb456ddc 100644 --- a/modules/dnn/test/test_int8_layers.cpp +++ b/modules/dnn/test/test_int8_layers.cpp @@ -370,7 +370,7 @@ TEST_P(Test_Int8_layers, InnerProduct) testLayer("matmul_layout", "TensorFlow", 0.035, 0.06); testLayer("tf2_dense", "TensorFlow", 0, 0); testLayer("matmul_add", "ONNX", 0.041, 0.082); - testLayer("linear", "ONNX", 0.0018, 0.0029); + testLayer("linear", "ONNX", 0.0027, 0.0046); if (backend == DNN_BACKEND_TIMVX) testLayer("constant", "ONNX", 0.00048, 0.0013); @@ -388,7 +388,7 @@ TEST_P(Test_Int8_layers, InnerProduct) testLayer("matmul_layout", "TensorFlow", 0.035, 0.095, 1, 1, false, true, false, false); testLayer("tf2_dense", "TensorFlow", 0, 0, 1, 1, false, true, false, false); testLayer("matmul_add", "ONNX", 0.041, 0.082, 1, 1, false, true, false, false); - testLayer("linear", "ONNX", 0.0022, 0.004, 1, 1, false, true, false, false); + testLayer("linear", "ONNX", 0.0027, 0.005, 1, 1, false, true, false, false); testLayer("constant", "ONNX", 0.00038, 0.0012, 1, 1, false, true, false, false); testLayer("lin_with_constant", "ONNX", 0.0011, 0.0016, 1, 1, false, true, false, false); } @@ -841,7 +841,7 @@ TEST_P(Test_Int8_nets, RCNN_ILSVRC13) if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); - float l1 = 0.02, lInf = 0.042; + float l1 = 0.02, lInf = 0.047; testONNXNet("rcnn_ilsvrc13", l1, lInf); } diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 0630833b1f..d9d3285b32 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -111,10 +111,7 @@ "test_dynamicquantizelinear_min_adjusted_expanded", "test_edge_pad", "test_einsum_batch_diagonal", -"test_einsum_batch_matmul", "test_einsum_inner_prod", -"test_einsum_sum", -"test_einsum_transpose", "test_equal", "test_equal_bcast", "test_expand_dim_changed", diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 0b734ae985..6ce46dc6e9 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -249,6 +249,10 @@ TEST_P(Test_ONNX_layers, GatherMulti) testONNXModels("gather_multi", npy, 0, 0, false, false); } +TEST_P(Test_ONNX_layers, Gather_shared_indices) { + testONNXModels("gather_shared_indices", npy, 0, 0, false, false, 1); +} + TEST_P(Test_ONNX_layers, Convolution3D) { if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) @@ -993,9 +997,21 @@ TEST_P(Test_ONNX_layers, MatMulAdd) TEST_P(Test_ONNX_layers, Expand) { testONNXModels("expand"); +} + +TEST_P(Test_ONNX_layers, ExpandIdentity) { testONNXModels("expand_identity"); +} + +TEST_P(Test_ONNX_layers, ExpandBatch) { testONNXModels("expand_batch"); +} + +TEST_P(Test_ONNX_layers, ExpandChannels) { testONNXModels("expand_channels"); +} + +TEST_P(Test_ONNX_layers, ExpandNegBatch) { testONNXModels("expand_neg_batch"); } @@ -1421,6 +1437,56 @@ TEST_P(Test_ONNX_layers, LSTM_layout_batch) testONNXModels("lstm_layout_1", npy, 0.005, 0.005, false, false, 3); } +TEST_P(Test_ONNX_layers, DISABLED_Einsum_1D) +{ + testONNXModels("einsum_1d", npy, 0, 0, false, false, 2); +} + +TEST_P(Test_ONNX_layers, Einsum_2D) +{ + testONNXModels("einsum_2d", npy, 0, 0, false, false, 2); +} + +TEST_P(Test_ONNX_layers, Einsum_3D) +{ + testONNXModels("einsum_3d", npy, 0, 0, false, false, 2); +} + +TEST_P(Test_ONNX_layers, Einsum_4D) +{ + testONNXModels("einsum_4d", npy, 0, 0, false, false, 2); +} + +TEST_P(Test_ONNX_layers, Einsum_5D) +{ + testONNXModels("einsum_5d", npy, 0, 0, false, false, 2); +} + +TEST_P(Test_ONNX_layers, DISABLED_Einsum_InnerProduct) +{ + testONNXModels("einsum_inner", npy, 0, 0, false, false, 2); +} + +TEST_P(Test_ONNX_layers, DISABLED_Einsum_HadamardProduct) +{ + testONNXModels("einsum_hadamard", npy, 0, 0, false, false, 2); +} + +TEST_P(Test_ONNX_layers, DISABLED_Einsum_Batch_Diagonal) +{ + testONNXModels("einsum_batch_diagonal", npy, 0, 0, false, false, 1); +} + +TEST_P(Test_ONNX_layers, Einsum_Sum) +{ + testONNXModels("einsum_sum", npy, 0, 0, false, false, 1); +} + +TEST_P(Test_ONNX_layers, Einsum_transpose) +{ + testONNXModels("einsum_transpose", npy, 0, 0, false, false, 1); +} + TEST_P(Test_ONNX_layers, Pad2d_Unfused) { testONNXModels("ReflectionPad2d"); @@ -2603,6 +2669,61 @@ TEST_P(Test_ONNX_layers, where_node) testONNXModels("where_layer"); } +TEST_P(Test_ONNX_layers, Conformance_Gemm_all_attributes) { + testONNXModels("test_gemm_all_attributes", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_alpha) { + testONNXModels("test_gemm_alpha", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_beta) { + testONNXModels("test_gemm_beta", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_default_matrix_bias) { + testONNXModels("test_gemm_default_matrix_bias", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_default_no_bias) { + testONNXModels("test_gemm_default_no_bias", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_default_scalar_bias) { + testONNXModels("test_gemm_default_scalar_bias", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_default_single_elem_vector_bias) { + testONNXModels("test_gemm_default_single_elem_vector_bias", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_default_vector_bias) { + testONNXModels("test_gemm_default_vector_bias", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_default_zero_bias) { + testONNXModels("test_gemm_default_zero_bias", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_transposeA) { + testONNXModels("test_gemm_transposeA", pb, 0, 0, false, true, 2); +} +TEST_P(Test_ONNX_layers, Conformance_Gemm_transposeB) { + testONNXModels("test_gemm_transposeB", pb, 0, 0, false, true, 2); +} + +// Note: These tests are converted from onnx/onnx so that they have constant shape as input. +// TODO: They can be moved into conformance tests once dynamic input is properly supported. +TEST_P(Test_ONNX_layers, Expand_dim_changed) { + testONNXModels("test_expand_dim_changed", pb, 0, 0, false, true, 1); +} +TEST_P(Test_ONNX_layers, Expand_dim_unchanged) { + testONNXModels("test_expand_dim_unchanged", pb, 0, 0, false, true, 1); +} +TEST_P(Test_ONNX_layers, Expand_shape_model1) { + testONNXModels("test_expand_shape_model1", pb, 0, 0, false, true, 1); +} +TEST_P(Test_ONNX_layers, Expand_shape_model2) { + testONNXModels("test_expand_shape_model2", pb, 0, 0, false, true, 1); +} +TEST_P(Test_ONNX_layers, Expand_shape_model3) { + testONNXModels("test_expand_shape_model3", pb, 0, 0, false, true, 1); +} +TEST_P(Test_ONNX_layers, Expand_shape_model4) { + testONNXModels("test_expand_shape_model4", pb, 0, 0, false, true, 1); +} + INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_nets, dnnBackendsAndTargets()); }} // namespace diff --git a/modules/features2d/src/sift.simd.hpp b/modules/features2d/src/sift.simd.hpp index 8589a0225c..2c5cf9f997 100644 --- a/modules/features2d/src/sift.simd.hpp +++ b/modules/features2d/src/sift.simd.hpp @@ -210,24 +210,24 @@ float calcOrientationHist( cv::hal::magnitude32f(X, Y, Mag, len); k = 0; -#if CV_SIMD - const int vecsize = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vecsize = VTraits::vlanes(); v_float32 nd360 = vx_setall_f32(n/360.f); v_int32 __n = vx_setall_s32(n); - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) bin_buf[vecsize]; - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) w_mul_mag_buf[vecsize]; + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) bin_buf[VTraits::max_nlanes]; + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) w_mul_mag_buf[VTraits::max_nlanes]; for( ; k <= len - vecsize; k += vecsize ) { v_float32 w = vx_load_aligned( W + k ); v_float32 mag = vx_load_aligned( Mag + k ); v_float32 ori = vx_load_aligned( Ori + k ); - v_int32 bin = v_round( nd360 * ori ); + v_int32 bin = v_round( v_mul(nd360, ori) ); - bin = v_select(bin >= __n, bin - __n, bin); - bin = v_select(bin < vx_setzero_s32(), bin + __n, bin); + bin = v_select(v_ge(bin, __n), v_sub(bin, __n), bin); + bin = v_select(v_lt(bin, vx_setzero_s32()), v_add(bin, __n), bin); - w = w * mag; + w = v_mul(w, mag); v_store_aligned(bin_buf, bin); v_store_aligned(w_mul_mag_buf, w); for(int vi = 0; vi < vecsize; vi++) @@ -253,19 +253,19 @@ float calcOrientationHist( temphist[n+1] = temphist[1]; i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 d_1_16 = vx_setall_f32(1.f/16.f); v_float32 d_4_16 = vx_setall_f32(4.f/16.f); v_float32 d_6_16 = vx_setall_f32(6.f/16.f); - for( ; i <= n - v_float32::nlanes; i += v_float32::nlanes ) + for( ; i <= n - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 tn2 = vx_load_aligned(temphist + i-2); v_float32 tn1 = vx_load(temphist + i-1); v_float32 t0 = vx_load(temphist + i); v_float32 t1 = vx_load(temphist + i+1); v_float32 t2 = vx_load(temphist + i+2); - v_float32 _hist = v_fma(tn2 + t2, d_1_16, - v_fma(tn1 + t1, d_4_16, t0 * d_6_16)); + v_float32 _hist = v_fma(v_add(tn2, t2), d_1_16, + v_fma(v_add(tn1, t1), d_4_16, v_mul(t0, d_6_16))); v_store(hist + i, _hist); } #endif @@ -452,8 +452,8 @@ public: const sift_wt* nextptr = next.ptr(r); int c = SIFT_IMG_BORDER; -#if CV_SIMD && !(DoG_TYPE_SHORT) - const int vecsize = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) && !(DoG_TYPE_SHORT) + const int vecsize = VTraits::vlanes(); for( ; c <= cols-SIFT_IMG_BORDER - vecsize; c += vecsize) { v_float32 val = vx_load(&currptr[c]); @@ -464,7 +464,7 @@ public: v_float32 vmin,vmax; - v_float32 cond = v_abs(val) > vx_setall_f32((float)threshold); + v_float32 cond = v_gt(v_abs(val), vx_setall_f32((float)this->threshold)); if (!v_check_any(cond)) { continue; @@ -477,10 +477,10 @@ public: vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); - v_float32 condp = cond & (val > vx_setall_f32(0)) & (val >= vmax); - v_float32 condm = cond & (val < vx_setall_f32(0)) & (val <= vmin); + v_float32 condp = v_and(v_and(cond, v_gt(val, vx_setall_f32(0))), v_ge(val, vmax)); + v_float32 condm = v_and(v_and(cond, v_lt(val, vx_setall_f32(0))), v_le(val, vmin)); - cond = condp | condm; + cond = v_or(condp, condm); if (!v_check_any(cond)) { continue; @@ -493,10 +493,10 @@ public: vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); - condp &= (val >= vmax); - condm &= (val <= vmin); + condp = v_and(condp, v_ge(val, vmax)); + condm = v_and(condm, v_le(val, vmin)); - cond = condp | condm; + cond = v_or(condp, condm); if (!v_check_any(cond)) { continue; @@ -515,10 +515,10 @@ public: vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); - condp &= (val >= v_max(vmax,max_middle)); - condm &= (val <= v_min(vmin,min_middle)); + condp = v_and(condp, v_ge(val, v_max(vmax, max_middle))); + condm = v_and(condm, v_le(val, v_min(vmin, min_middle))); - cond = condp | condm; + cond = v_or(condp, condm); if (!v_check_any(cond)) { continue; @@ -777,11 +777,11 @@ void calcSIFTDescriptor( cv::hal::exp32f(W, W, len); k = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { - const int vecsize = v_float32::nlanes; - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) idx_buf[vecsize]; - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rco_buf[8*vecsize]; + const int vecsize = VTraits::vlanes(); + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) idx_buf[VTraits::max_nlanes]; + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rco_buf[8*VTraits::max_nlanes]; const v_float32 __ori = vx_setall_f32(ori); const v_float32 __bins_per_rad = vx_setall_f32(bins_per_rad); const v_int32 __n = vx_setall_s32(n); @@ -792,28 +792,28 @@ void calcSIFTDescriptor( { v_float32 rbin = vx_load_aligned(RBin + k); v_float32 cbin = vx_load_aligned(CBin + k); - v_float32 obin = (vx_load_aligned(Ori + k) - __ori) * __bins_per_rad; - v_float32 mag = vx_load_aligned(Mag + k) * vx_load_aligned(W + k); + v_float32 obin = v_mul(v_sub(vx_load_aligned(Ori + k), __ori), __bins_per_rad); + v_float32 mag = v_mul(vx_load_aligned(Mag + k), vx_load_aligned(W + k)); v_int32 r0 = v_floor(rbin); v_int32 c0 = v_floor(cbin); v_int32 o0 = v_floor(obin); - rbin -= v_cvt_f32(r0); - cbin -= v_cvt_f32(c0); - obin -= v_cvt_f32(o0); + rbin = v_sub(rbin, v_cvt_f32(r0)); + cbin = v_sub(cbin, v_cvt_f32(c0)); + obin = v_sub(obin, v_cvt_f32(o0)); - o0 = v_select(o0 < vx_setzero_s32(), o0 + __n, o0); - o0 = v_select(o0 >= __n, o0 - __n, o0); + o0 = v_select(v_lt(o0, vx_setzero_s32()), v_add(o0, __n), o0); + o0 = v_select(v_ge(o0, __n), v_sub(o0, __n), o0); - v_float32 v_r1 = mag*rbin, v_r0 = mag - v_r1; - v_float32 v_rc11 = v_r1*cbin, v_rc10 = v_r1 - v_rc11; - v_float32 v_rc01 = v_r0*cbin, v_rc00 = v_r0 - v_rc01; - v_float32 v_rco111 = v_rc11*obin, v_rco110 = v_rc11 - v_rco111; - v_float32 v_rco101 = v_rc10*obin, v_rco100 = v_rc10 - v_rco101; - v_float32 v_rco011 = v_rc01*obin, v_rco010 = v_rc01 - v_rco011; - v_float32 v_rco001 = v_rc00*obin, v_rco000 = v_rc00 - v_rco001; + v_float32 v_r1 = v_mul(mag, rbin), v_r0 = v_sub(mag, v_r1); + v_float32 v_rc11 = v_mul(v_r1, cbin), v_rc10 = v_sub(v_r1, v_rc11); + v_float32 v_rc01 = v_mul(v_r0, cbin), v_rc00 = v_sub(v_r0, v_rc01); + v_float32 v_rco111 = v_mul(v_rc11, obin), v_rco110 = v_sub(v_rc11, v_rco111); + v_float32 v_rco101 = v_mul(v_rc10, obin), v_rco100 = v_sub(v_rc10, v_rco101); + v_float32 v_rco011 = v_mul(v_rc01, obin), v_rco010 = v_sub(v_rc01, v_rco011); + v_float32 v_rco001 = v_mul(v_rc00, obin), v_rco000 = v_sub(v_rc00, v_rco001); - v_int32 idx = v_muladd(v_muladd(r0+__1, __d_plus_2, c0+__1), __n_plus_2, o0); + v_int32 idx = v_muladd(v_muladd(v_add(r0, __1), __d_plus_2, v_add(c0, __1)), __n_plus_2, o0); v_store_aligned(idx_buf, idx); v_store_aligned(rco_buf, v_rco000); @@ -894,11 +894,11 @@ void calcSIFTDescriptor( float nrm2 = 0; len = d*d*n; k = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { v_float32 __nrm2 = vx_setzero_f32(); v_float32 __rawDst; - for( ; k <= len - v_float32::nlanes; k += v_float32::nlanes ) + for( ; k <= len - VTraits::vlanes(); k += VTraits::vlanes() ) { __rawDst = vx_load_aligned(rawDst + k); __nrm2 = v_fma(__rawDst, __rawDst, __nrm2); @@ -949,15 +949,15 @@ void calcSIFTDescriptor( if( dstMat.type() == CV_32F ) { float* dst = dstMat.ptr(row); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 __dst; v_float32 __min = vx_setzero_f32(); v_float32 __max = vx_setall_f32(255.0f); // max of uchar v_float32 __nrm2 = vx_setall_f32(nrm2); - for( k = 0; k <= len - v_float32::nlanes; k += v_float32::nlanes ) + for( k = 0; k <= len - VTraits::vlanes(); k += VTraits::vlanes() ) { __dst = vx_load_aligned(rawDst + k); - __dst = v_min(v_max(v_cvt_f32(v_round(__dst * __nrm2)), __min), __max); + __dst = v_min(v_max(v_cvt_f32(v_round(v_mul(__dst, __nrm2))), __min), __max); v_store(dst + k, __dst); } #endif @@ -976,16 +976,16 @@ if( dstMat.type() == CV_32F ) else // CV_8U { uint8_t* dst = dstMat.ptr(row); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 __dst0, __dst1; v_uint16 __pack01; v_float32 __nrm2 = vx_setall_f32(nrm2); - for( k = 0; k <= len - v_float32::nlanes * 2; k += v_float32::nlanes * 2 ) + for( k = 0; k <= len - VTraits::vlanes() * 2; k += VTraits::vlanes() * 2 ) { __dst0 = vx_load_aligned(rawDst + k); - __dst1 = vx_load_aligned(rawDst + k + v_float32::nlanes); + __dst1 = vx_load_aligned(rawDst + k + VTraits::vlanes()); - __pack01 = v_pack_u(v_round(__dst0 * __nrm2), v_round(__dst1 * __nrm2)); + __pack01 = v_pack_u(v_round(v_mul(__dst0, __nrm2)), v_round(v_mul(__dst1, __nrm2))); v_pack_store(dst + k, __pack01); } #endif diff --git a/modules/imgproc/src/accum.simd.hpp b/modules/imgproc/src/accum.simd.hpp index 6b0e6d6fbe..7fe7aabeaf 100644 --- a/modules/imgproc/src/accum.simd.hpp +++ b/modules/imgproc/src/accum.simd.hpp @@ -139,7 +139,7 @@ void acc_general_(const T* src, AT* dst, const uchar* mask, int len, int cn, int } #if CV_AVX && !CV_AVX2 _mm256_zeroupper(); -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -187,7 +187,7 @@ accSqr_general_( const T* src, AT* dst, const uchar* mask, int len, int cn, int } #if CV_AVX && !CV_AVX2 _mm256_zeroupper(); -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -236,7 +236,7 @@ accProd_general_( const T* src1, const T* src2, AT* dst, const uchar* mask, int } #if CV_AVX && !CV_AVX2 _mm256_zeroupper(); -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -285,16 +285,16 @@ accW_general_( const T* src, AT* dst, const uchar* mask, int len, int cn, double } #if CV_AVX && !CV_AVX2 _mm256_zeroupper(); -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint8::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -309,10 +309,10 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else @@ -323,9 +323,9 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint16 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); @@ -333,10 +333,10 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else if (cn == 3) @@ -344,12 +344,12 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src0, v_src1, v_src2; v_load_deinterleave(src + (x * cn), v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint16 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); @@ -373,18 +373,18 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) v_load_deinterleave(dst + ((x + step * 2) * cn), v_dst010, v_dst110, v_dst210); v_load_deinterleave(dst + ((x + step * 3) * cn), v_dst011, v_dst111, v_dst211); - v_dst000 += v_cvt_f32(v_reinterpret_as_s32(v_src000)); - v_dst100 += v_cvt_f32(v_reinterpret_as_s32(v_src100)); - v_dst200 += v_cvt_f32(v_reinterpret_as_s32(v_src200)); - v_dst001 += v_cvt_f32(v_reinterpret_as_s32(v_src001)); - v_dst101 += v_cvt_f32(v_reinterpret_as_s32(v_src101)); - v_dst201 += v_cvt_f32(v_reinterpret_as_s32(v_src201)); - v_dst010 += v_cvt_f32(v_reinterpret_as_s32(v_src010)); - v_dst110 += v_cvt_f32(v_reinterpret_as_s32(v_src110)); - v_dst210 += v_cvt_f32(v_reinterpret_as_s32(v_src210)); - v_dst011 += v_cvt_f32(v_reinterpret_as_s32(v_src011)); - v_dst111 += v_cvt_f32(v_reinterpret_as_s32(v_src111)); - v_dst211 += v_cvt_f32(v_reinterpret_as_s32(v_src211)); + v_dst000 = v_add(v_dst000, v_cvt_f32(v_reinterpret_as_s32(v_src000))); + v_dst100 = v_add(v_dst100, v_cvt_f32(v_reinterpret_as_s32(v_src100))); + v_dst200 = v_add(v_dst200, v_cvt_f32(v_reinterpret_as_s32(v_src200))); + v_dst001 = v_add(v_dst001, v_cvt_f32(v_reinterpret_as_s32(v_src001))); + v_dst101 = v_add(v_dst101, v_cvt_f32(v_reinterpret_as_s32(v_src101))); + v_dst201 = v_add(v_dst201, v_cvt_f32(v_reinterpret_as_s32(v_src201))); + v_dst010 = v_add(v_dst010, v_cvt_f32(v_reinterpret_as_s32(v_src010))); + v_dst110 = v_add(v_dst110, v_cvt_f32(v_reinterpret_as_s32(v_src110))); + v_dst210 = v_add(v_dst210, v_cvt_f32(v_reinterpret_as_s32(v_src210))); + v_dst011 = v_add(v_dst011, v_cvt_f32(v_reinterpret_as_s32(v_src011))); + v_dst111 = v_add(v_dst111, v_cvt_f32(v_reinterpret_as_s32(v_src111))); + v_dst211 = v_add(v_dst211, v_cvt_f32(v_reinterpret_as_s32(v_src211))); v_store_interleave(dst + (x * cn), v_dst000, v_dst100, v_dst200); v_store_interleave(dst + ((x + step) * cn), v_dst001, v_dst101, v_dst201); @@ -400,9 +400,9 @@ void acc_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -413,8 +413,8 @@ void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn v_uint32 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src0))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src1))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src0)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src1)))); } } else @@ -425,14 +425,14 @@ void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint32 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src0))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src1))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src0)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src1)))); } } else if (cn == 3) @@ -441,12 +441,12 @@ void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint32 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); @@ -456,12 +456,12 @@ void acc_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn v_load_deinterleave(dst + x * cn, v_dst00, v_dst10, v_dst20); v_load_deinterleave(dst + (x + step) * cn, v_dst01, v_dst11, v_dst21); - v_dst00 += v_cvt_f32(v_reinterpret_as_s32(v_src00)); - v_dst01 += v_cvt_f32(v_reinterpret_as_s32(v_src01)); - v_dst10 += v_cvt_f32(v_reinterpret_as_s32(v_src10)); - v_dst11 += v_cvt_f32(v_reinterpret_as_s32(v_src11)); - v_dst20 += v_cvt_f32(v_reinterpret_as_s32(v_src20)); - v_dst21 += v_cvt_f32(v_reinterpret_as_s32(v_src21)); + v_dst00 = v_add(v_dst00, v_cvt_f32(v_reinterpret_as_s32(v_src00))); + v_dst01 = v_add(v_dst01, v_cvt_f32(v_reinterpret_as_s32(v_src01))); + v_dst10 = v_add(v_dst10, v_cvt_f32(v_reinterpret_as_s32(v_src10))); + v_dst11 = v_add(v_dst11, v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_dst20 = v_add(v_dst20, v_cvt_f32(v_reinterpret_as_s32(v_src20))); + v_dst21 = v_add(v_dst21, v_cvt_f32(v_reinterpret_as_s32(v_src21))); v_store_interleave(dst + x * cn, v_dst00, v_dst10, v_dst20); v_store_interleave(dst + (x + step) * cn, v_dst01, v_dst11, v_dst21); @@ -551,9 +551,9 @@ void acc_simd_(const float* src, float* dst, const uchar* mask, int len, int cn) void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint8::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -586,14 +586,14 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn v_float64 v_dst6 = vx_load(dst + x + step * 6); v_float64 v_dst7 = vx_load(dst + x + step * 7); - v_dst0 = v_dst0 + v_src0; - v_dst1 = v_dst1 + v_src1; - v_dst2 = v_dst2 + v_src2; - v_dst3 = v_dst3 + v_src3; - v_dst4 = v_dst4 + v_src4; - v_dst5 = v_dst5 + v_src5; - v_dst6 = v_dst6 + v_src6; - v_dst7 = v_dst7 + v_src7; + v_dst0 = v_add(v_dst0, v_src0); + v_dst1 = v_add(v_dst1, v_src1); + v_dst2 = v_add(v_dst2, v_src2); + v_dst3 = v_add(v_dst3, v_src3); + v_dst4 = v_add(v_dst4, v_src4); + v_dst5 = v_add(v_dst5, v_src5); + v_dst6 = v_add(v_dst6, v_src6); + v_dst7 = v_add(v_dst7, v_src7); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -613,9 +613,9 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint8 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint16 v_int0, v_int1; v_expand(v_src, v_int0, v_int1); @@ -641,14 +641,14 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn v_float64 v_dst6 = vx_load(dst + x + step * 6); v_float64 v_dst7 = vx_load(dst + x + step * 7); - v_dst0 = v_dst0 + v_src0; - v_dst1 = v_dst1 + v_src1; - v_dst2 = v_dst2 + v_src2; - v_dst3 = v_dst3 + v_src3; - v_dst4 = v_dst4 + v_src4; - v_dst5 = v_dst5 + v_src5; - v_dst6 = v_dst6 + v_src6; - v_dst7 = v_dst7 + v_src7; + v_dst0 = v_add(v_dst0, v_src0); + v_dst1 = v_add(v_dst1, v_src1); + v_dst2 = v_add(v_dst2, v_src2); + v_dst3 = v_add(v_dst3, v_src3); + v_dst4 = v_add(v_dst4, v_src4); + v_dst5 = v_add(v_dst5, v_src5); + v_dst6 = v_add(v_dst6, v_src6); + v_dst7 = v_add(v_dst7, v_src7); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -665,12 +665,12 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src0, v_src1, v_src2; v_load_deinterleave(src + (x * cn), v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint16 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); @@ -726,14 +726,14 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn v_load_deinterleave(dst + ((x + step * 6) * cn), v_dst0110, v_dst1110, v_dst2110); v_load_deinterleave(dst + ((x + step * 7) * cn), v_dst0111, v_dst1111, v_dst2111); - v_store_interleave(dst + (x * cn), v_dst0000 + v_src0000, v_dst1000 + v_src1000, v_dst2000 + v_src2000); - v_store_interleave(dst + ((x + step) * cn), v_dst0001 + v_src0001, v_dst1001 + v_src1001, v_dst2001 + v_src2001); - v_store_interleave(dst + ((x + step * 2) * cn), v_dst0010 + v_src0010, v_dst1010 + v_src1010, v_dst2010 + v_src2010); - v_store_interleave(dst + ((x + step * 3) * cn), v_dst0011 + v_src0011, v_dst1011 + v_src1011, v_dst2011 + v_src2011); - v_store_interleave(dst + ((x + step * 4) * cn), v_dst0100 + v_src0100, v_dst1100 + v_src1100, v_dst2100 + v_src2100); - v_store_interleave(dst + ((x + step * 5) * cn), v_dst0101 + v_src0101, v_dst1101 + v_src1101, v_dst2101 + v_src2101); - v_store_interleave(dst + ((x + step * 6) * cn), v_dst0110 + v_src0110, v_dst1110 + v_src1110, v_dst2110 + v_src2110); - v_store_interleave(dst + ((x + step * 7) * cn), v_dst0111 + v_src0111, v_dst1111 + v_src1111, v_dst2111 + v_src2111); + v_store_interleave(dst + (x * cn), v_add(v_dst0000, v_src0000), v_add(v_dst1000, v_src1000), v_add(v_dst2000, v_src2000)); + v_store_interleave(dst + ((x + step) * cn), v_add(v_dst0001, v_src0001), v_add(v_dst1001, v_src1001), v_add(v_dst2001, v_src2001)); + v_store_interleave(dst + ((x + step * 2) * cn), v_add(v_dst0010, v_src0010), v_add(v_dst1010, v_src1010), v_add(v_dst2010, v_src2010)); + v_store_interleave(dst + ((x + step * 3) * cn), v_add(v_dst0011, v_src0011), v_add(v_dst1011, v_src1011), v_add(v_dst2011, v_src2011)); + v_store_interleave(dst + ((x + step * 4) * cn), v_add(v_dst0100, v_src0100), v_add(v_dst1100, v_src1100), v_add(v_dst2100, v_src2100)); + v_store_interleave(dst + ((x + step * 5) * cn), v_add(v_dst0101, v_src0101), v_add(v_dst1101, v_src1101), v_add(v_dst2101, v_src2101)); + v_store_interleave(dst + ((x + step * 6) * cn), v_add(v_dst0110, v_src0110), v_add(v_dst1110, v_src1110), v_add(v_dst2110, v_src2110)); + v_store_interleave(dst + ((x + step * 7) * cn), v_add(v_dst0111, v_src0111), v_add(v_dst1111, v_src1111), v_add(v_dst2111, v_src2111)); } } } @@ -744,9 +744,9 @@ void acc_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -767,10 +767,10 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c v_float64 v_dst2 = vx_load(dst + x + step * 2); v_float64 v_dst3 = vx_load(dst + x + step * 3); - v_dst0 = v_dst0 + v_src0; - v_dst1 = v_dst1 + v_src1; - v_dst2 = v_dst2 + v_src2; - v_dst3 = v_dst3 + v_src3; + v_dst0 = v_add(v_dst0, v_src0); + v_dst1 = v_add(v_dst1, v_src1); + v_dst2 = v_add(v_dst2, v_src2); + v_dst3 = v_add(v_dst3, v_src3); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -786,9 +786,9 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint32 v_int0, v_int1; v_expand(v_src, v_int0, v_int1); @@ -802,10 +802,10 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c v_float64 v_dst2 = vx_load(dst + x + step * 2); v_float64 v_dst3 = vx_load(dst + x + step * 3); - v_dst0 = v_dst0 + v_src0; - v_dst1 = v_dst1 + v_src1; - v_dst2 = v_dst2 + v_src2; - v_dst3 = v_dst3 + v_src3; + v_dst0 = v_add(v_dst0, v_src0); + v_dst1 = v_add(v_dst1, v_src1); + v_dst2 = v_add(v_dst2, v_src2); + v_dst3 = v_add(v_dst3, v_src3); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -818,12 +818,12 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c for ( ; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint32 v_int00, v_int01, v_int10, v_int11, v_int20, v_int21; v_expand(v_src0, v_int00, v_int01); v_expand(v_src1, v_int10, v_int11); @@ -848,10 +848,10 @@ void acc_simd_(const ushort* src, double* dst, const uchar* mask, int len, int c v_load_deinterleave(dst + (x + step * 2) * cn, v_dst02, v_dst12, v_dst22); v_load_deinterleave(dst + (x + step * 3) * cn, v_dst03, v_dst13, v_dst23); - v_store_interleave(dst + x * cn, v_dst00 + v_src00, v_dst10 + v_src10, v_dst20 + v_src20); - v_store_interleave(dst + (x + step) * cn, v_dst01 + v_src01, v_dst11 + v_src11, v_dst21 + v_src21); - v_store_interleave(dst + (x + step * 2) * cn, v_dst02 + v_src02, v_dst12 + v_src12, v_dst22 + v_src22); - v_store_interleave(dst + (x + step * 3) * cn, v_dst03 + v_src03, v_dst13 + v_src13, v_dst23 + v_src23); + v_store_interleave(dst + x * cn, v_add(v_dst00, v_src00), v_add(v_dst10, v_src10), v_add(v_dst20, v_src20)); + v_store_interleave(dst + (x + step) * cn, v_add(v_dst01, v_src01), v_add(v_dst11, v_src11), v_add(v_dst21, v_src21)); + v_store_interleave(dst + (x + step * 2) * cn, v_add(v_dst02, v_src02), v_add(v_dst12, v_src12), v_add(v_dst22, v_src22)); + v_store_interleave(dst + (x + step * 3) * cn, v_add(v_dst03, v_src03), v_add(v_dst13, v_src13), v_add(v_dst23, v_src23)); } } } @@ -1033,9 +1033,9 @@ void acc_simd_(const double* src, double* dst, const uchar* mask, int len, int c void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint8::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1052,10 +1052,10 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else @@ -1066,9 +1066,9 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int for ( ; x <= len - cVectorWidth ; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint16 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); v_src0 = v_mul_wrap(v_src0, v_src0); @@ -1078,10 +1078,10 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else if (cn == 3) @@ -1089,13 +1089,13 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int for ( ; x <= len - cVectorWidth ; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint8 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint16 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_expand(v_src0, v_src00, v_src01); @@ -1126,20 +1126,20 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int v_load_deinterleave(dst + (x + step * 2) * cn, v_dst010, v_dst110, v_dst210); v_load_deinterleave(dst + (x + step * 3) * cn, v_dst011, v_dst111, v_dst211); - v_dst000 += v_cvt_f32(v_reinterpret_as_s32(v_src000)); - v_dst001 += v_cvt_f32(v_reinterpret_as_s32(v_src001)); - v_dst010 += v_cvt_f32(v_reinterpret_as_s32(v_src010)); - v_dst011 += v_cvt_f32(v_reinterpret_as_s32(v_src011)); + v_dst000 = v_add(v_dst000, v_cvt_f32(v_reinterpret_as_s32(v_src000))); + v_dst001 = v_add(v_dst001, v_cvt_f32(v_reinterpret_as_s32(v_src001))); + v_dst010 = v_add(v_dst010, v_cvt_f32(v_reinterpret_as_s32(v_src010))); + v_dst011 = v_add(v_dst011, v_cvt_f32(v_reinterpret_as_s32(v_src011))); - v_dst100 += v_cvt_f32(v_reinterpret_as_s32(v_src100)); - v_dst101 += v_cvt_f32(v_reinterpret_as_s32(v_src101)); - v_dst110 += v_cvt_f32(v_reinterpret_as_s32(v_src110)); - v_dst111 += v_cvt_f32(v_reinterpret_as_s32(v_src111)); + v_dst100 = v_add(v_dst100, v_cvt_f32(v_reinterpret_as_s32(v_src100))); + v_dst101 = v_add(v_dst101, v_cvt_f32(v_reinterpret_as_s32(v_src101))); + v_dst110 = v_add(v_dst110, v_cvt_f32(v_reinterpret_as_s32(v_src110))); + v_dst111 = v_add(v_dst111, v_cvt_f32(v_reinterpret_as_s32(v_src111))); - v_dst200 += v_cvt_f32(v_reinterpret_as_s32(v_src200)); - v_dst201 += v_cvt_f32(v_reinterpret_as_s32(v_src201)); - v_dst210 += v_cvt_f32(v_reinterpret_as_s32(v_src210)); - v_dst211 += v_cvt_f32(v_reinterpret_as_s32(v_src211)); + v_dst200 = v_add(v_dst200, v_cvt_f32(v_reinterpret_as_s32(v_src200))); + v_dst201 = v_add(v_dst201, v_cvt_f32(v_reinterpret_as_s32(v_src201))); + v_dst210 = v_add(v_dst210, v_cvt_f32(v_reinterpret_as_s32(v_src210))); + v_dst211 = v_add(v_dst211, v_cvt_f32(v_reinterpret_as_s32(v_src211))); v_store_interleave(dst + x * cn, v_dst000, v_dst100, v_dst200); v_store_interleave(dst + (x + step) * cn, v_dst001, v_dst101, v_dst201); @@ -1155,9 +1155,9 @@ void accSqr_simd_(const uchar* src, float* dst, const uchar* mask, int len, int void accSqr_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1186,13 +1186,13 @@ void accSqr_simd_(const ushort* src, float* dst, const uchar* mask, int len, int v_uint16 v_mask16 = vx_load_expand(mask + x); v_uint32 v_mask0, v_mask1; v_expand(v_mask16, v_mask0, v_mask1); - v_mask0 = ~(v_mask0 == v_0); - v_mask1 = ~(v_mask1 == v_0); + v_mask0 = v_not(v_eq(v_mask0, v_0)); + v_mask1 = v_not(v_eq(v_mask1, v_0)); v_uint16 v_src = vx_load(src + x); v_uint32 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); - v_src0 = v_src0 & v_mask0; - v_src1 = v_src1 & v_mask1; + v_src0 = v_and(v_src0, v_mask0); + v_src1 = v_and(v_src1, v_mask1); v_float32 v_float0, v_float1; v_float0 = v_cvt_f32(v_reinterpret_as_s32(v_src0)); @@ -1209,8 +1209,8 @@ void accSqr_simd_(const ushort* src, float* dst, const uchar* mask, int len, int v_uint16 v_mask16 = vx_load_expand(mask + x); v_uint32 v_mask0, v_mask1; v_expand(v_mask16, v_mask0, v_mask1); - v_mask0 = ~(v_mask0 == v_0); - v_mask1 = ~(v_mask1 == v_0); + v_mask0 = v_not(v_eq(v_mask0, v_0)); + v_mask1 = v_not(v_eq(v_mask1, v_0)); v_uint16 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); @@ -1218,12 +1218,12 @@ void accSqr_simd_(const ushort* src, float* dst, const uchar* mask, int len, int v_expand(v_src0, v_int00, v_int01); v_expand(v_src1, v_int10, v_int11); v_expand(v_src2, v_int20, v_int21); - v_int00 = v_int00 & v_mask0; - v_int01 = v_int01 & v_mask1; - v_int10 = v_int10 & v_mask0; - v_int11 = v_int11 & v_mask1; - v_int20 = v_int20 & v_mask0; - v_int21 = v_int21 & v_mask1; + v_int00 = v_and(v_int00, v_mask0); + v_int01 = v_and(v_int01, v_mask1); + v_int10 = v_and(v_int10, v_mask0); + v_int11 = v_and(v_int11, v_mask1); + v_int20 = v_and(v_int20, v_mask0); + v_int21 = v_and(v_int21, v_mask1); v_float32 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_src00 = v_cvt_f32(v_reinterpret_as_s32(v_int00)); @@ -1347,9 +1347,9 @@ void accSqr_simd_(const float* src, float* dst, const uchar* mask, int len, int void accSqr_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1390,9 +1390,9 @@ void accSqr_simd_(const uchar* src, double* dst, const uchar* mask, int len, int for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src = vx_load_expand(src + x); - v_uint16 v_int = v_src & v_mask; + v_uint16 v_int = v_and(v_src, v_mask); v_uint32 v_int0, v_int1; v_expand(v_int, v_int0, v_int1); @@ -1430,10 +1430,10 @@ void accSqr_simd_(const uchar* src, double* dst, const uchar* mask, int len, int v_uint16 v_int2 = v_expand_low(v_src2); v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); - v_int0 = v_int0 & v_mask; - v_int1 = v_int1 & v_mask; - v_int2 = v_int2 & v_mask; + v_mask = v_not(v_eq(v_mask, v_0)); + v_int0 = v_and(v_int0, v_mask); + v_int1 = v_and(v_int1, v_mask); + v_int2 = v_and(v_int2, v_mask); v_uint32 v_int00, v_int01, v_int10, v_int11, v_int20, v_int21; v_expand(v_int0, v_int00, v_int01); @@ -1486,9 +1486,9 @@ void accSqr_simd_(const uchar* src, double* dst, const uchar* mask, int len, int void accSqr_simd_(const ushort* src, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1531,9 +1531,9 @@ void accSqr_simd_(const ushort* src, double* dst, const uchar* mask, int len, in for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src = vx_load(src + x); - v_src = v_src & v_mask; + v_src = v_and(v_src, v_mask); v_uint32 v_int_0, v_int_1; v_expand(v_src, v_int_0, v_int_1); @@ -1566,12 +1566,12 @@ void accSqr_simd_(const ushort* src, double* dst, const uchar* mask, int len, in for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_src0, v_src1, v_src2; v_load_deinterleave(src + x * cn, v_src0, v_src1, v_src2); - v_src0 = v_src0 & v_mask; - v_src1 = v_src1 & v_mask; - v_src2 = v_src2 & v_mask; + v_src0 = v_and(v_src0, v_mask); + v_src1 = v_and(v_src1, v_mask); + v_src2 = v_and(v_src2, v_mask); v_uint32 v_int00, v_int01, v_int10, v_int11, v_int20, v_int21; v_expand(v_src0, v_int00, v_int01); v_expand(v_src1, v_int10, v_int11); @@ -1810,9 +1810,9 @@ void accSqr_simd_(const double* src, double* dst, const uchar* mask, int len, in void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint8::nlanes; - const int step = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1829,10 +1829,10 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else @@ -1843,11 +1843,11 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint8 v_1src = vx_load(src1 + x); v_uint8 v_2src = vx_load(src2 + x); - v_1src = v_1src & v_mask; - v_2src = v_2src & v_mask; + v_1src = v_and(v_1src, v_mask); + v_2src = v_and(v_2src, v_mask); v_uint16 v_src0, v_src1; v_mul_expand(v_1src, v_2src, v_src0, v_src1); @@ -1856,10 +1856,10 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar v_expand(v_src0, v_src00, v_src01); v_expand(v_src1, v_src10, v_src11); - v_store(dst + x, vx_load(dst + x) + v_cvt_f32(v_reinterpret_as_s32(v_src00))); - v_store(dst + x + step, vx_load(dst + x + step) + v_cvt_f32(v_reinterpret_as_s32(v_src01))); - v_store(dst + x + step * 2, vx_load(dst + x + step * 2) + v_cvt_f32(v_reinterpret_as_s32(v_src10))); - v_store(dst + x + step * 3, vx_load(dst + x + step * 3) + v_cvt_f32(v_reinterpret_as_s32(v_src11))); + v_store(dst + x, v_add(vx_load(dst + x), v_cvt_f32(v_reinterpret_as_s32(v_src00)))); + v_store(dst + x + step, v_add(vx_load(dst + x + step), v_cvt_f32(v_reinterpret_as_s32(v_src01)))); + v_store(dst + x + step * 2, v_add(vx_load(dst + x + step * 2), v_cvt_f32(v_reinterpret_as_s32(v_src10)))); + v_store(dst + x + step * 3, v_add(vx_load(dst + x + step * 3), v_cvt_f32(v_reinterpret_as_s32(v_src11)))); } } else if (cn == 3) @@ -1867,16 +1867,16 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint8 v_mask = vx_load(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint8 v_1src0, v_1src1, v_1src2, v_2src0, v_2src1, v_2src2; v_load_deinterleave(src1 + x * cn, v_1src0, v_1src1, v_1src2); v_load_deinterleave(src2 + x * cn, v_2src0, v_2src1, v_2src2); - v_1src0 = v_1src0 & v_mask; - v_1src1 = v_1src1 & v_mask; - v_1src2 = v_1src2 & v_mask; - v_2src0 = v_2src0 & v_mask; - v_2src1 = v_2src1 & v_mask; - v_2src2 = v_2src2 & v_mask; + v_1src0 = v_and(v_1src0, v_mask); + v_1src1 = v_and(v_1src1, v_mask); + v_1src2 = v_and(v_1src2, v_mask); + v_2src0 = v_and(v_2src0, v_mask); + v_2src1 = v_and(v_2src1, v_mask); + v_2src2 = v_and(v_2src2, v_mask); v_uint16 v_src00, v_src01, v_src10, v_src11, v_src20, v_src21; v_mul_expand(v_1src0, v_2src0, v_src00, v_src01); @@ -1896,18 +1896,18 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar v_load_deinterleave(dst + (x + step) * cn, v_dst001, v_dst101, v_dst201); v_load_deinterleave(dst + (x + step * 2) * cn, v_dst002, v_dst102, v_dst202); v_load_deinterleave(dst + (x + step * 3) * cn, v_dst003, v_dst103, v_dst203); - v_dst000 = v_dst000 + v_cvt_f32(v_reinterpret_as_s32(v_src000)); - v_dst001 = v_dst001 + v_cvt_f32(v_reinterpret_as_s32(v_src001)); - v_dst002 = v_dst002 + v_cvt_f32(v_reinterpret_as_s32(v_src002)); - v_dst003 = v_dst003 + v_cvt_f32(v_reinterpret_as_s32(v_src003)); - v_dst100 = v_dst100 + v_cvt_f32(v_reinterpret_as_s32(v_src100)); - v_dst101 = v_dst101 + v_cvt_f32(v_reinterpret_as_s32(v_src101)); - v_dst102 = v_dst102 + v_cvt_f32(v_reinterpret_as_s32(v_src102)); - v_dst103 = v_dst103 + v_cvt_f32(v_reinterpret_as_s32(v_src103)); - v_dst200 = v_dst200 + v_cvt_f32(v_reinterpret_as_s32(v_src200)); - v_dst201 = v_dst201 + v_cvt_f32(v_reinterpret_as_s32(v_src201)); - v_dst202 = v_dst202 + v_cvt_f32(v_reinterpret_as_s32(v_src202)); - v_dst203 = v_dst203 + v_cvt_f32(v_reinterpret_as_s32(v_src203)); + v_dst000 = v_add(v_dst000, v_cvt_f32(v_reinterpret_as_s32(v_src000))); + v_dst001 = v_add(v_dst001, v_cvt_f32(v_reinterpret_as_s32(v_src001))); + v_dst002 = v_add(v_dst002, v_cvt_f32(v_reinterpret_as_s32(v_src002))); + v_dst003 = v_add(v_dst003, v_cvt_f32(v_reinterpret_as_s32(v_src003))); + v_dst100 = v_add(v_dst100, v_cvt_f32(v_reinterpret_as_s32(v_src100))); + v_dst101 = v_add(v_dst101, v_cvt_f32(v_reinterpret_as_s32(v_src101))); + v_dst102 = v_add(v_dst102, v_cvt_f32(v_reinterpret_as_s32(v_src102))); + v_dst103 = v_add(v_dst103, v_cvt_f32(v_reinterpret_as_s32(v_src103))); + v_dst200 = v_add(v_dst200, v_cvt_f32(v_reinterpret_as_s32(v_src200))); + v_dst201 = v_add(v_dst201, v_cvt_f32(v_reinterpret_as_s32(v_src201))); + v_dst202 = v_add(v_dst202, v_cvt_f32(v_reinterpret_as_s32(v_src202))); + v_dst203 = v_add(v_dst203, v_cvt_f32(v_reinterpret_as_s32(v_src203))); v_store_interleave(dst + x * cn, v_dst000, v_dst100, v_dst200); v_store_interleave(dst + (x + step) * cn, v_dst001, v_dst101, v_dst201); @@ -1923,9 +1923,9 @@ void accProd_simd_(const uchar* src1, const uchar* src2, float* dst, const uchar void accProd_simd_(const ushort* src1, const ushort* src2, float* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -1956,10 +1956,10 @@ void accProd_simd_(const ushort* src1, const ushort* src2, float* dst, const uch for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); - v_uint16 v_1src = vx_load(src1 + x) & v_mask; - v_uint16 v_2src = vx_load(src2 + x) & v_mask; + v_uint16 v_1src = v_and(vx_load(src1 + x), v_mask); + v_uint16 v_2src = v_and(vx_load(src2 + x), v_mask); v_uint32 v_1src0, v_1src1, v_2src0, v_2src1; v_expand(v_1src, v_1src0, v_1src1); @@ -1979,17 +1979,17 @@ void accProd_simd_(const ushort* src1, const ushort* src2, float* dst, const uch for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_0 == v_mask); + v_mask = v_not(v_eq(v_0, v_mask)); v_uint16 v_1src0, v_1src1, v_1src2, v_2src0, v_2src1, v_2src2; v_load_deinterleave(src1 + x * cn, v_1src0, v_1src1, v_1src2); v_load_deinterleave(src2 + x * cn, v_2src0, v_2src1, v_2src2); - v_1src0 = v_1src0 & v_mask; - v_1src1 = v_1src1 & v_mask; - v_1src2 = v_1src2 & v_mask; - v_2src0 = v_2src0 & v_mask; - v_2src1 = v_2src1 & v_mask; - v_2src2 = v_2src2 & v_mask; + v_1src0 = v_and(v_1src0, v_mask); + v_1src1 = v_and(v_1src1, v_mask); + v_1src2 = v_and(v_1src2, v_mask); + v_2src0 = v_and(v_2src0, v_mask); + v_2src1 = v_and(v_2src1, v_mask); + v_2src2 = v_and(v_2src2, v_mask); v_uint32 v_1src00, v_1src01, v_1src10, v_1src11, v_1src20, v_1src21, v_2src00, v_2src01, v_2src10, v_2src11, v_2src20, v_2src21; v_expand(v_1src0, v_1src00, v_1src01); @@ -2108,9 +2108,9 @@ void accProd_simd_(const float* src1, const float* src2, float* dst, const uchar void accProd_simd_(const uchar* src1, const uchar* src2, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2153,9 +2153,9 @@ void accProd_simd_(const uchar* src1, const uchar* src2, double* dst, const ucha for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); - v_uint16 v_1int = vx_load_expand(src1 + x) & v_mask; - v_uint16 v_2int = vx_load_expand(src2 + x) & v_mask; + v_mask = v_not(v_eq(v_mask, v_0)); + v_uint16 v_1int = v_and(vx_load_expand(src1 + x), v_mask); + v_uint16 v_2int = v_and(vx_load_expand(src2 + x), v_mask); v_uint32 v_1int_0, v_1int_1, v_2int_0, v_2int_1; v_expand(v_1int, v_1int_0, v_1int_1); @@ -2198,13 +2198,13 @@ void accProd_simd_(const uchar* src1, const uchar* src2, double* dst, const ucha v_uint16 v_2int2 = v_expand_low(v_2src2); v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); - v_1int0 = v_1int0 & v_mask; - v_1int1 = v_1int1 & v_mask; - v_1int2 = v_1int2 & v_mask; - v_2int0 = v_2int0 & v_mask; - v_2int1 = v_2int1 & v_mask; - v_2int2 = v_2int2 & v_mask; + v_mask = v_not(v_eq(v_mask, v_0)); + v_1int0 = v_and(v_1int0, v_mask); + v_1int1 = v_and(v_1int1, v_mask); + v_1int2 = v_and(v_1int2, v_mask); + v_2int0 = v_and(v_2int0, v_mask); + v_2int1 = v_and(v_2int1, v_mask); + v_2int2 = v_and(v_2int2, v_mask); v_uint32 v_1int00, v_1int01, v_1int10, v_1int11, v_1int20, v_1int21; v_uint32 v_2int00, v_2int01, v_2int10, v_2int11, v_2int20, v_2int21; @@ -2248,9 +2248,9 @@ void accProd_simd_(const uchar* src1, const uchar* src2, double* dst, const ucha void accProd_simd_(const ushort* src1, const ushort* src2, double* dst, const uchar* mask, int len, int cn) { int x = 0; -#if CV_SIMD_64F - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2293,11 +2293,11 @@ void accProd_simd_(const ushort* src1, const ushort* src2, double* dst, const uc for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_1src = vx_load(src1 + x); v_uint16 v_2src = vx_load(src2 + x); - v_1src = v_1src & v_mask; - v_2src = v_2src & v_mask; + v_1src = v_and(v_1src, v_mask); + v_2src = v_and(v_2src, v_mask); v_uint32 v_1int_0, v_1int_1, v_2int_0, v_2int_1; v_expand(v_1src, v_1int_0, v_1int_1); @@ -2329,16 +2329,16 @@ void accProd_simd_(const ushort* src1, const ushort* src2, double* dst, const uc for (; x <= len - cVectorWidth; x += cVectorWidth) { v_uint16 v_mask = vx_load_expand(mask + x); - v_mask = ~(v_mask == v_0); + v_mask = v_not(v_eq(v_mask, v_0)); v_uint16 v_1src0, v_1src1, v_1src2, v_2src0, v_2src1, v_2src2; v_load_deinterleave(src1 + x * cn, v_1src0, v_1src1, v_1src2); v_load_deinterleave(src2 + x * cn, v_2src0, v_2src1, v_2src2); - v_1src0 = v_1src0 & v_mask; - v_1src1 = v_1src1 & v_mask; - v_1src2 = v_1src2 & v_mask; - v_2src0 = v_2src0 & v_mask; - v_2src1 = v_2src1 & v_mask; - v_2src2 = v_2src2 & v_mask; + v_1src0 = v_and(v_1src0, v_mask); + v_1src1 = v_and(v_1src1, v_mask); + v_1src2 = v_and(v_1src2, v_mask); + v_2src0 = v_and(v_2src0, v_mask); + v_2src1 = v_and(v_2src1, v_mask); + v_2src2 = v_and(v_2src2, v_mask); v_uint32 v_1int_00, v_1int_01, v_2int_00, v_2int_01; v_uint32 v_1int_10, v_1int_11, v_2int_10, v_2int_11; @@ -2594,11 +2594,11 @@ void accProd_simd_(const double* src1, const double* src2, double* dst, const uc void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn, double alpha) { int x = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const v_float32 v_alpha = vx_setall_f32((float)alpha); const v_float32 v_beta = vx_setall_f32((float)(1.0f - alpha)); - const int cVectorWidth = v_uint8::nlanes; - const int step = v_float32::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2619,10 +2619,10 @@ void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn v_float32 v_dst10 = vx_load(dst + x + step * 2); v_float32 v_dst11 = vx_load(dst + x + step * 3); - v_dst00 = v_fma(v_dst00, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src00)) * v_alpha); - v_dst01 = v_fma(v_dst01, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src01)) * v_alpha); - v_dst10 = v_fma(v_dst10, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src10)) * v_alpha); - v_dst11 = v_fma(v_dst11, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src11)) * v_alpha); + v_dst00 = v_fma(v_dst00, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src00)), v_alpha)); + v_dst01 = v_fma(v_dst01, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src01)), v_alpha)); + v_dst10 = v_fma(v_dst10, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src10)), v_alpha)); + v_dst11 = v_fma(v_dst11, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src11)), v_alpha)); v_store(dst + x , v_dst00); v_store(dst + x + step , v_dst01); @@ -2663,15 +2663,15 @@ void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn v_float32 v_dst10 = vx_load(dst + x + step * 2); v_float32 v_dst11 = vx_load(dst + x + step * 3); - v_mf00 = v_mf00 != zero; - v_mf01 = v_mf01 != zero; - v_mf10 = v_mf10 != zero; - v_mf11 = v_mf11 != zero; + v_mf00 = v_ne(v_mf00, zero); + v_mf01 = v_ne(v_mf01, zero); + v_mf10 = v_ne(v_mf10, zero); + v_mf11 = v_ne(v_mf11, zero); - v_dst00 = v_select(v_mf00, v_fma(v_dst00, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src00)) * v_alpha), v_dst00); - v_dst01 = v_select(v_mf01, v_fma(v_dst01, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src01)) * v_alpha), v_dst01); - v_dst10 = v_select(v_mf10, v_fma(v_dst10, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src10)) * v_alpha), v_dst10); - v_dst11 = v_select(v_mf11, v_fma(v_dst11, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src11)) * v_alpha), v_dst11); + v_dst00 = v_select(v_mf00, v_fma(v_dst00, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src00)), v_alpha)), v_dst00); + v_dst01 = v_select(v_mf01, v_fma(v_dst01, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src01)), v_alpha)), v_dst01); + v_dst10 = v_select(v_mf10, v_fma(v_dst10, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src10)), v_alpha)), v_dst10); + v_dst11 = v_select(v_mf11, v_fma(v_dst11, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src11)), v_alpha)), v_dst11); v_store(dst + x , v_dst00); v_store(dst + x + step , v_dst01); @@ -2719,25 +2719,25 @@ void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn v_mf10 = v_cvt_f32(v_reinterpret_as_s32(v_m10)); v_mf11 = v_cvt_f32(v_reinterpret_as_s32(v_m11)); - v_mf00 = v_mf00 != zero; - v_mf01 = v_mf01 != zero; - v_mf10 = v_mf10 != zero; - v_mf11 = v_mf11 != zero; + v_mf00 = v_ne(v_mf00, zero); + v_mf01 = v_ne(v_mf01, zero); + v_mf10 = v_ne(v_mf10, zero); + v_mf11 = v_ne(v_mf11, zero); - v_dst00 = v_select(v_mf00, v_fma(v_dst00, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src000)) * v_alpha), v_dst00); - v_dst01 = v_select(v_mf01, v_fma(v_dst01, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src001)) * v_alpha), v_dst01); - v_dst02 = v_select(v_mf10, v_fma(v_dst02, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src010)) * v_alpha), v_dst02); - v_dst03 = v_select(v_mf11, v_fma(v_dst03, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src011)) * v_alpha), v_dst03); + v_dst00 = v_select(v_mf00, v_fma(v_dst00, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src000)), v_alpha)), v_dst00); + v_dst01 = v_select(v_mf01, v_fma(v_dst01, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src001)), v_alpha)), v_dst01); + v_dst02 = v_select(v_mf10, v_fma(v_dst02, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src010)), v_alpha)), v_dst02); + v_dst03 = v_select(v_mf11, v_fma(v_dst03, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src011)), v_alpha)), v_dst03); - v_dst10 = v_select(v_mf00, v_fma(v_dst10, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src100)) * v_alpha), v_dst10); - v_dst11 = v_select(v_mf01, v_fma(v_dst11, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src101)) * v_alpha), v_dst11); - v_dst12 = v_select(v_mf10, v_fma(v_dst12, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src110)) * v_alpha), v_dst12); - v_dst13 = v_select(v_mf11, v_fma(v_dst13, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src111)) * v_alpha), v_dst13); + v_dst10 = v_select(v_mf00, v_fma(v_dst10, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src100)), v_alpha)), v_dst10); + v_dst11 = v_select(v_mf01, v_fma(v_dst11, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src101)), v_alpha)), v_dst11); + v_dst12 = v_select(v_mf10, v_fma(v_dst12, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src110)), v_alpha)), v_dst12); + v_dst13 = v_select(v_mf11, v_fma(v_dst13, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src111)), v_alpha)), v_dst13); - v_dst20 = v_select(v_mf00, v_fma(v_dst20, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src200)) * v_alpha), v_dst20); - v_dst21 = v_select(v_mf01, v_fma(v_dst21, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src201)) * v_alpha), v_dst21); - v_dst22 = v_select(v_mf10, v_fma(v_dst22, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src210)) * v_alpha), v_dst22); - v_dst23 = v_select(v_mf11, v_fma(v_dst23, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src211)) * v_alpha), v_dst23); + v_dst20 = v_select(v_mf00, v_fma(v_dst20, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src200)), v_alpha)), v_dst20); + v_dst21 = v_select(v_mf01, v_fma(v_dst21, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src201)), v_alpha)), v_dst21); + v_dst22 = v_select(v_mf10, v_fma(v_dst22, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src210)), v_alpha)), v_dst22); + v_dst23 = v_select(v_mf11, v_fma(v_dst23, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src211)), v_alpha)), v_dst23); v_store_interleave(dst + x * cn , v_dst00, v_dst10, v_dst20); v_store_interleave(dst + ( x + step ) * cn, v_dst01, v_dst11, v_dst21); @@ -2753,11 +2753,11 @@ void accW_simd_(const uchar* src, float* dst, const uchar* mask, int len, int cn void accW_simd_(const ushort* src, float* dst, const uchar* mask, int len, int cn, double alpha) { int x = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const v_float32 v_alpha = vx_setall_f32((float)alpha); const v_float32 v_beta = vx_setall_f32((float)(1.0f - alpha)); - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2770,8 +2770,8 @@ void accW_simd_(const ushort* src, float* dst, const uchar* mask, int len, int c v_float32 v_dst0 = vx_load(dst + x); v_float32 v_dst1 = vx_load(dst + x + step); - v_dst0 = v_fma(v_dst0, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_int0)) * v_alpha); - v_dst1 = v_fma(v_dst1, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_int1)) * v_alpha); + v_dst0 = v_fma(v_dst0, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_int0)), v_alpha)); + v_dst1 = v_fma(v_dst1, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_int1)), v_alpha)); v_store(dst + x , v_dst0); v_store(dst + x + step, v_dst1); @@ -2799,11 +2799,11 @@ void accW_simd_(const ushort* src, float* dst, const uchar* mask, int len, int c v_float32 v_dst0 = vx_load(dst + x); v_float32 v_dst1 = vx_load(dst + x + step); - v_mf0 = v_mf0 != zero; - v_mf1 = v_mf1 != zero; + v_mf0 = v_ne(v_mf0, zero); + v_mf1 = v_ne(v_mf1, zero); - v_dst0 = v_select(v_mf0, v_fma(v_dst0, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src0)) * v_alpha), v_dst0); - v_dst1 = v_select(v_mf1, v_fma(v_dst1, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src1)) * v_alpha), v_dst1); + v_dst0 = v_select(v_mf0, v_fma(v_dst0, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src0)), v_alpha)), v_dst0); + v_dst1 = v_select(v_mf1, v_fma(v_dst1, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src1)), v_alpha)), v_dst1); v_store(dst + x , v_dst0); v_store(dst + x + step, v_dst1); @@ -2833,16 +2833,16 @@ void accW_simd_(const ushort* src, float* dst, const uchar* mask, int len, int c v_mf0 = v_cvt_f32(v_reinterpret_as_s32(v_m0)); v_mf1 = v_cvt_f32(v_reinterpret_as_s32(v_m1)); - v_mf0 = v_mf0 != zero; - v_mf1 = v_mf1 != zero; + v_mf0 = v_ne(v_mf0, zero); + v_mf1 = v_ne(v_mf1, zero); - v_dst00 = v_select(v_mf0, v_fma(v_dst00, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src00)) * v_alpha), v_dst00); - v_dst10 = v_select(v_mf0, v_fma(v_dst10, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src10)) * v_alpha), v_dst10); - v_dst20 = v_select(v_mf0, v_fma(v_dst20, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src20)) * v_alpha), v_dst20); + v_dst00 = v_select(v_mf0, v_fma(v_dst00, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src00)), v_alpha)), v_dst00); + v_dst10 = v_select(v_mf0, v_fma(v_dst10, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src10)), v_alpha)), v_dst10); + v_dst20 = v_select(v_mf0, v_fma(v_dst20, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src20)), v_alpha)), v_dst20); - v_dst01 = v_select(v_mf1, v_fma(v_dst01, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src01)) * v_alpha), v_dst01); - v_dst11 = v_select(v_mf1, v_fma(v_dst11, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src11)) * v_alpha), v_dst11); - v_dst21 = v_select(v_mf1, v_fma(v_dst21, v_beta, v_cvt_f32(v_reinterpret_as_s32(v_src21)) * v_alpha), v_dst21); + v_dst01 = v_select(v_mf1, v_fma(v_dst01, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src01)), v_alpha)), v_dst01); + v_dst11 = v_select(v_mf1, v_fma(v_dst11, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src11)), v_alpha)), v_dst11); + v_dst21 = v_select(v_mf1, v_fma(v_dst21, v_beta, v_mul(v_cvt_f32(v_reinterpret_as_s32(v_src21)), v_alpha)), v_dst21); v_store_interleave(dst + x * cn , v_dst00, v_dst10, v_dst20); v_store_interleave(dst + ( x + step ) * cn, v_dst01, v_dst11, v_dst21); @@ -2870,11 +2870,11 @@ void accW_simd_(const float* src, float* dst, const uchar* mask, int len, int cn _mm256_storeu_ps(dst + x + 8, _mm256_add_ps(_mm256_mul_ps(_mm256_loadu_ps(dst + x + 8), v_beta), _mm256_mul_ps(_mm256_loadu_ps(src + x + 8), v_alpha))); } } -#elif CV_SIMD +#elif (CV_SIMD || CV_SIMD_SCALABLE) const v_float32 v_alpha = vx_setall_f32((float)alpha); const v_float32 v_beta = vx_setall_f32((float)(1.0f - alpha)); - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float32::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2884,8 +2884,8 @@ void accW_simd_(const float* src, float* dst, const uchar* mask, int len, int cn v_float32 v_dst0 = vx_load(dst + x); v_float32 v_dst1 = vx_load(dst + x + step); - v_dst0 = v_fma(v_dst0, v_beta, vx_load(src + x) * v_alpha); - v_dst1 = v_fma(v_dst1, v_beta, vx_load(src + x + step) * v_alpha); + v_dst0 = v_fma(v_dst0, v_beta, v_mul(vx_load(src + x), v_alpha)); + v_dst1 = v_fma(v_dst1, v_beta, v_mul(vx_load(src + x + step), v_alpha)); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -2898,11 +2898,11 @@ void accW_simd_(const float* src, float* dst, const uchar* mask, int len, int cn void accW_simd_(const uchar* src, double* dst, const uchar* mask, int len, int cn, double alpha) { int x = 0; -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) const v_float64 v_alpha = vx_setall_f64(alpha); const v_float64 v_beta = vx_setall_f64(1.0f - alpha); - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2927,10 +2927,10 @@ void accW_simd_(const uchar* src, double* dst, const uchar* mask, int len, int c v_float64 v_dst2 = vx_load(dst + x + step * 2); v_float64 v_dst3 = vx_load(dst + x + step * 3); - v_dst0 = v_fma(v_dst0, v_beta, v_src0 * v_alpha); - v_dst1 = v_fma(v_dst1, v_beta, v_src1 * v_alpha); - v_dst2 = v_fma(v_dst2, v_beta, v_src2 * v_alpha); - v_dst3 = v_fma(v_dst3, v_beta, v_src3 * v_alpha); + v_dst0 = v_fma(v_dst0, v_beta, v_mul(v_src0, v_alpha)); + v_dst1 = v_fma(v_dst1, v_beta, v_mul(v_src1, v_alpha)); + v_dst2 = v_fma(v_dst2, v_beta, v_mul(v_src2, v_alpha)); + v_dst3 = v_fma(v_dst3, v_beta, v_mul(v_src3, v_alpha)); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); @@ -2945,11 +2945,11 @@ void accW_simd_(const uchar* src, double* dst, const uchar* mask, int len, int c void accW_simd_(const ushort* src, double* dst, const uchar* mask, int len, int cn, double alpha) { int x = 0; -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) const v_float64 v_alpha = vx_setall_f64(alpha); const v_float64 v_beta = vx_setall_f64(1.0f - alpha); - const int cVectorWidth = v_uint16::nlanes; - const int step = v_float64::nlanes; + const int cVectorWidth = VTraits::vlanes(); + const int step = VTraits::vlanes(); if (!mask) { @@ -2973,10 +2973,10 @@ void accW_simd_(const ushort* src, double* dst, const uchar* mask, int len, int v_float64 v_dst10 = vx_load(dst + x + step * 2); v_float64 v_dst11 = vx_load(dst + x + step * 3); - v_dst00 = v_fma(v_dst00, v_beta, v_src00 * v_alpha); - v_dst01 = v_fma(v_dst01, v_beta, v_src01 * v_alpha); - v_dst10 = v_fma(v_dst10, v_beta, v_src10 * v_alpha); - v_dst11 = v_fma(v_dst11, v_beta, v_src11 * v_alpha); + v_dst00 = v_fma(v_dst00, v_beta, v_mul(v_src00, v_alpha)); + v_dst01 = v_fma(v_dst01, v_beta, v_mul(v_src01, v_alpha)); + v_dst10 = v_fma(v_dst10, v_beta, v_mul(v_src10, v_alpha)); + v_dst11 = v_fma(v_dst11, v_beta, v_mul(v_src11, v_alpha)); v_store(dst + x, v_dst00); v_store(dst + x + step, v_dst01); @@ -3014,11 +3014,11 @@ void accW_simd_(const float* src, double* dst, const uchar* mask, int len, int c _mm256_storeu_pd(dst + x + 12, _mm256_add_pd(_mm256_mul_pd(_mm256_loadu_pd(dst + x + 12), v_beta), _mm256_mul_pd(v_src11, v_alpha))); } } -#elif CV_SIMD_64F +#elif (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) const v_float64 v_alpha = vx_setall_f64(alpha); const v_float64 v_beta = vx_setall_f64(1.0f - alpha); - const int cVectorWidth = v_float32::nlanes * 2; - const int step = v_float64::nlanes; + const int cVectorWidth = VTraits::vlanes() * 2; + const int step = VTraits::vlanes(); if (!mask) { @@ -3026,7 +3026,7 @@ void accW_simd_(const float* src, double* dst, const uchar* mask, int len, int c for (; x <= size - cVectorWidth; x += cVectorWidth) { v_float32 v_src0 = vx_load(src + x); - v_float32 v_src1 = vx_load(src + x + v_float32::nlanes); + v_float32 v_src1 = vx_load(src + x + VTraits::vlanes()); v_float64 v_src00 = v_cvt_f64(v_src0); v_float64 v_src01 = v_cvt_f64_high(v_src0); v_float64 v_src10 = v_cvt_f64(v_src1); @@ -3037,10 +3037,10 @@ void accW_simd_(const float* src, double* dst, const uchar* mask, int len, int c v_float64 v_dst10 = vx_load(dst + x + step * 2); v_float64 v_dst11 = vx_load(dst + x + step * 3); - v_dst00 = v_fma(v_dst00, v_beta, v_src00 * v_alpha); - v_dst01 = v_fma(v_dst01, v_beta, v_src01 * v_alpha); - v_dst10 = v_fma(v_dst10, v_beta, v_src10 * v_alpha); - v_dst11 = v_fma(v_dst11, v_beta, v_src11 * v_alpha); + v_dst00 = v_fma(v_dst00, v_beta, v_mul(v_src00, v_alpha)); + v_dst01 = v_fma(v_dst01, v_beta, v_mul(v_src01, v_alpha)); + v_dst10 = v_fma(v_dst10, v_beta, v_mul(v_src10, v_alpha)); + v_dst11 = v_fma(v_dst11, v_beta, v_mul(v_src11, v_alpha)); v_store(dst + x, v_dst00); v_store(dst + x + step, v_dst01); @@ -3072,11 +3072,11 @@ void accW_simd_(const double* src, double* dst, const uchar* mask, int len, int _mm256_storeu_pd(dst + x + 4, _mm256_add_pd(_mm256_mul_pd(_mm256_loadu_pd(dst + x + 4), v_beta), _mm256_mul_pd(v_src1, v_alpha))); } } -#elif CV_SIMD_64F +#elif (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) const v_float64 v_alpha = vx_setall_f64(alpha); const v_float64 v_beta = vx_setall_f64(1.0f - alpha); - const int cVectorWidth = v_float64::nlanes * 2; - const int step = v_float64::nlanes; + const int cVectorWidth = VTraits::vlanes() * 2; + const int step = VTraits::vlanes(); if (!mask) { @@ -3089,8 +3089,8 @@ void accW_simd_(const double* src, double* dst, const uchar* mask, int len, int v_float64 v_dst0 = vx_load(dst + x); v_float64 v_dst1 = vx_load(dst + x + step); - v_dst0 = v_fma(v_dst0, v_beta, v_src0 * v_alpha); - v_dst1 = v_fma(v_dst1, v_beta, v_src1 * v_alpha); + v_dst0 = v_fma(v_dst0, v_beta, v_mul(v_src0, v_alpha)); + v_dst1 = v_fma(v_dst1, v_beta, v_mul(v_src1, v_alpha)); v_store(dst + x, v_dst0); v_store(dst + x + step, v_dst1); diff --git a/modules/imgproc/src/bilateral_filter.simd.hpp b/modules/imgproc/src/bilateral_filter.simd.hpp index 0d2c394368..332b36646c 100644 --- a/modules/imgproc/src/bilateral_filter.simd.hpp +++ b/modules/imgproc/src/bilateral_filter.simd.hpp @@ -99,33 +99,33 @@ public: const uchar* ksptr2 = sptr + space_ofs[k+2]; const uchar* ksptr3 = sptr + space_ofs[k+3]; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight0 = vx_setall_f32(space_weight[k]); v_float32 kweight1 = vx_setall_f32(space_weight[k+1]); v_float32 kweight2 = vx_setall_f32(space_weight[k+2]); v_float32 kweight3 = vx_setall_f32(space_weight[k+3]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint32 rval = vx_load_expand_q(sptr + j); v_uint32 val = vx_load_expand_q(ksptr0 + j); - v_float32 w = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); - v_float32 v_wsum = vx_load_aligned(wsum + j) + w; + v_float32 w = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); + v_float32 v_wsum = v_add(vx_load_aligned(wsum + j), w); v_float32 v_sum = v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, vx_load_aligned(sum + j)); val = vx_load_expand_q(ksptr1 + j); - w = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); - v_wsum += w; + w = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); + v_wsum = v_add(v_wsum, w); v_sum = v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, v_sum); val = vx_load_expand_q(ksptr2 + j); - w = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); - v_wsum += w; + w = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); + v_wsum = v_add(v_wsum, w); v_sum = v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, v_sum); val = vx_load_expand_q(ksptr3 + j); - w = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval))); - v_wsum += w; + w = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, rval)))); + v_wsum = v_add(v_wsum, w); v_sum = v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, v_sum); v_store_aligned(wsum + j, v_wsum); @@ -172,13 +172,13 @@ public: { const uchar* ksptr = sptr + space_ofs[k]; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight = vx_setall_f32(space_weight[k]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint32 val = vx_load_expand_q(ksptr + j); - v_float32 w = kweight * v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, vx_load_expand_q(sptr + j)))); - v_store_aligned(wsum + j, vx_load_aligned(wsum + j) + w); + v_float32 w = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(v_absdiff(val, vx_load_expand_q(sptr + j))))); + v_store_aligned(wsum + j, v_add(vx_load_aligned(wsum + j), w)); v_store_aligned(sum + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val)), w, vx_load_aligned(sum + j))); } #endif @@ -191,10 +191,10 @@ public: } } j = 0; -#if CV_SIMD - for (; j <= size.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes) - v_pack_u_store(dptr + j, v_pack(v_round(vx_load_aligned(sum + j ) / vx_load_aligned(wsum + j )), - v_round(vx_load_aligned(sum + j + v_float32::nlanes) / vx_load_aligned(wsum + j + v_float32::nlanes)))); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j <= size.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) + v_pack_u_store(dptr + j, v_pack(v_round(v_div(vx_load_aligned(sum + j), vx_load_aligned(wsum + j))), + v_round(v_div(vx_load_aligned(sum + j + VTraits::vlanes()), vx_load_aligned(wsum + j + VTraits::vlanes()))))); #endif for (; j < size.width; j++) { @@ -221,13 +221,13 @@ public: const uchar* ksptr3 = sptr + space_ofs[k+3]; const uchar* rsptr = sptr; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight0 = vx_setall_f32(space_weight[k]); v_float32 kweight1 = vx_setall_f32(space_weight[k+1]); v_float32 kweight2 = vx_setall_f32(space_weight[k+2]); v_float32 kweight3 = vx_setall_f32(space_weight[k+3]); - for (; j <= size.width - v_uint8::nlanes; j += v_uint8::nlanes, rsptr += 3*v_uint8::nlanes, - ksptr0 += 3*v_uint8::nlanes, ksptr1 += 3*v_uint8::nlanes, ksptr2 += 3*v_uint8::nlanes, ksptr3 += 3*v_uint8::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), rsptr += 3*VTraits::vlanes(), + ksptr0 += 3*VTraits::vlanes(), ksptr1 += 3*VTraits::vlanes(), ksptr2 += 3*VTraits::vlanes(), ksptr3 += 3*VTraits::vlanes()) { v_uint8 kb, kg, kr, rb, rg, rr; v_load_deinterleave(rsptr, rb, rg, rr); @@ -236,163 +236,163 @@ public: v_uint16 val0, val1, val2, val3, val4; v_expand(v_absdiff(kb, rb), val0, val1); v_expand(v_absdiff(kg, rg), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(v_absdiff(kr, rr), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_uint32 vall, valh; v_expand(val0, vall, valh); - v_float32 w0 = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - v_float32 w1 = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j, w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); + v_float32 w0 = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(vall))); + v_float32 w1 = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j, v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); v_expand(kb, val0, val2); v_expand(val0, vall, valh); v_store_aligned(sum_b + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); v_expand(kg, val0, val3); v_expand(val0, vall, valh); v_store_aligned(sum_g + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); v_expand(kr, val0, val4); v_expand(val0, vall, valh); v_store_aligned(sum_r + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); v_expand(val1, vall, valh); - w0 = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight0 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j + 2 * v_float32::nlanes, w0 + vx_load_aligned(wsum + j + 2 * v_float32::nlanes)); - v_store_aligned(wsum + j + 3 * v_float32::nlanes, w1 + vx_load_aligned(wsum + j + 3 * v_float32::nlanes)); + w0 = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight0, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j + 2 * VTraits::vlanes(), v_add(w0, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3 * VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(val2, vall, valh); - v_store_aligned(sum_b + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_b + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_b + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))); v_expand(val3, vall, valh); - v_store_aligned(sum_g + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_g + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_g + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))); v_expand(val4, vall, valh); - v_store_aligned(sum_r + j + 2*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2*v_float32::nlanes))); - v_store_aligned(sum_r + j + 3*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3*v_float32::nlanes))); + v_store_aligned(sum_r + j + 2*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2*VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3*VTraits::vlanes()))); v_load_deinterleave(ksptr1, kb, kg, kr); v_expand(v_absdiff(kb, rb), val0, val1); v_expand(v_absdiff(kg, rg), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(v_absdiff(kr, rr), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(val0, vall, valh); - w0 = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j, w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); + w0 = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j, v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); v_expand(kb, val0, val2); v_expand(val0, vall, valh); v_store_aligned(sum_b + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); v_expand(kg, val0, val3); v_expand(val0, vall, valh); v_store_aligned(sum_g + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); v_expand(kr, val0, val4); v_expand(val0, vall, valh); v_store_aligned(sum_r + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); v_expand(val1, vall, valh); - w0 = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight1 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j + 2 * v_float32::nlanes, w0 + vx_load_aligned(wsum + j + 2 * v_float32::nlanes)); - v_store_aligned(wsum + j + 3 * v_float32::nlanes, w1 + vx_load_aligned(wsum + j + 3 * v_float32::nlanes)); + w0 = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight1, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j + 2 * VTraits::vlanes(), v_add(w0, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3 * VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(val2, vall, valh); - v_store_aligned(sum_b + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_b + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_b + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))); v_expand(val3, vall, valh); - v_store_aligned(sum_g + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_g + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_g + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))); v_expand(val4, vall, valh); - v_store_aligned(sum_r + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_r + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_r + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * VTraits::vlanes()))); v_load_deinterleave(ksptr2, kb, kg, kr); v_expand(v_absdiff(kb, rb), val0, val1); v_expand(v_absdiff(kg, rg), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(v_absdiff(kr, rr), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(val0, vall, valh); - w0 = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j, w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); + w0 = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j, v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); v_expand(kb, val0, val2); v_expand(val0, vall, valh); v_store_aligned(sum_b + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); v_expand(kg, val0, val3); v_expand(val0, vall, valh); v_store_aligned(sum_g + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); v_expand(kr, val0, val4); v_expand(val0, vall, valh); v_store_aligned(sum_r + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); v_expand(val1, vall, valh); - w0 = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight2 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j + 2 * v_float32::nlanes, w0 + vx_load_aligned(wsum + j + 2 * v_float32::nlanes)); - v_store_aligned(wsum + j + 3 * v_float32::nlanes, w1 + vx_load_aligned(wsum + j + 3 * v_float32::nlanes)); + w0 = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight2, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j + 2 * VTraits::vlanes(), v_add(w0, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3 * VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(val2, vall, valh); - v_store_aligned(sum_b + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_b + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_b + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))); v_expand(val3, vall, valh); - v_store_aligned(sum_g + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_g + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_g + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))); v_expand(val4, vall, valh); - v_store_aligned(sum_r + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_r + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_r + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * VTraits::vlanes()))); v_load_deinterleave(ksptr3, kb, kg, kr); v_expand(v_absdiff(kb, rb), val0, val1); v_expand(v_absdiff(kg, rg), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(v_absdiff(kr, rr), val2, val3); - val0 += val2; val1 += val3; + val0 = v_add(val0, val2); val1 = v_add(val1, val3); v_expand(val0, vall, valh); - w0 = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j, w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); + w0 = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j, v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); v_expand(kb, val0, val2); v_expand(val0, vall, valh); v_store_aligned(sum_b + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); v_expand(kg, val0, val3); v_expand(val0, vall, valh); v_store_aligned(sum_g + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); v_expand(kr, val0, val4); v_expand(val0, vall, valh); v_store_aligned(sum_r + j, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); v_expand(val1, vall, valh); - w0 = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(vall)); - w1 = kweight3 * v_lut(color_weight, v_reinterpret_as_s32(valh)); - v_store_aligned(wsum + j + 2 * v_float32::nlanes, w0 + vx_load_aligned(wsum + j + 2 * v_float32::nlanes)); - v_store_aligned(wsum + j + 3 * v_float32::nlanes, w1 + vx_load_aligned(wsum + j + 3 * v_float32::nlanes)); + w0 = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(vall))); + w1 = v_mul(kweight3, v_lut(color_weight, v_reinterpret_as_s32(valh))); + v_store_aligned(wsum + j + 2 * VTraits::vlanes(), v_add(w0, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3 * VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(val2, vall, valh); - v_store_aligned(sum_b + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_b + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_b + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))); v_expand(val3, vall, valh); - v_store_aligned(sum_g + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_g + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_g + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))); v_expand(val4, vall, valh); - v_store_aligned(sum_r + j + 2 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * v_float32::nlanes))); - v_store_aligned(sum_r + j + 3 * v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * v_float32::nlanes))); + v_store_aligned(sum_r + j + 2 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(vall)), w0, vx_load_aligned(sum_r + j + 2 * VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3 * VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(valh)), w1, vx_load_aligned(sum_r + j + 3 * VTraits::vlanes()))); } #endif #if CV_SIMD128 @@ -442,9 +442,9 @@ public: const uchar* ksptr = sptr + space_ofs[k]; const uchar* rsptr = sptr; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight = vx_setall_f32(space_weight[k]); - for (; j <= size.width - v_uint8::nlanes; j += v_uint8::nlanes, ksptr += 3*v_uint8::nlanes, rsptr += 3*v_uint8::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), ksptr += 3*VTraits::vlanes(), rsptr += 3*VTraits::vlanes()) { v_uint8 kb, kg, kr, rb, rg, rr; v_load_deinterleave(ksptr, kb, kg, kr); @@ -456,39 +456,39 @@ public: v_expand(v_absdiff(kr, rr), r_l, r_h); v_uint32 val0, val1, val2, val3; - v_expand(b_l + g_l + r_l, val0, val1); - v_expand(b_h + g_h + r_h, val2, val3); + v_expand(v_add(v_add(b_l, g_l), r_l), val0, val1); + v_expand(v_add(v_add(b_h, g_h), r_h), val2, val3); v_expand(kb, b_l, b_h); v_expand(kg, g_l, g_h); v_expand(kr, r_l, r_h); - v_float32 w0 = kweight * v_lut(color_weight, v_reinterpret_as_s32(val0)); - v_float32 w1 = kweight * v_lut(color_weight, v_reinterpret_as_s32(val1)); - v_float32 w2 = kweight * v_lut(color_weight, v_reinterpret_as_s32(val2)); - v_float32 w3 = kweight * v_lut(color_weight, v_reinterpret_as_s32(val3)); - v_store_aligned(wsum + j , w0 + vx_load_aligned(wsum + j)); - v_store_aligned(wsum + j + v_float32::nlanes, w1 + vx_load_aligned(wsum + j + v_float32::nlanes)); - v_store_aligned(wsum + j + 2*v_float32::nlanes, w2 + vx_load_aligned(wsum + j + 2*v_float32::nlanes)); - v_store_aligned(wsum + j + 3*v_float32::nlanes, w3 + vx_load_aligned(wsum + j + 3*v_float32::nlanes)); + v_float32 w0 = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(val0))); + v_float32 w1 = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(val1))); + v_float32 w2 = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(val2))); + v_float32 w3 = v_mul(kweight, v_lut(color_weight, v_reinterpret_as_s32(val3))); + v_store_aligned(wsum + j , v_add(w0, vx_load_aligned(wsum + j))); + v_store_aligned(wsum + j + VTraits::vlanes(), v_add(w1, vx_load_aligned(wsum + j + VTraits::vlanes()))); + v_store_aligned(wsum + j + 2*VTraits::vlanes(), v_add(w2, vx_load_aligned(wsum + j + 2 * VTraits::vlanes()))); + v_store_aligned(wsum + j + 3*VTraits::vlanes(), v_add(w3, vx_load_aligned(wsum + j + 3 * VTraits::vlanes()))); v_expand(b_l, val0, val1); v_expand(b_h, val2, val3); v_store_aligned(sum_b + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(val0)), w0, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_b + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_b + j + v_float32::nlanes))); - v_store_aligned(sum_b + j + 2*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_b + j + 2*v_float32::nlanes))); - v_store_aligned(sum_b + j + 3*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_b + j + 3*v_float32::nlanes))); + v_store_aligned(sum_b + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_b + j + VTraits::vlanes()))); + v_store_aligned(sum_b + j + 2*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_b + j + 2*VTraits::vlanes()))); + v_store_aligned(sum_b + j + 3*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_b + j + 3*VTraits::vlanes()))); v_expand(g_l, val0, val1); v_expand(g_h, val2, val3); v_store_aligned(sum_g + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(val0)), w0, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_g + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_g + j + v_float32::nlanes))); - v_store_aligned(sum_g + j + 2*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_g + j + 2*v_float32::nlanes))); - v_store_aligned(sum_g + j + 3*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_g + j + 3*v_float32::nlanes))); + v_store_aligned(sum_g + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_g + j + VTraits::vlanes()))); + v_store_aligned(sum_g + j + 2*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_g + j + 2*VTraits::vlanes()))); + v_store_aligned(sum_g + j + 3*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_g + j + 3*VTraits::vlanes()))); v_expand(r_l, val0, val1); v_expand(r_h, val2, val3); v_store_aligned(sum_r + j , v_muladd(v_cvt_f32(v_reinterpret_as_s32(val0)), w0, vx_load_aligned(sum_r + j))); - v_store_aligned(sum_r + j + v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_r + j + v_float32::nlanes))); - v_store_aligned(sum_r + j + 2*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_r + j + 2*v_float32::nlanes))); - v_store_aligned(sum_r + j + 3*v_float32::nlanes, v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_r + j + 3*v_float32::nlanes))); + v_store_aligned(sum_r + j + VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val1)), w1, vx_load_aligned(sum_r + j + VTraits::vlanes()))); + v_store_aligned(sum_r + j + 2*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val2)), w2, vx_load_aligned(sum_r + j + 2*VTraits::vlanes()))); + v_store_aligned(sum_r + j + 3*VTraits::vlanes(), v_muladd(v_cvt_f32(v_reinterpret_as_s32(val3)), w3, vx_load_aligned(sum_r + j + 3*VTraits::vlanes()))); } #endif for(; j < size.width; j++, ksptr += 3, rsptr += 3) @@ -500,27 +500,27 @@ public: } } j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 v_one = vx_setall_f32(1.f); - for(; j <= size.width - v_uint8::nlanes; j += v_uint8::nlanes, dptr += 3*v_uint8::nlanes) + for(; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), dptr += 3*VTraits::vlanes()) { - v_float32 w0 = v_one / vx_load_aligned(wsum + j); - v_float32 w1 = v_one / vx_load_aligned(wsum + j + v_float32::nlanes); - v_float32 w2 = v_one / vx_load_aligned(wsum + j + 2*v_float32::nlanes); - v_float32 w3 = v_one / vx_load_aligned(wsum + j + 3*v_float32::nlanes); + v_float32 w0 = v_div(v_one, vx_load_aligned(wsum + j)); + v_float32 w1 = v_div(v_one, vx_load_aligned(wsum + j + VTraits::vlanes())); + v_float32 w2 = v_div(v_one, vx_load_aligned(wsum + j + 2 * VTraits::vlanes())); + v_float32 w3 = v_div(v_one, vx_load_aligned(wsum + j + 3 * VTraits::vlanes())); - v_store_interleave(dptr, v_pack_u(v_pack(v_round(w0 * vx_load_aligned(sum_b + j)), - v_round(w1 * vx_load_aligned(sum_b + j + v_float32::nlanes))), - v_pack(v_round(w2 * vx_load_aligned(sum_b + j + 2*v_float32::nlanes)), - v_round(w3 * vx_load_aligned(sum_b + j + 3*v_float32::nlanes)))), - v_pack_u(v_pack(v_round(w0 * vx_load_aligned(sum_g + j)), - v_round(w1 * vx_load_aligned(sum_g + j + v_float32::nlanes))), - v_pack(v_round(w2 * vx_load_aligned(sum_g + j + 2*v_float32::nlanes)), - v_round(w3 * vx_load_aligned(sum_g + j + 3*v_float32::nlanes)))), - v_pack_u(v_pack(v_round(w0 * vx_load_aligned(sum_r + j)), - v_round(w1 * vx_load_aligned(sum_r + j + v_float32::nlanes))), - v_pack(v_round(w2 * vx_load_aligned(sum_r + j + 2*v_float32::nlanes)), - v_round(w3 * vx_load_aligned(sum_r + j + 3*v_float32::nlanes))))); + v_store_interleave(dptr, v_pack_u(v_pack(v_round(v_mul(w0, vx_load_aligned(sum_b + j))), + v_round(v_mul(w1, vx_load_aligned(sum_b + j + VTraits::vlanes())))), + v_pack(v_round(v_mul(w2, vx_load_aligned(sum_b + j + 2 * VTraits::vlanes()))), + v_round(v_mul(w3, vx_load_aligned(sum_b + j + 3 * VTraits::vlanes()))))), + v_pack_u(v_pack(v_round(v_mul(w0, vx_load_aligned(sum_g + j))), + v_round(v_mul(w1, vx_load_aligned(sum_g + j + VTraits::vlanes())))), + v_pack(v_round(v_mul(w2, vx_load_aligned(sum_g + j + 2 * VTraits::vlanes()))), + v_round(v_mul(w3, vx_load_aligned(sum_g + j + 3 * VTraits::vlanes()))))), + v_pack_u(v_pack(v_round(v_mul(w0, vx_load_aligned(sum_r + j))), + v_round(v_mul(w1, vx_load_aligned(sum_r + j + VTraits::vlanes())))), + v_pack(v_round(v_mul(w2, vx_load_aligned(sum_r + j + 2 * VTraits::vlanes()))), + v_round(v_mul(w3, vx_load_aligned(sum_r + j + 3 * VTraits::vlanes())))))); } #endif for(; j < size.width; j++) @@ -533,7 +533,7 @@ public: } } } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -589,7 +589,7 @@ public: memset(buf.data(), 0, buf.size() * sizeof(float)); float *sum = alignPtr(buf.data(), CV_SIMD_WIDTH); float *wsum = sum + alignSize(size.width, CV_SIMD_WIDTH); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 v_one = vx_setall_f32(1.f); v_float32 sindex = vx_setall_f32(scale_index); #endif @@ -601,50 +601,50 @@ public: const float* ksptr2 = sptr + space_ofs[k + 2]; const float* ksptr3 = sptr + space_ofs[k + 3]; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight0 = vx_setall_f32(space_weight[k]); v_float32 kweight1 = vx_setall_f32(space_weight[k+1]); v_float32 kweight2 = vx_setall_f32(space_weight[k+2]); v_float32 kweight3 = vx_setall_f32(space_weight[k+3]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 rval = vx_load(sptr + j); v_float32 val = vx_load(ksptr0 + j); v_float32 knan = v_not_nan(val); - v_float32 alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + v_float32 alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); v_int32 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - v_float32 w = (kweight0 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one-alpha))) & knan; - v_float32 v_wsum = vx_load_aligned(wsum + j) + w; - v_float32 v_sum = v_muladd(val & knan, w, vx_load_aligned(sum + j)); + alpha = v_sub(alpha, v_cvt_f32(idx)); + v_float32 w = v_and(v_mul(kweight0, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_float32 v_wsum = v_add(vx_load_aligned(wsum + j), w); + v_float32 v_sum = v_muladd(v_and(val, knan), w, vx_load_aligned(sum + j)); val = vx_load(ksptr1 + j); knan = v_not_nan(val); - alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight1 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum = v_muladd(val & knan, w, v_sum); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight1, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum = v_muladd(v_and(val, knan), w, v_sum); val = vx_load(ksptr2 + j); knan = v_not_nan(val); - alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight2 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum = v_muladd(val & knan, w, v_sum); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight2, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum = v_muladd(v_and(val, knan), w, v_sum); val = vx_load(ksptr3 + j); knan = v_not_nan(val); - alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight3 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum = v_muladd(val & knan, w, v_sum); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight3, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum = v_muladd(v_and(val, knan), w, v_sum); v_store_aligned(wsum + j, v_wsum); v_store_aligned(sum + j, v_sum); @@ -720,20 +720,20 @@ public: { const float* ksptr = sptr + space_ofs[k]; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight = vx_setall_f32(space_weight[k]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 val = vx_load(ksptr + j); v_float32 rval = vx_load(sptr + j); v_float32 knan = v_not_nan(val); - v_float32 alpha = (v_absdiff(val, rval) * sindex) & v_not_nan(rval) & knan; + v_float32 alpha = v_and(v_and(v_mul(v_absdiff(val, rval), sindex), v_not_nan(rval)), knan); v_int32 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); + alpha = v_sub(alpha, v_cvt_f32(idx)); - v_float32 w = (kweight * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one-alpha))) & knan; - v_store_aligned(wsum + j, vx_load_aligned(wsum + j) + w); - v_store_aligned(sum + j, v_muladd(val & knan, w, vx_load_aligned(sum + j))); + v_float32 w = v_and(v_mul(kweight, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_store_aligned(wsum + j, v_add(vx_load_aligned(wsum + j), w)); + v_store_aligned(sum + j, v_muladd(v_and(val, knan), w, vx_load_aligned(sum + j))); } #endif for (; j < size.width; j++) @@ -752,11 +752,11 @@ public: } } j = 0; -#if CV_SIMD - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_val = vx_load(sptr + j); - v_store(dptr + j, (vx_load_aligned(sum + j) + (v_val & v_not_nan(v_val))) / (vx_load_aligned(wsum + j) + (v_one & v_not_nan(v_val)))); + v_store(dptr + j, v_div(v_add(vx_load_aligned(sum + j), v_and(v_val, v_not_nan(v_val))), v_add(vx_load_aligned(wsum + j), v_and(v_one, v_not_nan(v_val))))); } #endif for (; j < size.width; j++) @@ -774,7 +774,7 @@ public: float *sum_g = sum_b + alignSize(size.width, CV_SIMD_WIDTH); float *sum_r = sum_g + alignSize(size.width, CV_SIMD_WIDTH); float *wsum = sum_r + alignSize(size.width, CV_SIMD_WIDTH); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 v_one = vx_setall_f32(1.f); v_float32 sindex = vx_setall_f32(scale_index); #endif @@ -787,60 +787,60 @@ public: const float* ksptr3 = sptr + space_ofs[k+3]; const float* rsptr = sptr; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight0 = vx_setall_f32(space_weight[k]); v_float32 kweight1 = vx_setall_f32(space_weight[k+1]); v_float32 kweight2 = vx_setall_f32(space_weight[k+2]); v_float32 kweight3 = vx_setall_f32(space_weight[k+3]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes, rsptr += 3 * v_float32::nlanes, - ksptr0 += 3 * v_float32::nlanes, ksptr1 += 3 * v_float32::nlanes, ksptr2 += 3 * v_float32::nlanes, ksptr3 += 3 * v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), rsptr += 3 * VTraits::vlanes(), + ksptr0 += 3 * VTraits::vlanes(), ksptr1 += 3 * VTraits::vlanes(), ksptr2 += 3 * VTraits::vlanes(), ksptr3 += 3 * VTraits::vlanes()) { v_float32 kb, kg, kr, rb, rg, rr; v_load_deinterleave(rsptr, rb, rg, rr); v_load_deinterleave(ksptr0, kb, kg, kr); - v_float32 knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - v_float32 alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + v_float32 knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + v_float32 alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); v_int32 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - v_float32 w = (kweight0 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_float32 v_wsum = vx_load_aligned(wsum + j) + w; - v_float32 v_sum_b = v_muladd(kb & knan, w, vx_load_aligned(sum_b + j)); - v_float32 v_sum_g = v_muladd(kg & knan, w, vx_load_aligned(sum_g + j)); - v_float32 v_sum_r = v_muladd(kr & knan, w, vx_load_aligned(sum_r + j)); + alpha = v_sub(alpha, v_cvt_f32(idx)); + v_float32 w = v_and(v_mul(kweight0, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_float32 v_wsum = v_add(vx_load_aligned(wsum + j), w); + v_float32 v_sum_b = v_muladd(v_and(kb, knan), w, vx_load_aligned(sum_b + j)); + v_float32 v_sum_g = v_muladd(v_and(kg, knan), w, vx_load_aligned(sum_g + j)); + v_float32 v_sum_r = v_muladd(v_and(kr, knan), w, vx_load_aligned(sum_r + j)); v_load_deinterleave(ksptr1, kb, kg, kr); - knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight1 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum_b = v_muladd(kb & knan, w, v_sum_b); - v_sum_g = v_muladd(kg & knan, w, v_sum_g); - v_sum_r = v_muladd(kr & knan, w, v_sum_r); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight1, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum_b = v_muladd(v_and(kb, knan), w, v_sum_b); + v_sum_g = v_muladd(v_and(kg, knan), w, v_sum_g); + v_sum_r = v_muladd(v_and(kr, knan), w, v_sum_r); v_load_deinterleave(ksptr2, kb, kg, kr); - knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight2 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum_b = v_muladd(kb & knan, w, v_sum_b); - v_sum_g = v_muladd(kg & knan, w, v_sum_g); - v_sum_r = v_muladd(kr & knan, w, v_sum_r); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight2, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum_b = v_muladd(v_and(kb, knan), w, v_sum_b); + v_sum_g = v_muladd(v_and(kg, knan), w, v_sum_g); + v_sum_r = v_muladd(v_and(kr, knan), w, v_sum_r); v_load_deinterleave(ksptr3, kb, kg, kr); - knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); - w = (kweight3 * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_wsum += w; - v_sum_b = v_muladd(kb & knan, w, v_sum_b); - v_sum_g = v_muladd(kg & knan, w, v_sum_g); - v_sum_r = v_muladd(kr & knan, w, v_sum_r); + alpha = v_sub(alpha, v_cvt_f32(idx)); + w = v_and(v_mul(kweight3, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_wsum = v_add(v_wsum, w); + v_sum_b = v_muladd(v_and(kb, knan), w, v_sum_b); + v_sum_g = v_muladd(v_and(kg, knan), w, v_sum_g); + v_sum_r = v_muladd(v_and(kr, knan), w, v_sum_r); v_store_aligned(wsum + j, v_wsum); v_store_aligned(sum_b + j, v_sum_b); @@ -938,24 +938,24 @@ public: const float* ksptr = sptr + space_ofs[k]; const float* rsptr = sptr; j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 kweight = vx_setall_f32(space_weight[k]); - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes, ksptr += 3*v_float32::nlanes, rsptr += 3*v_float32::nlanes) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), ksptr += 3*VTraits::vlanes(), rsptr += 3*VTraits::vlanes()) { v_float32 kb, kg, kr, rb, rg, rr; v_load_deinterleave(ksptr, kb, kg, kr); v_load_deinterleave(rsptr, rb, rg, rr); - v_float32 knan = v_not_nan(kb) & v_not_nan(kg) & v_not_nan(kr); - v_float32 alpha = ((v_absdiff(kb, rb) + v_absdiff(kg, rg) + v_absdiff(kr, rr)) * sindex) & v_not_nan(rb) & v_not_nan(rg) & v_not_nan(rr) & knan; + v_float32 knan = v_and(v_and(v_not_nan(kb), v_not_nan(kg)), v_not_nan(kr)); + v_float32 alpha = v_and(v_and(v_and(v_and(v_mul(v_add(v_add(v_absdiff(kb, rb), v_absdiff(kg, rg)), v_absdiff(kr, rr)), sindex), v_not_nan(rb)), v_not_nan(rg)), v_not_nan(rr)), knan); v_int32 idx = v_trunc(alpha); - alpha -= v_cvt_f32(idx); + alpha = v_sub(alpha, v_cvt_f32(idx)); - v_float32 w = (kweight * v_muladd(v_lut(expLUT + 1, idx), alpha, v_lut(expLUT, idx) * (v_one - alpha))) & knan; - v_store_aligned(wsum + j, vx_load_aligned(wsum + j) + w); - v_store_aligned(sum_b + j, v_muladd(kb & knan, w, vx_load_aligned(sum_b + j))); - v_store_aligned(sum_g + j, v_muladd(kg & knan, w, vx_load_aligned(sum_g + j))); - v_store_aligned(sum_r + j, v_muladd(kr & knan, w, vx_load_aligned(sum_r + j))); + v_float32 w = v_and(v_mul(kweight, v_muladd(v_lut(this->expLUT + 1, idx), alpha, v_mul(v_lut(this->expLUT, idx), v_sub(v_one, alpha)))), knan); + v_store_aligned(wsum + j, v_add(vx_load_aligned(wsum + j), w)); + v_store_aligned(sum_b + j, v_muladd(v_and(kb, knan), w, vx_load_aligned(sum_b + j))); + v_store_aligned(sum_g + j, v_muladd(v_and(kg, knan), w, vx_load_aligned(sum_g + j))); + v_store_aligned(sum_r + j, v_muladd(v_and(kr, knan), w, vx_load_aligned(sum_r + j))); } #endif for (; j < size.width; j++, ksptr += 3, rsptr += 3) @@ -978,14 +978,14 @@ public: } } j = 0; -#if CV_SIMD - for (; j <= size.width - v_float32::nlanes; j += v_float32::nlanes, sptr += 3*v_float32::nlanes, dptr += 3*v_float32::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes(), sptr += 3*VTraits::vlanes(), dptr += 3*VTraits::vlanes()) { v_float32 b, g, r; v_load_deinterleave(sptr, b, g, r); - v_float32 mask = v_not_nan(b) & v_not_nan(g) & v_not_nan(r); - v_float32 w = v_one / (vx_load_aligned(wsum + j) + (v_one & mask)); - v_store_interleave(dptr, (vx_load_aligned(sum_b + j) + (b & mask)) * w, (vx_load_aligned(sum_g + j) + (g & mask)) * w, (vx_load_aligned(sum_r + j) + (r & mask)) * w); + v_float32 mask = v_and(v_and(v_not_nan(b), v_not_nan(g)), v_not_nan(r)); + v_float32 w = v_div(v_one, v_add(vx_load_aligned(wsum + j), v_and(v_one, mask))); + v_store_interleave(dptr, v_mul(v_add(vx_load_aligned(sum_b + j), v_and(b, mask)), w), v_mul(v_add(vx_load_aligned(sum_g + j), v_and(g, mask)), w), v_mul(v_add(vx_load_aligned(sum_r + j), v_and(r, mask)), w)); } #endif for (; j < size.width; j++) @@ -1011,7 +1011,7 @@ public: } } } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } diff --git a/modules/imgproc/src/blend.cpp b/modules/imgproc/src/blend.cpp index 5a1296b509..accb45e7ad 100644 --- a/modules/imgproc/src/blend.cpp +++ b/modules/imgproc/src/blend.cpp @@ -48,12 +48,12 @@ #include "opencv2/core/hal/intrin.hpp" namespace cv { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline v_float32 blend(const v_float32& v_src1, const v_float32& v_src2, const v_float32& v_w1, const v_float32& v_w2) { const v_float32 v_eps = vx_setall_f32(1e-5f); - v_float32 v_denom = v_w1 + v_w2 + v_eps; - return (v_src1 * v_w1 + v_src2 * v_w2) / v_denom; + v_float32 v_denom = v_add(v_add(v_w1, v_w2), v_eps); + return v_div(v_add(v_mul(v_src1, v_w1), v_mul(v_src2, v_w2)), v_denom); } static inline v_float32 blend(const v_float32& v_src1, const v_float32& v_src2, const float* w_ptr1, const float* w_ptr2, int offset) { @@ -105,7 +105,7 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, switch(cn) { case 1: - for(int weight_offset = 0 ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes, weight_offset += v_uint8::nlanes) + for(int weight_offset = 0 ; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src12, v_src13; v_float32 v_src20, v_src21, v_src22, v_src23; @@ -113,15 +113,15 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, load_expand_u8tof32(src2 + x, v_src20, v_src21, v_src22, v_src23); v_float32 v_dst0 = blend(v_src10, v_src20, weights1, weights2, weight_offset); - v_float32 v_dst1 = blend(v_src11, v_src21, weights1, weights2, weight_offset + v_float32::nlanes); - v_float32 v_dst2 = blend(v_src12, v_src22, weights1, weights2, weight_offset + 2*v_float32::nlanes); - v_float32 v_dst3 = blend(v_src13, v_src23, weights1, weights2, weight_offset + 3*v_float32::nlanes); + v_float32 v_dst1 = blend(v_src11, v_src21, weights1, weights2, weight_offset + VTraits::vlanes()); + v_float32 v_dst2 = blend(v_src12, v_src22, weights1, weights2, weight_offset + 2*VTraits::vlanes()); + v_float32 v_dst3 = blend(v_src13, v_src23, weights1, weights2, weight_offset + 3*VTraits::vlanes()); store_pack_f32tou8(dst + x, v_dst0, v_dst1, v_dst2, v_dst3); } break; case 2: - for(int weight_offset = 0 ; x <= width - 2*v_uint8::nlanes; x += 2*v_uint8::nlanes, weight_offset += v_uint8::nlanes) + for(int weight_offset = 0 ; x <= width - 2*VTraits::vlanes(); x += 2*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_uint8 v_src10, v_src11, v_src20, v_src21; v_load_deinterleave(src1 + x, v_src10, v_src11); @@ -135,12 +135,12 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, v_float32 v_dst0 = blend(v_src100, v_src200, weights1, weights2, weight_offset); v_float32 v_dst1 = blend(v_src110, v_src210, weights1, weights2, weight_offset); - v_float32 v_dst2 = blend(v_src101, v_src201, weights1, weights2, weight_offset + v_float32::nlanes); - v_float32 v_dst3 = blend(v_src111, v_src211, weights1, weights2, weight_offset + v_float32::nlanes); - v_float32 v_dst4 = blend(v_src102, v_src202, weights1, weights2, weight_offset + 2*v_float32::nlanes); - v_float32 v_dst5 = blend(v_src112, v_src212, weights1, weights2, weight_offset + 2*v_float32::nlanes); - v_float32 v_dst6 = blend(v_src103, v_src203, weights1, weights2, weight_offset + 3*v_float32::nlanes); - v_float32 v_dst7 = blend(v_src113, v_src213, weights1, weights2, weight_offset + 3*v_float32::nlanes); + v_float32 v_dst2 = blend(v_src101, v_src201, weights1, weights2, weight_offset + VTraits::vlanes()); + v_float32 v_dst3 = blend(v_src111, v_src211, weights1, weights2, weight_offset + VTraits::vlanes()); + v_float32 v_dst4 = blend(v_src102, v_src202, weights1, weights2, weight_offset + 2*VTraits::vlanes()); + v_float32 v_dst5 = blend(v_src112, v_src212, weights1, weights2, weight_offset + 2*VTraits::vlanes()); + v_float32 v_dst6 = blend(v_src103, v_src203, weights1, weights2, weight_offset + 3*VTraits::vlanes()); + v_float32 v_dst7 = blend(v_src113, v_src213, weights1, weights2, weight_offset + 3*VTraits::vlanes()); v_uint8 v_dsta = pack_f32tou8(v_dst0, v_dst2, v_dst4, v_dst6); v_uint8 v_dstb = pack_f32tou8(v_dst1, v_dst3, v_dst5, v_dst7); @@ -148,7 +148,7 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, } break; case 3: - for(int weight_offset = 0 ; x <= width - 3*v_uint8::nlanes; x += 3*v_uint8::nlanes, weight_offset += v_uint8::nlanes) + for(int weight_offset = 0 ; x <= width - 3*VTraits::vlanes(); x += 3*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_uint8 v_src10, v_src11, v_src12, v_src20, v_src21, v_src22; v_load_deinterleave(src1 + x, v_src10, v_src11, v_src12); @@ -164,13 +164,13 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, expand_u8tof32(v_src22, v_src220, v_src221, v_src222, v_src223); v_float32 v_w10 = vx_load(weights1 + weight_offset); - v_float32 v_w11 = vx_load(weights1 + weight_offset + v_float32::nlanes); - v_float32 v_w12 = vx_load(weights1 + weight_offset + 2*v_float32::nlanes); - v_float32 v_w13 = vx_load(weights1 + weight_offset + 3*v_float32::nlanes); + v_float32 v_w11 = vx_load(weights1 + weight_offset + VTraits::vlanes()); + v_float32 v_w12 = vx_load(weights1 + weight_offset + 2*VTraits::vlanes()); + v_float32 v_w13 = vx_load(weights1 + weight_offset + 3*VTraits::vlanes()); v_float32 v_w20 = vx_load(weights2 + weight_offset); - v_float32 v_w21 = vx_load(weights2 + weight_offset + v_float32::nlanes); - v_float32 v_w22 = vx_load(weights2 + weight_offset + 2*v_float32::nlanes); - v_float32 v_w23 = vx_load(weights2 + weight_offset + 3*v_float32::nlanes); + v_float32 v_w21 = vx_load(weights2 + weight_offset + VTraits::vlanes()); + v_float32 v_w22 = vx_load(weights2 + weight_offset + 2*VTraits::vlanes()); + v_float32 v_w23 = vx_load(weights2 + weight_offset + 3*VTraits::vlanes()); v_src100 = blend(v_src100, v_src200, v_w10, v_w20); v_src110 = blend(v_src110, v_src210, v_w10, v_w20); v_src120 = blend(v_src120, v_src220, v_w10, v_w20); @@ -192,7 +192,7 @@ int blendLinearSimd(const uchar* src1, const uchar* src2, const float* weights1, } break; case 4: - for(int weight_offset = 0 ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src12, v_src13; v_float32 v_src20, v_src21, v_src22, v_src23; @@ -229,7 +229,7 @@ int blendLinearSimd(const float* src1, const float* src2, const float* weights1, switch(cn) { case 1: - for(int weight_offset = 0 ; x <= width - v_float32::nlanes; x += v_float32::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src1 = vx_load(src1 + x); v_float32 v_src2 = vx_load(src2 + x); @@ -242,7 +242,7 @@ int blendLinearSimd(const float* src1, const float* src2, const float* weights1, } break; case 2: - for(int weight_offset = 0 ; x <= width - 2*v_float32::nlanes; x += 2*v_float32::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - 2*VTraits::vlanes(); x += 2*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src20, v_src21; v_load_deinterleave(src1 + x, v_src10, v_src11); @@ -257,7 +257,7 @@ int blendLinearSimd(const float* src1, const float* src2, const float* weights1, } break; case 3: - for(int weight_offset = 0 ; x <= width - 3*v_float32::nlanes; x += 3*v_float32::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - 3*VTraits::vlanes(); x += 3*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src12, v_src20, v_src21, v_src22; v_load_deinterleave(src1 + x, v_src10, v_src11, v_src12); @@ -273,7 +273,7 @@ int blendLinearSimd(const float* src1, const float* src2, const float* weights1, } break; case 4: - for(int weight_offset = 0 ; x <= width - 4*v_float32::nlanes; x += 4*v_float32::nlanes, weight_offset += v_float32::nlanes) + for(int weight_offset = 0 ; x <= width - 4*VTraits::vlanes(); x += 4*VTraits::vlanes(), weight_offset += VTraits::vlanes()) { v_float32 v_src10, v_src11, v_src12, v_src13, v_src20, v_src21, v_src22, v_src23; v_load_deinterleave(src1 + x, v_src10, v_src11, v_src12, v_src13); @@ -320,7 +320,7 @@ public: T * const dst_row = dst->ptr(y); int x = 0; - #if CV_SIMD + #if (CV_SIMD || CV_SIMD_SCALABLE) x = blendLinearSimd(src1_row, src2_row, weights1_row, weights2_row, dst_row, x, width, cn); #endif diff --git a/modules/imgproc/src/box_filter.simd.hpp b/modules/imgproc/src/box_filter.simd.hpp index 4a4d205216..735935c04f 100644 --- a/modules/imgproc/src/box_filter.simd.hpp +++ b/modules/imgproc/src/box_filter.simd.hpp @@ -309,15 +309,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; int i = 0; -#if CV_SIMD - for (; i <= width - v_int32::nlanes; i += v_int32::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -339,37 +339,37 @@ struct ColumnSum : if( haveScale ) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for( ; i <= width - v_uint16::nlanes; i += v_uint16::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); - v_uint32 v_s0d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s0) * _v_scale)); - v_uint32 v_s01d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s01) * _v_scale)); + v_uint32 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), _v_scale))); + v_uint32 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), _v_scale))); v_uint16 v_dst = v_pack(v_s0d, v_s01d); v_pack_store(D + i, v_dst); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); - v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s0) * v_scale)); - v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s01) * v_scale)); + v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale))); + v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale))); v_uint16x8 v_dst = v_pack(v_s0d, v_s01d); v_pack_store(D + i, v_dst); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -383,29 +383,29 @@ struct ColumnSum : else { int i = 0; -#if CV_SIMD - for( ; i <= width-v_uint16::nlanes; i+=v_uint16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); v_uint16 v_dst = v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01)); v_pack_store(D + i, v_dst); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); v_uint16x8 v_dst = v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01)); v_pack_store(D + i, v_dst); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -480,15 +480,15 @@ public BaseColumnFilter { const ushort* Sp = (const ushort*)src[0]; int i = 0; -#if CV_SIMD - for( ; i <= width - v_uint16::nlanes; i += v_uint16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width - v_uint16x8::nlanes; i += v_uint16x8::nlanes ) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -510,27 +510,27 @@ public BaseColumnFilter if( haveScale ) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_uint32 _ds4 = vx_setall_u32((unsigned)ds); v_uint16 _dd8 = vx_setall_u16((ushort)dd); - for( ; i <= width-v_uint8::nlanes; i+=v_uint8::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { v_uint16 _sm0 = vx_load(Sm + i); - v_uint16 _sm1 = vx_load(Sm + i + v_uint16::nlanes); + v_uint16 _sm1 = vx_load(Sm + i + VTraits::vlanes()); v_uint16 _s0 = v_add_wrap(vx_load(SUM + i), vx_load(Sp + i)); - v_uint16 _s1 = v_add_wrap(vx_load(SUM + i + v_uint16::nlanes), vx_load(Sp + i + v_uint16::nlanes)); + v_uint16 _s1 = v_add_wrap(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); v_uint32 _s00, _s01, _s10, _s11; - v_expand(_s0 + _dd8, _s00, _s01); - v_expand(_s1 + _dd8, _s10, _s11); + v_expand(v_add(_s0, _dd8), _s00, _s01); + v_expand(v_add(_s1, _dd8), _s10, _s11); - _s00 = v_shr(_s00*_ds4); - _s01 = v_shr(_s01*_ds4); - _s10 = v_shr(_s10*_ds4); - _s11 = v_shr(_s11*_ds4); + _s00 = v_shr(v_mul(_s00, _ds4)); + _s01 = v_shr(v_mul(_s01, _ds4)); + _s10 = v_shr(v_mul(_s10, _ds4)); + _s11 = v_shr(v_mul(_s11, _ds4)); v_int16 r0 = v_pack(v_reinterpret_as_s32(_s00), v_reinterpret_as_s32(_s01)); v_int16 r1 = v_pack(v_reinterpret_as_s32(_s10), v_reinterpret_as_s32(_s11)); @@ -540,9 +540,9 @@ public BaseColumnFilter v_store(D + i, v_pack_u(r0, r1)); v_store(SUM + i, _s0); - v_store(SUM + i + v_uint16::nlanes, _s1); + v_store(SUM + i + VTraits::vlanes(), _s1); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_uint32x4 ds4 = v_setall_u32((unsigned)ds); v_uint16x8 dd8 = v_setall_u16((ushort)dd); @@ -556,13 +556,13 @@ public BaseColumnFilter v_uint32x4 _s00, _s01, _s10, _s11; - v_expand(_s0 + dd8, _s00, _s01); - v_expand(_s1 + dd8, _s10, _s11); + v_expand(v_add(_s0, dd8), _s00, _s01); + v_expand(v_add(_s1, dd8), _s10, _s11); - _s00 = v_shr(_s00*ds4); - _s01 = v_shr(_s01*ds4); - _s10 = v_shr(_s10*ds4); - _s11 = v_shr(_s11*ds4); + _s00 = v_shr(v_mul(_s00, ds4)); + _s01 = v_shr(v_mul(_s01, ds4)); + _s10 = v_shr(v_mul(_s10, ds4)); + _s11 = v_shr(v_mul(_s11, ds4)); v_int16x8 r0 = v_pack(v_reinterpret_as_s32(_s00), v_reinterpret_as_s32(_s01)); v_int16x8 r1 = v_pack(v_reinterpret_as_s32(_s10), v_reinterpret_as_s32(_s11)); @@ -643,15 +643,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; i = 0; -#if CV_SIMD - for( ; i <= width - v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -673,33 +673,33 @@ struct ColumnSum : if( haveScale ) { i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for( ; i <= width-v_int16::nlanes; i+=v_int16::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); - v_int32 v_s0d = v_round(v_cvt_f32(v_s0) * _v_scale); - v_int32 v_s01d = v_round(v_cvt_f32(v_s01) * _v_scale); + v_int32 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), _v_scale)); + v_int32 v_s01d = v_round(v_mul(v_cvt_f32(v_s01), _v_scale)); v_store(D + i, v_pack(v_s0d, v_s01d)); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); - v_int32x4 v_s0d = v_round(v_cvt_f32(v_s0) * v_scale); - v_int32x4 v_s01d = v_round(v_cvt_f32(v_s01) * v_scale); + v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale)); + v_int32x4 v_s01d = v_round(v_mul(v_cvt_f32(v_s01), v_scale)); v_store(D + i, v_pack(v_s0d, v_s01d)); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -713,27 +713,27 @@ struct ColumnSum : else { i = 0; -#if CV_SIMD - for( ; i <= width-v_int16::nlanes; i+=v_int16::nlanes ) +#if CV_SIMD // TODO: enable for CV_SIMD_SCALABLE, GCC 13 related + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); v_store(D + i, v_pack(v_s0, v_s01)); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_int16x8::nlanes; i+=v_int16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); v_store(D + i, v_pack(v_s0, v_s01)); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -792,15 +792,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; int i = 0; -#if CV_SIMD - for (; i <= width - v_int32::nlanes; i += v_int32::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -822,33 +822,33 @@ struct ColumnSum : if( haveScale ) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for( ; i <= width-v_uint16::nlanes; i+=v_uint16::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); - v_uint32 v_s0d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s0) * _v_scale)); - v_uint32 v_s01d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s01) * _v_scale)); + v_uint32 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), _v_scale))); + v_uint32 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), _v_scale))); v_store(D + i, v_pack(v_s0d, v_s01d)); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); - v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s0) * v_scale)); - v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_cvt_f32(v_s01) * v_scale)); + v_uint32x4 v_s0d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s0), v_scale))); + v_uint32x4 v_s01d = v_reinterpret_as_u32(v_round(v_mul(v_cvt_f32(v_s01), v_scale))); v_store(D + i, v_pack(v_s0d, v_s01d)); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -862,27 +862,27 @@ struct ColumnSum : else { int i = 0; -#if CV_SIMD - for( ; i <= width-v_uint16::nlanes; i+=v_uint16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s01 = vx_load(SUM + i + v_int32::nlanes) + vx_load(Sp + i + v_int32::nlanes); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s01 = v_add(vx_load(SUM + i + VTraits::vlanes()), vx_load(Sp + i + VTraits::vlanes())); v_store(D + i, v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01))); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); - v_store(SUM + i + v_int32::nlanes, v_s01 - vx_load(Sm + i + v_int32::nlanes)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); + v_store(SUM + i + VTraits::vlanes(), v_sub(v_s01, vx_load(Sm + i + VTraits::vlanes()))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_uint16x8::nlanes; i+=v_uint16x8::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s01 = v_load(SUM + i + v_int32x4::nlanes) + v_load(Sp + i + v_int32x4::nlanes); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s01 = v_add(v_load(SUM + i + v_int32x4::nlanes), v_load(Sp + i + v_int32x4::nlanes)); v_store(D + i, v_pack(v_reinterpret_as_u32(v_s0), v_reinterpret_as_u32(v_s01))); - v_store(SUM + i, v_s0 - v_load(Sm + i)); - v_store(SUM + i + v_int32x4::nlanes, v_s01 - v_load(Sm + i + v_int32x4::nlanes)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); + v_store(SUM + i + v_int32x4::nlanes, v_sub(v_s01, v_load(Sm + i + v_int32x4::nlanes))); } #endif #endif @@ -939,15 +939,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; int i = 0; -#if CV_SIMD - for( ; i <= width - v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -969,25 +969,25 @@ struct ColumnSum : if( haveScale ) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for( ; i <= width-v_int32::nlanes; i+=v_int32::nlanes ) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_int32 v_s0d = v_round(v_cvt_f32(v_s0) * _v_scale); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_int32 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), _v_scale)); v_store(D + i, v_s0d); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_int32x4 v_s0d = v_round(v_cvt_f32(v_s0) * v_scale); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_int32x4 v_s0d = v_round(v_mul(v_cvt_f32(v_s0), v_scale)); v_store(D + i, v_s0d); - v_store(SUM + i, v_s0 - v_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); } #endif #endif @@ -1001,21 +1001,21 @@ struct ColumnSum : else { int i = 0; -#if CV_SIMD - for( ; i <= width-v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); v_store(D + i, v_s0); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); v_store(D + i, v_s0); - v_store(SUM + i, v_s0 - v_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); } #endif #endif @@ -1073,15 +1073,15 @@ struct ColumnSum : { const int* Sp = (const int*)src[0]; int i = 0; -#if CV_SIMD - for( ; i <= width - v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width - VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_store(SUM + i, vx_load(SUM + i) + vx_load(Sp + i)); + v_store(SUM + i, v_add(vx_load(SUM + i), vx_load(Sp + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width - v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_store(SUM + i, v_load(SUM + i) + v_load(Sp + i)); + v_store(SUM + i, v_add(v_load(SUM + i), v_load(Sp + i))); } #endif #endif @@ -1105,21 +1105,21 @@ struct ColumnSum : { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 _v_scale = vx_setall_f32((float)_scale); - for (; i <= width - v_int32::nlanes; i += v_int32::nlanes) + for (; i <= width - VTraits::vlanes(); i += VTraits::vlanes()) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); - v_store(D + i, v_cvt_f32(v_s0) * _v_scale); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); + v_store(D + i, v_mul(v_cvt_f32(v_s0), _v_scale)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 v_float32x4 v_scale = v_setall_f32((float)_scale); for (; i <= width - v_int32x4::nlanes; i += v_int32x4::nlanes) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); - v_store(D + i, v_cvt_f32(v_s0) * v_scale); - v_store(SUM + i, v_s0 - v_load(Sm + i)); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); + v_store(D + i, v_mul(v_cvt_f32(v_s0), v_scale)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); } #endif #endif @@ -1134,19 +1134,19 @@ struct ColumnSum : { int i = 0; -#if CV_SIMD - for( ; i <= width-v_int32::nlanes; i+=v_int32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; i <= width-VTraits::vlanes(); i+=VTraits::vlanes() ) { - v_int32 v_s0 = vx_load(SUM + i) + vx_load(Sp + i); + v_int32 v_s0 = v_add(vx_load(SUM + i), vx_load(Sp + i)); v_store(D + i, v_cvt_f32(v_s0)); - v_store(SUM + i, v_s0 - vx_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, vx_load(Sm + i))); } -#if CV_SIMD_WIDTH > 16 +#if !CV_SIMD_SCALABLE && CV_SIMD_WIDTH > 16 for( ; i <= width-v_int32x4::nlanes; i+=v_int32x4::nlanes ) { - v_int32x4 v_s0 = v_load(SUM + i) + v_load(Sp + i); + v_int32x4 v_s0 = v_add(v_load(SUM + i), v_load(Sp + i)); v_store(D + i, v_cvt_f32(v_s0)); - v_store(SUM + i, v_s0 - v_load(Sm + i)); + v_store(SUM + i, v_sub(v_s0, v_load(Sm + i))); } #endif #endif diff --git a/modules/imgproc/src/canny.cpp b/modules/imgproc/src/canny.cpp index dbd7dc04a7..a4c16c6b8c 100644 --- a/modules/imgproc/src/canny.cpp +++ b/modules/imgproc/src/canny.cpp @@ -306,11 +306,11 @@ public: src(_src), src2(_src), map(_map), _borderPeaksParallel(borderPeaksParallel), low(_low), high(_high), aperture_size(_aperture_size), L2gradient(_L2gradient) { -#if CV_SIMD - for(int i = 0; i < v_int8::nlanes; ++i) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for(int i = 0; i < VTraits::vlanes(); ++i) { smask[i] = 0; - smask[i + v_int8::nlanes] = (schar)-1; + smask[i + VTraits::vlanes()] = (schar)-1; } if (true) _map.create(src.rows + 2, (int)alignSize((size_t)(src.cols + CV_SIMD_WIDTH + 1), CV_SIMD_WIDTH), CV_8UC1); @@ -330,11 +330,11 @@ public: src(_dx), src2(_dy), map(_map), _borderPeaksParallel(borderPeaksParallel), low(_low), high(_high), aperture_size(0), L2gradient(_L2gradient) { -#if CV_SIMD - for(int i = 0; i < v_int8::nlanes; ++i) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for(int i = 0; i < VTraits::vlanes(); ++i) { smask[i] = 0; - smask[i + v_int8::nlanes] = (schar)-1; + smask[i + VTraits::vlanes()] = (schar)-1; } if (true) _map.create(src.rows + 2, (int)alignSize((size_t)(src.cols + CV_SIMD_WIDTH + 1), CV_SIMD_WIDTH), CV_8UC1); @@ -396,7 +396,7 @@ public: } // _mag_p: previous row, _mag_a: actual row, _mag_n: next row -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) AutoBuffer buffer(3 * (mapstep * cn + CV_SIMD_WIDTH)); _mag_p = alignPtr(buffer.data() + 1, CV_SIMD_WIDTH); _mag_a = alignPtr(_mag_p + mapstep * cn, CV_SIMD_WIDTH); @@ -436,8 +436,8 @@ public: if (L2gradient) { int j = 0, width = src.cols * cn; -#if CV_SIMD - for ( ; j <= width - v_int16::nlanes; j += v_int16::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for ( ; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) { v_int16 v_dx = vx_load((const short*)(_dx + j)); v_int16 v_dy = vx_load((const short*)(_dy + j)); @@ -447,8 +447,8 @@ public: v_expand(v_dx, v_dxp_low, v_dxp_high); v_expand(v_dy, v_dyp_low, v_dyp_high); - v_store_aligned((int *)(_mag_n + j), v_dxp_low*v_dxp_low+v_dyp_low*v_dyp_low); - v_store_aligned((int *)(_mag_n + j + v_int32::nlanes), v_dxp_high*v_dxp_high+v_dyp_high*v_dyp_high); + v_store_aligned((int *)(_mag_n + j), v_add(v_mul(v_dxp_low, v_dxp_low), v_mul(v_dyp_low, v_dyp_low))); + v_store_aligned((int *)(_mag_n + j + VTraits::vlanes()), v_add(v_mul(v_dxp_high, v_dxp_high), v_mul(v_dyp_high, v_dyp_high))); } #endif for ( ; j < width; ++j) @@ -457,8 +457,8 @@ public: else { int j = 0, width = src.cols * cn; -#if CV_SIMD - for(; j <= width - v_int16::nlanes; j += v_int16::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for(; j <= width - VTraits::vlanes(); j += VTraits::vlanes()) { v_int16 v_dx = vx_load((const short *)(_dx + j)); v_int16 v_dy = vx_load((const short *)(_dy + j)); @@ -470,8 +470,8 @@ public: v_expand(v_dx, v_dx_ml, v_dx_mh); v_expand(v_dy, v_dy_ml, v_dy_mh); - v_store_aligned((int *)(_mag_n + j), v_dx_ml + v_dy_ml); - v_store_aligned((int *)(_mag_n + j + v_int32::nlanes), v_dx_mh + v_dy_mh); + v_store_aligned((int *)(_mag_n + j), v_add(v_dx_ml, v_dy_ml)); + v_store_aligned((int *)(_mag_n + j + VTraits::vlanes()), v_add(v_dx_mh, v_dy_mh)); } #endif for ( ; j < width; ++j) @@ -515,7 +515,7 @@ public: // From here actual src row is (i - 1) // Set left and right border to 1 -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (true) _pmap = map.ptr(i) + CV_SIMD_WIDTH; else @@ -537,22 +537,22 @@ public: const int TG22 = 13573; int j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { const v_int32 v_low = vx_setall_s32(low); const v_int8 v_one = vx_setall_s8(1); - for (; j <= src.cols - v_int8::nlanes; j += v_int8::nlanes) + for (; j <= src.cols - VTraits::vlanes(); j += VTraits::vlanes()) { v_store_aligned((signed char*)(_pmap + j), v_one); - v_int8 v_cmp = v_pack(v_pack(vx_load_aligned((const int*)(_mag_a + j )) > v_low, - vx_load_aligned((const int*)(_mag_a + j + v_int32::nlanes)) > v_low), - v_pack(vx_load_aligned((const int*)(_mag_a + j + 2*v_int32::nlanes)) > v_low, - vx_load_aligned((const int*)(_mag_a + j + 3*v_int32::nlanes)) > v_low)); + v_int8 v_cmp = v_pack(v_pack(v_gt(vx_load_aligned((const int *)(_mag_a + j)), v_low), + v_gt(vx_load_aligned((const int *)(_mag_a + j + VTraits::vlanes())), v_low)), + v_pack(v_gt(vx_load_aligned((const int *)(_mag_a + j + 2 * VTraits::vlanes())), v_low), + v_gt(vx_load_aligned((const int *)(_mag_a + j + 3 * VTraits::vlanes())), v_low))); while (v_check_any(v_cmp)) { int l = v_scan_forward(v_cmp); - v_cmp &= vx_load(smask + v_int8::nlanes - 1 - l); + v_cmp = v_and(v_cmp, vx_load(smask + VTraits::vlanes() - 1 - l)); int k = j + l; int m = _mag_a[k]; @@ -693,8 +693,8 @@ private: ptrdiff_t mapstep; int cn; mutable Mutex mutex; -#if CV_SIMD - schar smask[2*v_int8::nlanes]; +#if (CV_SIMD || CV_SIMD_SCALABLE) + schar smask[2*VTraits::max_nlanes]; #endif }; @@ -718,31 +718,31 @@ public: int j = 0; uchar *pdst = dst.ptr(i); const uchar *pmap = map.ptr(i + 1); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (true) pmap += CV_SIMD_WIDTH; else #endif pmap += 1; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { const v_uint8 v_zero = vx_setzero_u8(); - const v_uint8 v_ff = ~v_zero; + const v_uint8 v_ff = v_not(v_zero); const v_uint8 v_two = vx_setall_u8(2); - for (; j <= dst.cols - v_uint8::nlanes; j += v_uint8::nlanes) + for (; j <= dst.cols - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v_pmap = vx_load_aligned((const unsigned char*)(pmap + j)); - v_pmap = v_select(v_pmap == v_two, v_ff, v_zero); + v_pmap = v_select(v_eq(v_pmap, v_two), v_ff, v_zero); v_store((pdst + j), v_pmap); } - if (j <= dst.cols - v_uint8::nlanes/2) + if (j <= dst.cols - VTraits::vlanes()/2) { v_uint8 v_pmap = vx_load_low((const unsigned char*)(pmap + j)); - v_pmap = v_select(v_pmap == v_two, v_ff, v_zero); + v_pmap = v_select(v_eq(v_pmap, v_two), v_ff, v_zero); v_store_low((pdst + j), v_pmap); - j += v_uint8::nlanes/2; + j += VTraits::vlanes()/2; } } #endif diff --git a/modules/imgproc/src/color_lab.cpp b/modules/imgproc/src/color_lab.cpp index 3b18944a0c..d111efdc47 100644 --- a/modules/imgproc/src/color_lab.cpp +++ b/modules/imgproc/src/color_lab.cpp @@ -56,40 +56,38 @@ template static inline _Tp splineInterpolate(_Tp x, const _Tp* tab return ((tab[3]*x + tab[2])*x + tab[1])*x + tab[0]; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template static inline cv::v_float32 splineInterpolate(const cv::v_float32& x, const _Tp* tab, int n) { using namespace cv; v_int32 ix = v_min(v_max(v_trunc(x), vx_setzero_s32()), vx_setall_s32(n-1)); - cv::v_float32 xx = x - v_cvt_f32(ix); - ix = ix << 2; + cv::v_float32 xx = v_sub(x, v_cvt_f32(ix)); + ix = v_shl<2>(ix); - v_float32 t[4]; + v_float32 t0, t1, t2, t3; // assume that v_float32::nlanes == v_int32::nlanes - if(v_float32::nlanes == 4) + if(VTraits::vlanes() == 4) { -#if CV_SIMD_WIDTH == 16 int32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) idx[4]; v_store_aligned(idx, ix); - v_float32x4 tt[4]; - tt[0] = v_load(tab + idx[0]); - tt[1] = v_load(tab + idx[1]); - tt[2] = v_load(tab + idx[2]); - tt[3] = v_load(tab + idx[3]); - v_transpose4x4(tt[0], tt[1], tt[2], tt[3], - t[0], t[1], t[2], t[3]); -#endif + v_float32 tt0, tt1, tt2, tt3; + tt0 = vx_load(tab + idx[0]); + tt1 = vx_load(tab + idx[1]); + tt2 = vx_load(tab + idx[2]); + tt3 = vx_load(tab + idx[3]); + v_transpose4x4(tt0, tt1, tt2, tt3, + t0, t1, t2, t3); } else { - t[0] = v_lut(tab + 0, ix); - t[1] = v_lut(tab + 1, ix); - t[2] = v_lut(tab + 2, ix); - t[3] = v_lut(tab + 3, ix); + t0 = v_lut(tab + 0, ix); + t1 = v_lut(tab + 1, ix); + t2 = v_lut(tab + 2, ix); + t3 = v_lut(tab + 3, ix); } - return v_fma(v_fma(v_fma(t[3], xx, t[2]), xx, t[1]), xx, t[0]); + return v_fma(v_fma(v_fma(t3, xx, t2), xx, t1), xx, t0); } #endif @@ -207,8 +205,8 @@ struct RGB2XYZ_f C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; int i = 0; -#if CV_SIMD - const int vsize = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2); v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4), vc5 = vx_setall_f32(C5); v_float32 vc6 = vx_setall_f32(C6), vc7 = vx_setall_f32(C7), vc8 = vx_setall_f32(C8); @@ -226,9 +224,9 @@ struct RGB2XYZ_f } v_float32 x, y, z; - x = v_fma(b, vc0, v_fma(g, vc1, r*vc2)); - y = v_fma(b, vc3, v_fma(g, vc4, r*vc5)); - z = v_fma(b, vc6, v_fma(g, vc7, r*vc8)); + x = v_fma(b, vc0, v_fma(g, vc1, v_mul(r, vc2))); + y = v_fma(b, vc3, v_fma(g, vc4, v_mul(r, vc5))); + z = v_fma(b, vc6, v_fma(g, vc7, v_mul(r, vc8))); v_store_interleave(dst, x, y, z); } @@ -313,8 +311,8 @@ struct RGB2XYZ_i C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); int descaleShift = 1 << (shift-1); v_int16 vdescale = vx_setall_s16((short)descaleShift); v_int16 cxbg, cxr1, cybg, cyr1, czbg, czr1; @@ -349,27 +347,36 @@ struct RGB2XYZ_i sg0 = v_reinterpret_as_s16(g0); sg1 = v_reinterpret_as_s16(g1); sb0 = v_reinterpret_as_s16(b0); sb1 = v_reinterpret_as_s16(b1); - v_int16 bg[4], rd[4]; - v_zip(sb0, sg0, bg[0], bg[1]); - v_zip(sb1, sg1, bg[2], bg[3]); - v_zip(sr0, vdescale, rd[0], rd[1]); - v_zip(sr1, vdescale, rd[2], rd[3]); + v_int16 bg0, bg1, bg2, bg3, rd0, rd1, rd2, rd3; + v_zip(sb0, sg0, bg0, bg1); + v_zip(sb1, sg1, bg2, bg3); + v_zip(sr0, vdescale, rd0, rd1); + v_zip(sr1, vdescale, rd2, rd3); - v_uint32 vx[4], vy[4], vz[4]; - for(int j = 0; j < 4; j++) - { - vx[j] = v_reinterpret_as_u32(v_dotprod(bg[j], cxbg) + v_dotprod(rd[j], cxr1)) >> shift; - vy[j] = v_reinterpret_as_u32(v_dotprod(bg[j], cybg) + v_dotprod(rd[j], cyr1)) >> shift; - vz[j] = v_reinterpret_as_u32(v_dotprod(bg[j], czbg) + v_dotprod(rd[j], czr1)) >> shift; - } + v_uint32 vx0, vx1, vx2, vx3; + v_uint32 vy0, vy1, vy2, vy3; + v_uint32 vz0, vz1, vz2, vz3; + + vx0 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg0, cxbg), v_dotprod(rd0, cxr1)))); + vy0 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg0, cybg), v_dotprod(rd0, cyr1)))); + vz0 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg0, czbg), v_dotprod(rd0, czr1)))); + vx1 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg1, cxbg), v_dotprod(rd1, cxr1)))); + vy1 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg1, cybg), v_dotprod(rd1, cyr1)))); + vz1 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg1, czbg), v_dotprod(rd1, czr1)))); + vx2 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg2, cxbg), v_dotprod(rd2, cxr1)))); + vy2 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg2, cybg), v_dotprod(rd2, cyr1)))); + vz2 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg2, czbg), v_dotprod(rd2, czr1)))); + vx3 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg3, cxbg), v_dotprod(rd3, cxr1)))); + vy3 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg3, cybg), v_dotprod(rd3, cyr1)))); + vz3 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg3, czbg), v_dotprod(rd3, czr1)))); v_uint16 x0, x1, y0, y1, z0, z1; - x0 = v_pack(vx[0], vx[1]); - x1 = v_pack(vx[2], vx[3]); - y0 = v_pack(vy[0], vy[1]); - y1 = v_pack(vy[2], vy[3]); - z0 = v_pack(vz[0], vz[1]); - z1 = v_pack(vz[2], vz[3]); + x0 = v_pack(vx0, vx1); + x1 = v_pack(vx2, vx3); + y0 = v_pack(vy0, vy1); + y1 = v_pack(vy2, vy3); + z0 = v_pack(vz0, vz1); + z1 = v_pack(vz2, vz3); v_uint8 x, y, z; x = v_pack(x0, x1); @@ -424,8 +431,8 @@ struct RGB2XYZ_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_int16 vdescale = vx_setall_s16(descaleShift); v_int16 vc0 = vx_setall_s16((short)C0), vc1 = vx_setall_s16((short)C1), vc2 = vx_setall_s16((short)C2); @@ -464,29 +471,29 @@ struct RGB2XYZ_i v_int16 ymr, ymg, ymb; v_int16 zmr, zmg, zmb; - v_int16 mr = sr < zero, mg = sg < zero, mb = sb < zero; + v_int16 mr = v_lt(sr, zero), mg = v_lt(sg, zero), mb = v_lt(sb, zero); - xmb = mb & vc0; - xmg = mg & vc1; - xmr = mr & vc2; - ymb = mb & vc3; - ymg = mg & vc4; - ymr = mr & vc5; - zmb = mb & vc6; - zmg = mg & vc7; - zmr = mr & vc8; + xmb = v_and(mb, vc0); + xmg = v_and(mg, vc1); + xmr = v_and(mr, vc2); + ymb = v_and(mb, vc3); + ymg = v_and(mg, vc4); + ymr = v_and(mr, vc5); + zmb = v_and(mb, vc6); + zmg = v_and(mg, vc7); + zmr = v_and(mr, vc8); v_int32 xfix0, xfix1, yfix0, yfix1, zfix0, zfix1; - v_expand(xmr + xmg + xmb, xfix0, xfix1); - v_expand(ymr + ymg + ymb, yfix0, yfix1); - v_expand(zmr + zmg + zmb, zfix0, zfix1); + v_expand(v_add(v_add(xmr, xmg), xmb), xfix0, xfix1); + v_expand(v_add(v_add(ymr, ymg), ymb), yfix0, yfix1); + v_expand(v_add(v_add(zmr, zmg), zmb), zfix0, zfix1); - xfix0 = xfix0 << 16; - xfix1 = xfix1 << 16; - yfix0 = yfix0 << 16; - yfix1 = yfix1 << 16; - zfix0 = zfix0 << 16; - zfix1 = zfix1 << 16; + xfix0 = v_shl<16>(xfix0); + xfix1 = v_shl<16>(xfix1); + yfix0 = v_shl<16>(yfix0); + yfix1 = v_shl<16>(yfix1); + zfix0 = v_shl<16>(zfix0); + zfix1 = v_shl<16>(zfix1); v_int16 bg0, bg1, rd0, rd1; v_zip(sb, sg, bg0, bg1); @@ -494,12 +501,12 @@ struct RGB2XYZ_i v_uint32 x0, x1, y0, y1, z0, z1; - x0 = v_reinterpret_as_u32(v_dotprod(bg0, cxbg) + v_dotprod(rd0, cxr1) + xfix0) >> shift; - x1 = v_reinterpret_as_u32(v_dotprod(bg1, cxbg) + v_dotprod(rd1, cxr1) + xfix1) >> shift; - y0 = v_reinterpret_as_u32(v_dotprod(bg0, cybg) + v_dotprod(rd0, cyr1) + yfix0) >> shift; - y1 = v_reinterpret_as_u32(v_dotprod(bg1, cybg) + v_dotprod(rd1, cyr1) + yfix1) >> shift; - z0 = v_reinterpret_as_u32(v_dotprod(bg0, czbg) + v_dotprod(rd0, czr1) + zfix0) >> shift; - z1 = v_reinterpret_as_u32(v_dotprod(bg1, czbg) + v_dotprod(rd1, czr1) + zfix1) >> shift; + x0 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg0, cxbg), v_dotprod(rd0, cxr1)), xfix0))); + x1 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg1, cxbg), v_dotprod(rd1, cxr1)), xfix1))); + y0 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg0, cybg), v_dotprod(rd0, cyr1)), yfix0))); + y1 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg1, cybg), v_dotprod(rd1, cyr1)), yfix1))); + z0 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg0, czbg), v_dotprod(rd0, czr1)), zfix0))); + z1 = v_shr(v_reinterpret_as_u32(v_add(v_add(v_dotprod(bg1, czbg), v_dotprod(rd1, czr1)), zfix1))); v_uint16 x, y, z; x = v_pack(x0, x1); @@ -593,8 +600,8 @@ struct XYZ2RGB_f C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; int i = 0; -#if CV_SIMD - const int vsize = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_float32 valpha = vx_setall_f32(alpha); v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2); v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4), vc5 = vx_setall_f32(C5); @@ -606,9 +613,9 @@ struct XYZ2RGB_f v_load_deinterleave(src, x, y, z); v_float32 b, g, r; - b = v_fma(x, vc0, v_fma(y, vc1, z*vc2)); - g = v_fma(x, vc3, v_fma(y, vc4, z*vc5)); - r = v_fma(x, vc6, v_fma(y, vc7, z*vc8)); + b = v_fma(x, vc0, v_fma(y, vc1, v_mul(z, vc2))); + g = v_fma(x, vc3, v_fma(y, vc4, v_mul(z, vc5))); + r = v_fma(x, vc6, v_fma(y, vc7, v_mul(z, vc8))); if(dcn == 4) { @@ -707,8 +714,8 @@ struct XYZ2RGB_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift - 1); v_uint8 valpha = vx_setall_u8(alpha); v_int16 vdescale = vx_setall_s16(descaleShift); @@ -739,25 +746,35 @@ struct XYZ2RGB_i z0 = v_reinterpret_as_s16(uz0); z1 = v_reinterpret_as_s16(uz1); - v_int32 b[4], g[4], r[4]; + v_int32 bb0, bb1, bb2, bb3, + gg0, gg1, gg2, gg3, + rr0, rr1, rr2, rr3; - v_int16 xy[4], zd[4]; - v_zip(x0, y0, xy[0], xy[1]); - v_zip(x1, y1, xy[2], xy[3]); - v_zip(z0, vdescale, zd[0], zd[1]); - v_zip(z1, vdescale, zd[2], zd[3]); + v_int16 xy0, xy1, xy2, xy3; + v_int16 zd0, zd1, zd2, zd3; - for(int j = 0; j < 4; j++) - { - b[j] = (v_dotprod(xy[j], cbxy) + v_dotprod(zd[j], cbz1)) >> shift; - g[j] = (v_dotprod(xy[j], cgxy) + v_dotprod(zd[j], cgz1)) >> shift; - r[j] = (v_dotprod(xy[j], crxy) + v_dotprod(zd[j], crz1)) >> shift; - } + v_zip(x0, y0, xy0, xy1); + v_zip(x1, y1, xy2, xy3); + v_zip(z0, vdescale, zd0, zd1); + v_zip(z1, vdescale, zd2, zd3); + + bb0 = v_shr(v_add(v_dotprod(xy0, cbxy), v_dotprod(zd0, cbz1))); + gg0 = v_shr(v_add(v_dotprod(xy0, cgxy), v_dotprod(zd0, cgz1))); + rr0 = v_shr(v_add(v_dotprod(xy0, crxy), v_dotprod(zd0, crz1))); + bb1 = v_shr(v_add(v_dotprod(xy1, cbxy), v_dotprod(zd1, cbz1))); + gg1 = v_shr(v_add(v_dotprod(xy1, cgxy), v_dotprod(zd1, cgz1))); + rr1 = v_shr(v_add(v_dotprod(xy1, crxy), v_dotprod(zd1, crz1))); + bb2 = v_shr(v_add(v_dotprod(xy2, cbxy), v_dotprod(zd2, cbz1))); + gg2 = v_shr(v_add(v_dotprod(xy2, cgxy), v_dotprod(zd2, cgz1))); + rr2 = v_shr(v_add(v_dotprod(xy2, crxy), v_dotprod(zd2, crz1))); + bb3 = v_shr(v_add(v_dotprod(xy3, cbxy), v_dotprod(zd3, cbz1))); + gg3 = v_shr(v_add(v_dotprod(xy3, cgxy), v_dotprod(zd3, cgz1))); + rr3 = v_shr(v_add(v_dotprod(xy3, crxy), v_dotprod(zd3, crz1))); v_uint16 b0, b1, g0, g1, r0, r1; - b0 = v_pack_u(b[0], b[1]); b1 = v_pack_u(b[2], b[3]); - g0 = v_pack_u(g[0], g[1]); g1 = v_pack_u(g[2], g[3]); - r0 = v_pack_u(r[0], r[1]); r1 = v_pack_u(r[2], r[3]); + b0 = v_pack_u(bb0, bb1); b1 = v_pack_u(bb2, bb3); + g0 = v_pack_u(gg0, gg1); g1 = v_pack_u(gg2, gg3); + r0 = v_pack_u(rr0, rr1); r1 = v_pack_u(rr2, rr3); v_uint8 bb, gg, rr; bb = v_pack(b0, b1); @@ -820,8 +837,8 @@ struct XYZ2RGB_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4], C5 = coeffs[5], C6 = coeffs[6], C7 = coeffs[7], C8 = coeffs[8]; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_uint16 valpha = vx_setall_u16(alpha); v_int16 vdescale = vx_setall_s16(descaleShift); @@ -850,30 +867,30 @@ struct XYZ2RGB_i sz = v_reinterpret_as_s16(z); // fixing 16bit signed multiplication - v_int16 mx = sx < zero, my = sy < zero, mz = sz < zero; + v_int16 mx = v_lt(sx, zero), my = v_lt(sy, zero), mz = v_lt(sz, zero); v_int16 bmx, bmy, bmz; v_int16 gmx, gmy, gmz; v_int16 rmx, rmy, rmz; - bmx = mx & vc0; - bmy = my & vc1; - bmz = mz & vc2; - gmx = mx & vc3; - gmy = my & vc4; - gmz = mz & vc5; - rmx = mx & vc6; - rmy = my & vc7; - rmz = mz & vc8; + bmx = v_and(mx, vc0); + bmy = v_and(my, vc1); + bmz = v_and(mz, vc2); + gmx = v_and(mx, vc3); + gmy = v_and(my, vc4); + gmz = v_and(mz, vc5); + rmx = v_and(mx, vc6); + rmy = v_and(my, vc7); + rmz = v_and(mz, vc8); v_int32 bfix0, bfix1, gfix0, gfix1, rfix0, rfix1; - v_expand(bmx + bmy + bmz, bfix0, bfix1); - v_expand(gmx + gmy + gmz, gfix0, gfix1); - v_expand(rmx + rmy + rmz, rfix0, rfix1); + v_expand(v_add(v_add(bmx, bmy), bmz), bfix0, bfix1); + v_expand(v_add(v_add(gmx, gmy), gmz), gfix0, gfix1); + v_expand(v_add(v_add(rmx, rmy), rmz), rfix0, rfix1); - bfix0 = bfix0 << 16; bfix1 = bfix1 << 16; - gfix0 = gfix0 << 16; gfix1 = gfix1 << 16; - rfix0 = rfix0 << 16; rfix1 = rfix1 << 16; + bfix0 = v_shl<16>(bfix0); bfix1 = v_shl<16>(bfix1); + gfix0 = v_shl<16>(gfix0); gfix1 = v_shl<16>(gfix1); + rfix0 = v_shl<16>(rfix0); rfix1 = v_shl<16>(rfix1); v_int16 xy0, xy1, zd0, zd1; v_zip(sx, sy, xy0, xy1); @@ -881,12 +898,12 @@ struct XYZ2RGB_i v_int32 b0, b1, g0, g1, r0, r1; - b0 = (v_dotprod(xy0, cbxy) + v_dotprod(zd0, cbz1) + bfix0) >> shift; - b1 = (v_dotprod(xy1, cbxy) + v_dotprod(zd1, cbz1) + bfix1) >> shift; - g0 = (v_dotprod(xy0, cgxy) + v_dotprod(zd0, cgz1) + gfix0) >> shift; - g1 = (v_dotprod(xy1, cgxy) + v_dotprod(zd1, cgz1) + gfix1) >> shift; - r0 = (v_dotprod(xy0, crxy) + v_dotprod(zd0, crz1) + rfix0) >> shift; - r1 = (v_dotprod(xy1, crxy) + v_dotprod(zd1, crz1) + rfix1) >> shift; + b0 = v_shr(v_add(v_add(v_dotprod(xy0, cbxy), v_dotprod(zd0, cbz1)), bfix0)); + b1 = v_shr(v_add(v_add(v_dotprod(xy1, cbxy), v_dotprod(zd1, cbz1)), bfix1)); + g0 = v_shr(v_add(v_add(v_dotprod(xy0, cgxy), v_dotprod(zd0, cgz1)), gfix0)); + g1 = v_shr(v_add(v_add(v_dotprod(xy1, cgxy), v_dotprod(zd1, cgz1)), gfix1)); + r0 = v_shr(v_add(v_add(v_dotprod(xy0, crxy), v_dotprod(zd0, crz1)), rfix0)); + r1 = v_shr(v_add(v_add(v_dotprod(xy1, crxy), v_dotprod(zd1, crz1)), rfix1)); v_uint16 b, g, r; b = v_pack_u(b0, b1); g = v_pack_u(g0, g1); r = v_pack_u(r0, r1); @@ -1452,19 +1469,19 @@ static inline void trilinearPackedInterpolate(const v_uint16x8& inX, const v_uin #undef DOT_SHIFT_PACK } -#elif CV_SIMD +#elif CV_SIMD // Fixed size v_int16x8 used below, CV_SIMD_SCALABLE is disabled. // inValues are in [0; LAB_BASE] static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint16& inY, const v_uint16& inZ, const int16_t* LUT, v_uint16& outA, v_uint16& outB, v_uint16& outC) { - const int vsize = v_uint16::nlanes; + const int vsize = VTraits::max_nlanes; // LUT idx of origin pt of cube - v_uint16 tx = inX >> (lab_base_shift - lab_lut_shift); - v_uint16 ty = inY >> (lab_base_shift - lab_lut_shift); - v_uint16 tz = inZ >> (lab_base_shift - lab_lut_shift); + v_uint16 tx = v_shr(inX); + v_uint16 ty = v_shr(inY); + v_uint16 tz = v_shr(inZ); v_uint32 btmp00, btmp01, btmp10, btmp11, btmp20, btmp21; v_uint32 baseIdx0, baseIdx1; @@ -1472,8 +1489,8 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 v_mul_expand(tx, vx_setall_u16(3*8), btmp00, btmp01); v_mul_expand(ty, vx_setall_u16(3*8*LAB_LUT_DIM), btmp10, btmp11); v_mul_expand(tz, vx_setall_u16(3*8*LAB_LUT_DIM*LAB_LUT_DIM), btmp20, btmp21); - baseIdx0 = btmp00 + btmp10 + btmp20; - baseIdx1 = btmp01 + btmp11 + btmp21; + baseIdx0 = v_add(v_add(btmp00, btmp10), btmp20); + baseIdx1 = v_add(v_add(btmp01, btmp11), btmp21); uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vbaseIdx[vsize]; v_store_aligned(vbaseIdx + 0*vsize/2, baseIdx0); @@ -1482,9 +1499,9 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 // fracX, fracY, fracZ are [0; TRILINEAR_BASE) const uint16_t bitMask = (1 << trilinear_shift) - 1; v_uint16 bitMaskReg = vx_setall_u16(bitMask); - v_uint16 fracX = (inX >> (lab_base_shift - 8 - 1)) & bitMaskReg; - v_uint16 fracY = (inY >> (lab_base_shift - 8 - 1)) & bitMaskReg; - v_uint16 fracZ = (inZ >> (lab_base_shift - 8 - 1)) & bitMaskReg; + v_uint16 fracX = v_and(v_shr(inX), bitMaskReg); + v_uint16 fracY = v_and(v_shr(inY), bitMaskReg); + v_uint16 fracZ = v_and(v_shr(inZ), bitMaskReg); // trilinearIdx = 8*x + 8*TRILINEAR_BASE*y + 8*TRILINEAR_BASE*TRILINEAR_BASE*z v_uint32 trilinearIdx0, trilinearIdx1; @@ -1493,8 +1510,8 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 v_expand(fracY, fracY0, fracY1); v_expand(fracZ, fracZ0, fracZ1); - trilinearIdx0 = (fracX0 << 3) + (fracY0 << (3+trilinear_shift)) + (fracZ0 << (3+trilinear_shift*2)); - trilinearIdx1 = (fracX1 << 3) + (fracY1 << (3+trilinear_shift)) + (fracZ1 << (3+trilinear_shift*2)); + trilinearIdx0 = v_add(v_add(v_shl<3>(fracX0), v_shl<3 + trilinear_shift>(fracY0)), v_shl<3 + trilinear_shift * 2>(fracZ0)); + trilinearIdx1 = v_add(v_add(v_shl<3>(fracX1), v_shl<3 + trilinear_shift>(fracY1)), v_shl<3 + trilinear_shift * 2>(fracZ1)); uint32_t CV_DECL_ALIGNED(CV_SIMD_WIDTH) vtrilinearIdx[vsize]; v_store_aligned(vtrilinearIdx + 0*vsize/2, trilinearIdx0); @@ -1528,12 +1545,12 @@ static inline void trilinearPackedInterpolate(const v_uint16& inX, const v_uint1 // CV_DESCALE const v_uint32 descaleShift = vx_setall_u32(1 << (trilinear_shift*3 - 1)); - a0 = (a0 + descaleShift) >> (trilinear_shift*3); - a1 = (a1 + descaleShift) >> (trilinear_shift*3); - b0 = (b0 + descaleShift) >> (trilinear_shift*3); - b1 = (b1 + descaleShift) >> (trilinear_shift*3); - c0 = (c0 + descaleShift) >> (trilinear_shift*3); - c1 = (c1 + descaleShift) >> (trilinear_shift*3); + a0 = v_shr(v_add(a0, descaleShift)); + a1 = v_shr(v_add(a1, descaleShift)); + b0 = v_shr(v_add(b0, descaleShift)); + b1 = v_shr(v_add(b1, descaleShift)); + c0 = v_shr(v_add(c0, descaleShift)); + c1 = v_shr(v_add(c1, descaleShift)); outA = v_pack(a0, a1); outB = v_pack(b0, b1); outC = v_pack(c0, c1); } diff --git a/modules/imgproc/src/color_rgb.simd.hpp b/modules/imgproc/src/color_rgb.simd.hpp index 6e11020197..67e2febd5b 100644 --- a/modules/imgproc/src/color_rgb.simd.hpp +++ b/modules/imgproc/src/color_rgb.simd.hpp @@ -122,8 +122,8 @@ struct RGB2RGB int i = 0; _Tp alphav = ColorChannel<_Tp>::max(); -#if CV_SIMD - const int vsize = vt::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); for(; i <= n-vsize; i += vsize, src += vsize*scn, dst += vsize*dcn) @@ -138,8 +138,13 @@ struct RGB2RGB v_load_deinterleave(src, a, b, c); d = v_set<_Tp>::set(alphav); } - if(bi == 2) + if(bi == 2) { + #if CV_SIMD_SCALABLE + auto t = a; a = c; c = t; // swap(a, c); + #else swap(a, c); + #endif + } if(dcn == 4) { @@ -185,53 +190,57 @@ struct RGB5x52RGB int dcn = dstcn, bidx = blueIdx, gb = greenBits; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 vz = vx_setzero_u8(), vn0 = vx_setall_u8(255); for(; i <= n-vsize; i += vsize, src += vsize*sizeof(ushort), dst += vsize*dcn) { v_uint16 t0 = v_reinterpret_as_u16(vx_load(src)); v_uint16 t1 = v_reinterpret_as_u16(vx_load(src + - sizeof(ushort)*v_uint16::nlanes)); + sizeof(ushort)*VTraits::vlanes())); //TODO: shorten registers use when v_interleave is available v_uint8 r, g, b, a; - v_uint16 b0 = (t0 << 11) >> 8; - v_uint16 b1 = (t1 << 11) >> 8; + v_uint16 b0 = v_shr<8>(v_shl<11>(t0)); + v_uint16 b1 = v_shr<8>(v_shl<11>(t1)); b = v_pack(b0, b1); v_uint16 g0, g1, r0, r1, a0, a1; if( gb == 6 ) { - g0 = ((t0 >> 5) << 10) >> 8; - g1 = ((t1 >> 5) << 10) >> 8; + g0 = v_shr<8>(v_shl<10>(v_shr<5>(t0))); + g1 = v_shr<8>(v_shl<10>(v_shr<5>(t1))); - r0 = (t0 >> 11) << 3; - r1 = (t1 >> 11) << 3; + r0 = v_shl<3>(v_shr<11>(t0)); + r1 = v_shl<3>(v_shr<11>(t1)); a = vn0; } else { - g0 = ((t0 >> 5) << 11) >> 8; - g1 = ((t1 >> 5) << 11) >> 8; + g0 = v_shr<8>(v_shl<11>(v_shr<5>(t0))); + g1 = v_shr<8>(v_shl<11>(v_shr<5>(t1))); - r0 = ((t0 >> 10) << 11) >> 8; - r1 = ((t1 >> 10) << 11) >> 8; + r0 = v_shr<8>(v_shl<11>(v_shr<10>(t0))); + r1 = v_shr<8>(v_shl<11>(v_shr<10>(t1))); - a0 = t0 >> 15; - a1 = t1 >> 15; + a0 = v_shr<15>(t0); + a1 = v_shr<15>(t1); a = v_pack(a0, a1); - a = a != vz; + a = v_ne(a, vz); } g = v_pack(g0, g1); r = v_pack(r0, r1); - if(bidx == 2) + if(bidx == 2) { + #if CV_SIMD_SCALABLE + auto t = r; r = b; b = t; // swap(b, r); + #else swap(b, r); - + #endif + } if(dcn == 4) { v_store_interleave(dst, b, g, r, a); @@ -289,8 +298,8 @@ struct RGB2RGB5x5 int scn = srccn, bidx = blueIdx, gb = greenBits; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint16 vn3 = vx_setall_u16((ushort)(~3)); v_uint16 vn7 = vx_setall_u16((ushort)(~7)); v_uint16 vz = vx_setzero_u16(); @@ -308,10 +317,15 @@ struct RGB2RGB5x5 { v_load_deinterleave(src, b, g, r, a); } - if(bidx == 2) + if(bidx == 2){ + #if CV_SIMD_SCALABLE + auto t = r; r = b; b = t; // swap(b, r); + #else swap(b, r); + #endif + } - r = r & v7; + r = v_and(r, v7); //TODO: shorten registers use when v_deinterleave is available v_uint16 r0, r1, g0, g1, b0, b1, a0, a1; @@ -322,20 +336,20 @@ struct RGB2RGB5x5 v_uint16 d0, d1; - b0 = b0 >> 3; - b1 = b1 >> 3; - a0 = (a0 != vz) << 15; - a1 = (a1 != vz) << 15; + b0 = v_shr<3>(b0); + b1 = v_shr<3>(b1); + a0 = v_shl<15>(v_ne(a0, vz)); + a1 = v_shl<15>(v_ne(a1, vz)); if(gb == 6) { - d0 = b0 | ((g0 & vn3) << 3) | (r0 << 8); - d1 = b1 | ((g1 & vn3) << 3) | (r1 << 8); + d0 = v_or(v_or(b0, v_shl<3>(v_and(g0, vn3))), v_shl<8>(r0)); + d1 = v_or(v_or(b1, v_shl<3>(v_and(g1, vn3))), v_shl<8>(r1)); } else { - d0 = b0 | ((g0 & vn7) << 2) | (r0 << 7) | a0; - d1 = b1 | ((g1 & vn7) << 2) | (r1 << 7) | a1; + d0 = v_or(v_or(v_or(b0, v_shl<2>(v_and(g0, vn7))), v_shl<7>(r0)), a0); + d1 = v_or(v_or(v_or(b1, v_shl<2>(v_and(g1, vn7))), v_shl<7>(r1)), a1); } v_store((ushort*)dst, d0); @@ -382,8 +396,8 @@ struct Gray2RGB int i = 0; _Tp alpha = ColorChannel<_Tp>::max(); -#if CV_SIMD - const int vsize = vt::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); vt valpha = v_set<_Tp>::set(alpha); for(; i <= n-vsize; i += vsize, src += vsize, dst += vsize*dcn) @@ -424,8 +438,8 @@ struct Gray2RGB5x5 { int gb = greenBits; int i = 0; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint16 v3 = vx_setall_u16((ushort)(~3)); for(; i <= n-vsize; i += vsize, src += vsize, dst += vsize*sizeof(ushort)) @@ -433,16 +447,16 @@ struct Gray2RGB5x5 v_uint8 t8 = vx_load_low(src); v_uint16 t = v_expand_low(t8); - v_uint16 t3 = t >> 3; + v_uint16 t3 = v_shr<3>(t); v_uint16 d = t3; if(gb == 6) { - d |= ((t & v3) << 3) | (t3 << 11); + d = v_or(d, v_or(v_shl<3>(v_and(t, v3)), v_shl<11>(t3))); } else { - d |= (t3 << 5) | (t3 << 10); + d = v_or(d, v_or(v_shl<5>(t3), v_shl<10>(t3))); } v_store((ushort*)dst, d); @@ -488,8 +502,8 @@ struct RGB5x52Gray { int gb = greenBits; int i = 0; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_int16 bg2y; v_int16 r12y; @@ -504,17 +518,17 @@ struct RGB5x52Gray v_uint16 t = vx_load((ushort*)src); v_uint16 r, g, b; - b = (t << 11) >> 8; + b = v_shr<8>(v_shl<11>(t)); if(gb == 5) { - g = ((t >> 5) << 11) >> 8; - r = ((t >> 10) << 11) >> 8; + g = v_shr<8>(v_shl<11>(v_shr<5>(t))); + r = v_shr<8>(v_shl<11>(v_shr<10>(t))); } else { - g = ((t >> 5) << 10) >> 8; - r = (t >> 11) << 3; + g = v_shr<8>(v_shl<10>(v_shr<5>(t))); + r = v_shl<3>(v_shr<11>(t)); } v_uint8 d; @@ -530,11 +544,11 @@ struct RGB5x52Gray v_zip(sr, delta, rd0, rd1); v_uint32 d0, d1; - d0 = v_reinterpret_as_u32(v_dotprod(bg0, bg2y) + v_dotprod(rd0, r12y)); - d1 = v_reinterpret_as_u32(v_dotprod(bg1, bg2y) + v_dotprod(rd1, r12y)); + d0 = v_reinterpret_as_u32(v_add(v_dotprod(bg0, bg2y), v_dotprod(rd0, r12y))); + d1 = v_reinterpret_as_u32(v_add(v_dotprod(bg1, bg2y), v_dotprod(rd1, r12y))); - d0 = d0 >> shift; - d1 = d1 >> shift; + d0 = v_shr(d0); + d1 = v_shr(d1); dx = v_pack(d0, d1); // high part isn't used @@ -611,8 +625,8 @@ struct RGB2Gray int scn = srccn, i = 0; float cb = coeffs[0], cg = coeffs[1], cr = coeffs[2]; -#if CV_SIMD - const int vsize = v_float32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_float32 rv = vx_setall_f32(cr), gv = vx_setall_f32(cg), bv = vx_setall_f32(cb); for(; i <= n-vsize; i += vsize, src += vsize*scn, dst += vsize) @@ -627,7 +641,7 @@ struct RGB2Gray v_load_deinterleave(src, b, g, r, a); } - v_float32 d = v_fma(r, rv, v_fma(g, gv, b*bv)); + v_float32 d = v_fma(r, rv, v_fma(g, gv, v_mul(b, bv))); v_store(dst, d); } @@ -669,8 +683,8 @@ struct RGB2Gray short cb = coeffs[0], cg = coeffs[1], cr = coeffs[2]; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_int16 bg2y; v_int16 r12y; v_int16 dummy; @@ -706,10 +720,10 @@ struct RGB2Gray v_zip(v_reinterpret_as_s16(r1), delta, rd10, rd11); v_uint32 y00, y01, y10, y11; - y00 = v_reinterpret_as_u32(v_dotprod(bg00, bg2y) + v_dotprod(rd00, r12y)) >> shift; - y01 = v_reinterpret_as_u32(v_dotprod(bg01, bg2y) + v_dotprod(rd01, r12y)) >> shift; - y10 = v_reinterpret_as_u32(v_dotprod(bg10, bg2y) + v_dotprod(rd10, r12y)) >> shift; - y11 = v_reinterpret_as_u32(v_dotprod(bg11, bg2y) + v_dotprod(rd11, r12y)) >> shift; + y00 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg00, bg2y), v_dotprod(rd00, r12y)))); + y01 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg01, bg2y), v_dotprod(rd01, r12y)))); + y10 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg10, bg2y), v_dotprod(rd10, r12y)))); + y11 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg11, bg2y), v_dotprod(rd11, r12y)))); v_uint16 y0, y1; y0 = v_pack(y00, y01); @@ -762,8 +776,8 @@ struct RGB2Gray short cb = coeffs[0], cg = coeffs[1], cr = coeffs[2]; int i = 0; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_int16 b2y = vx_setall_s16(cb); v_int16 g2y = vx_setall_s16(cg); @@ -802,13 +816,13 @@ struct RGB2Gray // fixing 16bit signed multiplication v_int16 mr, mg, mb; - mr = (sr < z) & r2y; - mg = (sg < z) & g2y; - mb = (sb < z) & b2y; - v_int16 fixmul = v_add_wrap(mr, v_add_wrap(mg, mb)) << fix_shift; + mr = v_and(v_lt(sr, z), r2y); + mg = v_and(v_lt(sg, z), g2y); + mb = v_and(v_lt(sb, z), b2y); + v_int16 fixmul = v_shl(v_add_wrap(mr, v_add_wrap(mg, mb))); - v_int32 sy0 = (v_dotprod(bg0, bg2y) + v_dotprod(rd0, r12y)) >> shift; - v_int32 sy1 = (v_dotprod(bg1, bg2y) + v_dotprod(rd1, r12y)) >> shift; + v_int32 sy0 = v_shr(v_add(v_dotprod(bg0, bg2y), v_dotprod(rd0, r12y))); + v_int32 sy1 = v_shr(v_add(v_dotprod(bg1, bg2y), v_dotprod(rd1, r12y))); v_int16 y = v_add_wrap(v_pack(sy0, sy1), fixmul); @@ -973,8 +987,8 @@ struct mRGBA2RGBA uchar max_val = ColorChannel::max(); int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 amask = v_reinterpret_as_u8(vx_setall_u32(0xFF000000)); v_uint8 vmax = vx_setall_u8(max_val); @@ -989,9 +1003,9 @@ struct mRGBA2RGBA v_uint8 a; v_uint16 a16; v_uint32 a32; - a16 = v_reinterpret_as_u16(s & amask); - a32 = v_reinterpret_as_u32(a16 | (a16 >> 8)); - a = v_reinterpret_as_u8(a32 | (a32 >> 16)); + a16 = v_reinterpret_as_u16(v_and(s, amask)); + a32 = v_reinterpret_as_u32(v_or(a16, v_shr<8>(a16))); + a = v_reinterpret_as_u8(v_or(a32, v_shr<16>(a32))); // s *= max_val v_uint16 s0, s1; @@ -1000,7 +1014,7 @@ struct mRGBA2RGBA // s += a/2 v_uint16 ae0, ae1; v_expand(a, ae0, ae1); - s0 += ae0 >> 1; s1 += ae1 >> 1; + s0 = v_add(s0, v_shr<1>(ae0)); s1 = v_add(s1, v_shr<1>(ae1)); // s, a -> u32 -> float v_uint32 u00, u01, u10, u11; @@ -1035,10 +1049,10 @@ struct mRGBA2RGBA // float d = (float)s/(float)a v_float32 fd00, fd01, fd10, fd11; - fd00 = fs00/fa00; - fd01 = fs01/fa01; - fd10 = fs10/fa10; - fd11 = fs11/fa11; + fd00 = v_div(fs00, fa00); + fd01 = v_div(fs01, fa01); + fd10 = v_div(fs10, fa10); + fd11 = v_div(fs11, fa11); // d -> u32 -> u8 v_uint32 ud00, ud01, ud10, ud11; @@ -1054,8 +1068,8 @@ struct mRGBA2RGBA // if a == 0 then d = 0 v_uint8 am; - am = a != vx_setzero_u8(); - d = d & am; + am = v_ne(a, vx_setzero_u8()); + d = v_and(d, am); // put alpha values d = v_select(amask, a, d); diff --git a/modules/imgproc/src/color_yuv.simd.hpp b/modules/imgproc/src/color_yuv.simd.hpp index b5f73d873a..580329f660 100644 --- a/modules/imgproc/src/color_yuv.simd.hpp +++ b/modules/imgproc/src/color_yuv.simd.hpp @@ -49,6 +49,15 @@ void cvtOnePlaneYUVtoBGR(const uchar * src_data, size_t src_step, namespace { //constants for conversion from/to RGB and YUV, YCrCb according to BT.601 +#if CV_SIMD_SCALABLE +template +static void swap(T&a, T&b) { + T t = a; + a = b; + b = t; +} +#endif + //to YCbCr static const float YCBF = 0.564f; // == 1/2/(1-B2YF) static const float YCRF = 0.713f; // == 1/2/(1-R2YF) @@ -143,11 +152,11 @@ struct RGB2YCrCb_f float C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4]; int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1), vc2 = vx_setall_f32(C2); v_float32 vc3 = vx_setall_f32(C3), vc4 = vx_setall_f32(C4); v_float32 vdelta = vx_setall_f32(delta); - const int vsize = v_float32::nlanes; + const int vsize = VTraits::vlanes(); for( ; i <= n-vsize; i += vsize, src += vsize*scn, dst += vsize*3) { @@ -162,13 +171,13 @@ struct RGB2YCrCb_f } v_float32 y, cr, cb; - y = v_fma(b, vc0, v_fma(g, vc1, r*vc2)); + y = v_fma(b, vc0, v_fma(g, vc1, v_mul(r, vc2))); if(bidx) - std::swap(r, b); + swap(r, b); - cr = v_fma(r - y, vc3, vdelta); - cb = v_fma(b - y, vc4, vdelta); + cr = v_fma(v_sub(r, y), vc3, vdelta); + cb = v_fma(v_sub(b, y), vc4, vdelta); if(yuvOrder) { @@ -266,8 +275,8 @@ struct RGB2YCrCb_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4]; int sdelta = ColorChannel::half()*(1 << shift); int i = 0; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descale = 1 << (shift-1); v_int16 b2y = vx_setall_s16((short)C0); @@ -312,13 +321,13 @@ struct RGB2YCrCb_i // fixing 16bit signed multiplication v_int16 mr, mg, mb; - mr = (sr < z) & r2y; - mg = (sg < z) & g2y; - mb = (sb < z) & b2y; - v_int16 fixmul = v_add_wrap(mr, v_add_wrap(mg, mb)) << fix_shift; + mr = v_and(v_lt(sr, z), r2y); + mg = v_and(v_lt(sg, z), g2y); + mb = v_and(v_lt(sb, z), b2y); + v_int16 fixmul = v_shl(v_add_wrap(mr, v_add_wrap(mg, mb)), fix_shift); - v_int32 ssy0 = (v_dotprod(bg0, bg2y) + v_dotprod(rd0, r12y)) >> shift; - v_int32 ssy1 = (v_dotprod(bg1, bg2y) + v_dotprod(rd1, r12y)) >> shift; + v_int32 ssy0 = v_shr(v_add(v_dotprod(bg0, bg2y), v_dotprod(rd0, r12y)), shift); + v_int32 ssy1 = v_shr(v_add(v_dotprod(bg1, bg2y), v_dotprod(rd1, r12y)), shift); y = v_reinterpret_as_u16(v_add_wrap(v_pack(ssy0, ssy1), fixmul)); @@ -340,15 +349,15 @@ struct RGB2YCrCb_i v_int32 sy0 = v_reinterpret_as_s32(uy0); v_int32 sy1 = v_reinterpret_as_s32(uy1); - sr0 = sr0 - sy0; sr1 = sr1 - sy1; - sb0 = sb0 - sy0; sb1 = sb1 - sy1; + sr0 = v_sub(sr0, sy0); sr1 = v_sub(sr1, sy1); + sb0 = v_sub(sb0, sy0); sb1 = v_sub(sb1, sy1); v_int32 v_scr0, v_scr1, v_scb0, v_scb1; - v_scr0 = (sr0*vc3 + vdd) >> shift; - v_scr1 = (sr1*vc3 + vdd) >> shift; - v_scb0 = (sb0*vc4 + vdd) >> shift; - v_scb1 = (sb1*vc4 + vdd) >> shift; + v_scr0 = v_shr(v_add(v_mul(sr0, vc3), vdd), shift); + v_scr1 = v_shr(v_add(v_mul(sr1, vc3), vdd), shift); + v_scb0 = v_shr(v_add(v_mul(sb0, vc4), vdd), shift); + v_scb1 = v_shr(v_add(v_mul(sb1, vc4), vdd), shift); // saturate and pack cr = v_pack_u(v_scr0, v_scr1); @@ -407,8 +416,8 @@ struct RGB2YCrCb_i int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3], C4 = coeffs[4]; int delta = ColorChannel::half()*(1 << shift); -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_int16 bg2y; v_int16 r12y; @@ -458,10 +467,10 @@ struct RGB2YCrCb_i v_zip(sr0, vdescale, rd00, rd01); v_zip(sr1, vdescale, rd10, rd11); - y00 = v_reinterpret_as_u32(v_dotprod(bg00, bg2y) + v_dotprod(rd00, r12y)) >> shift; - y01 = v_reinterpret_as_u32(v_dotprod(bg01, bg2y) + v_dotprod(rd01, r12y)) >> shift; - y10 = v_reinterpret_as_u32(v_dotprod(bg10, bg2y) + v_dotprod(rd10, r12y)) >> shift; - y11 = v_reinterpret_as_u32(v_dotprod(bg11, bg2y) + v_dotprod(rd11, r12y)) >> shift; + y00 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg00, bg2y), v_dotprod(rd00, r12y))), shift); + y01 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg01, bg2y), v_dotprod(rd01, r12y))), shift); + y10 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg10, bg2y), v_dotprod(rd10, r12y))), shift); + y11 = v_shr(v_reinterpret_as_u32(v_add(v_dotprod(bg11, bg2y), v_dotprod(rd11, r12y))), shift); } v_uint16 y0, y1; @@ -512,15 +521,15 @@ struct RGB2YCrCb_i v_uint8 cr, cb; - cr00 = cr00 >> shift; - cr01 = cr01 >> shift; - cr10 = cr10 >> shift; - cr11 = cr11 >> shift; + cr00 = v_shr(cr00, shift); + cr01 = v_shr(cr01, shift); + cr10 = v_shr(cr10, shift); + cr11 = v_shr(cr11, shift); - cb00 = cb00 >> shift; - cb01 = cb01 >> shift; - cb10 = cb10 >> shift; - cb11 = cb11 >> shift; + cb00 = v_shr(cb00, shift); + cb01 = v_shr(cb01, shift); + cb10 = v_shr(cb10, shift); + cb11 = v_shr(cb11, shift); v_int16 cr0, cr1, cb0, cb1; cr0 = v_pack(cr00, cr01); cr1 = v_pack(cr10, cr11); @@ -623,12 +632,12 @@ struct YCrCb2RGB_f float C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3]; int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_float32 vc0 = vx_setall_f32(C0), vc1 = vx_setall_f32(C1); v_float32 vc2 = vx_setall_f32(C2), vc3 = vx_setall_f32(C3); v_float32 vdelta = vx_setall_f32(delta); v_float32 valpha = vx_setall_f32(alpha); - const int vsize = v_float32::nlanes; + const int vsize = VTraits::vlanes(); for( ; i <= n-vsize; i += vsize, src += vsize*3, dst += vsize*dcn) { @@ -640,7 +649,7 @@ struct YCrCb2RGB_f v_float32 b, g, r; - cb -= vdelta; cr -= vdelta; + cb = v_sub(cb, vdelta); cr = v_sub(cr, vdelta); b = v_fma(cb, vc3, y); g = v_fma(cr, vc1, v_fma(cb, vc2, y)); r = v_fma(cr, vc0, y); @@ -746,8 +755,8 @@ struct YCrCb2RGB_i const uchar delta = ColorChannel::half(), alpha = ColorChannel::max(); int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3]; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 valpha = vx_setall_u8(alpha); v_uint8 vdelta = vx_setall_u8(delta); const int descaleShift = 1 << (shift - 1); @@ -794,8 +803,8 @@ struct YCrCb2RGB_i v_int32 cb00, cb01, cb10, cb11; v_expand(v_scb0, cb00, cb01); v_expand(v_scb1, cb10, cb11); - b00 += cb00 << 15; b01 += cb01 << 15; - b10 += cb10 << 15; b11 += cb11 << 15; + b00 = v_add(b00, v_shl<15>(cb00)); b01 = v_add(b01, v_shl<15>(cb01)); + b10 = v_add(b10, v_shl<15>(cb10)); b11 = v_add(b11, v_shl<15>(cb11)); } v_int32 t00, t01, t10, t11; @@ -803,17 +812,17 @@ struct YCrCb2RGB_i v_mul_expand(v_scb1, vc2, t10, t11); v_mul_expand(v_scr0, vc1, g00, g01); v_mul_expand(v_scr1, vc1, g10, g11); - g00 += t00; g01 += t01; - g10 += t10; g11 += t11; + g00 = v_add(g00, t00); g01 = v_add(g01, t01); + g10 = v_add(g10, t10); g11 = v_add(g11, t11); v_mul_expand(v_scr0, vc0, r00, r01); v_mul_expand(v_scr1, vc0, r10, r11); - b00 = (b00 + vdescale) >> shift; b01 = (b01 + vdescale) >> shift; - b10 = (b10 + vdescale) >> shift; b11 = (b11 + vdescale) >> shift; - g00 = (g00 + vdescale) >> shift; g01 = (g01 + vdescale) >> shift; - g10 = (g10 + vdescale) >> shift; g11 = (g11 + vdescale) >> shift; - r00 = (r00 + vdescale) >> shift; r01 = (r01 + vdescale) >> shift; - r10 = (r10 + vdescale) >> shift; r11 = (r11 + vdescale) >> shift; + b00 = v_shr(v_add(b00, vdescale), shift); b01 = v_shr(v_add(b01, vdescale), shift); + b10 = v_shr(v_add(b10, vdescale), shift); b11 = v_shr(v_add(b11, vdescale), shift); + g00 = v_shr(v_add(g00, vdescale), shift); g01 = v_shr(v_add(g01, vdescale), shift); + g10 = v_shr(v_add(g10, vdescale), shift); g11 = v_shr(v_add(g11, vdescale), shift); + r00 = v_shr(v_add(r00, vdescale), shift); r01 = v_shr(v_add(r01, vdescale), shift); + r10 = v_shr(v_add(r10, vdescale), shift); r11 = v_shr(v_add(r11, vdescale), shift); v_int16 b0, b1, g0, g1, r0, r1; b0 = v_pack(b00, b01); b1 = v_pack(b10, b11); @@ -897,8 +906,8 @@ struct YCrCb2RGB_i const ushort delta = ColorChannel::half(), alpha = ColorChannel::max(); int C0 = coeffs[0], C1 = coeffs[1], C2 = coeffs[2], C3 = coeffs[3]; -#if CV_SIMD - const int vsize = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); const int descaleShift = 1 << (shift-1); v_uint16 valpha = vx_setall_u16(alpha); v_uint16 vdelta = vx_setall_u16(delta); @@ -939,22 +948,22 @@ struct YCrCb2RGB_i // so we fix the multiplication v_int32 cb0, cb1; v_expand(scb, cb0, cb1); - b0 += cb0 << 15; - b1 += cb1 << 15; + b0 = v_add(b0, v_shl<15>(cb0)); + b1 = v_add(b1, v_shl<15>(cb1)); } v_int32 t0, t1; v_mul_expand(scb, vc2, t0, t1); v_mul_expand(scr, vc1, g0, g1); - g0 += t0; g1 += t1; + g0 = v_add(g0, t0); g1 = v_add(g1, t1); v_mul_expand(scr, vc0, r0, r1); // shifted term doesn't fit into 16 bits, addition is to be done in 32 bits - b0 = ((b0 + vdescale) >> shift) + y0; - b1 = ((b1 + vdescale) >> shift) + y1; - g0 = ((g0 + vdescale) >> shift) + y0; - g1 = ((g1 + vdescale) >> shift) + y1; - r0 = ((r0 + vdescale) >> shift) + y0; - r1 = ((r1 + vdescale) >> shift) + y1; + b0 = v_add(v_shr(v_add(b0, vdescale), shift), y0); + b1 = v_add(v_shr(v_add(b1, vdescale), shift), y1); + g0 = v_add(v_shr(v_add(g0, vdescale), shift), y0); + g1 = v_add(v_shr(v_add(g1, vdescale), shift), y1); + r0 = v_add(v_shr(v_add(r0, vdescale), shift), y0); + r1 = v_add(v_shr(v_add(r1, vdescale), shift), y1); // saturate and pack v_uint16 b, g, r; @@ -1038,11 +1047,11 @@ static inline void uvToRGBuv(const uchar u, const uchar v, int& ruv, int& guv, i buv = (1 << (ITUR_BT_601_SHIFT - 1)) + ITUR_BT_601_CUB * uu; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline void uvToRGBuv(const v_uint8& u, const v_uint8& v, - v_int32 (&ruv)[4], - v_int32 (&guv)[4], - v_int32 (&buv)[4]) + v_int32 &ruv0, v_int32 &ruv1, v_int32 &ruv2, v_int32 &ruv3, + v_int32 &guv0, v_int32 &guv1, v_int32 &guv2, v_int32 &guv3, + v_int32 &buv0, v_int32 &buv1, v_int32 &buv2, v_int32 &buv3) { v_uint8 v128 = vx_setall_u8(128); v_int8 su = v_reinterpret_as_s8(v_sub_wrap(u, v128)); @@ -1051,9 +1060,10 @@ static inline void uvToRGBuv(const v_uint8& u, const v_uint8& v, v_int16 uu0, uu1, vv0, vv1; v_expand(su, uu0, uu1); v_expand(sv, vv0, vv1); - v_int32 uu[4], vv[4]; - v_expand(uu0, uu[0], uu[1]); v_expand(uu1, uu[2], uu[3]); - v_expand(vv0, vv[0], vv[1]); v_expand(vv1, vv[2], vv[3]); + v_int32 uuu0, uuu1, uuu2, uuu3; + v_int32 vvv0, vvv1, vvv2, vvv3; + v_expand(uu0, uuu0, uuu1); v_expand(uu1, uuu2, uuu3); + v_expand(vv0, vvv0, vvv1); v_expand(vv1, vvv2, vvv3); v_int32 vshift = vx_setall_s32(1 << (ITUR_BT_601_SHIFT - 1)); v_int32 vr = vx_setall_s32(ITUR_BT_601_CVR); @@ -1061,12 +1071,15 @@ static inline void uvToRGBuv(const v_uint8& u, const v_uint8& v, v_int32 ug = vx_setall_s32(ITUR_BT_601_CUG); v_int32 ub = vx_setall_s32(ITUR_BT_601_CUB); - for (int k = 0; k < 4; k++) - { - ruv[k] = vshift + vr * vv[k]; - guv[k] = vshift + vg * vv[k] + ug * uu[k]; - buv[k] = vshift + ub * uu[k]; - } + auto process_uv = [&](v_int32& ruv, v_int32& guv, v_int32& buv, const v_int32& vv, const v_int32& uu) { + ruv = v_add(vshift, v_mul(vr, vv)); + guv = v_add(v_add(vshift, v_mul(vg, vv)), v_mul(ug, uu)); + buv = v_add(vshift, v_mul(ub, uu)); + }; + process_uv(ruv0, guv0, buv0, vvv0, uuu0); + process_uv(ruv1, guv1, buv1, vvv1, uuu1); + process_uv(ruv2, guv2, buv2, vvv2, uuu2); + process_uv(ruv3, guv3, buv3, vvv3, uuu3); } #endif @@ -1081,44 +1094,48 @@ static inline void yRGBuvToRGBA(const uchar vy, const int ruv, const int guv, co a = uchar(0xff); } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline void yRGBuvToRGBA(const v_uint8& vy, - const v_int32 (&ruv)[4], - const v_int32 (&guv)[4], - const v_int32 (&buv)[4], + const v_int32 &ruv0, const v_int32 &ruv1, const v_int32 &ruv2, const v_int32 &ruv3, + const v_int32 &guv0, const v_int32 &guv1, const v_int32 &guv2, const v_int32 &guv3, + const v_int32 &buv0, const v_int32 &buv1, const v_int32 &buv2, const v_int32 &buv3, v_uint8& rr, v_uint8& gg, v_uint8& bb) { v_uint8 v16 = vx_setall_u8(16); - v_uint8 posY = vy - v16; + v_uint8 posY = v_sub(vy, v16); v_uint16 yy0, yy1; v_expand(posY, yy0, yy1); - v_int32 yy[4]; - v_int32 yy00, yy01, yy10, yy11; - v_expand(v_reinterpret_as_s16(yy0), yy[0], yy[1]); - v_expand(v_reinterpret_as_s16(yy1), yy[2], yy[3]); + v_int32 yyy0, yyy1, yyy2, yyy3; + v_expand(v_reinterpret_as_s16(yy0), yyy0, yyy1); + v_expand(v_reinterpret_as_s16(yy1), yyy2, yyy3); v_int32 vcy = vx_setall_s32(ITUR_BT_601_CY); - v_int32 y[4], r[4], g[4], b[4]; - for(int k = 0; k < 4; k++) - { - y[k] = yy[k]*vcy; - r[k] = (y[k] + ruv[k]) >> ITUR_BT_601_SHIFT; - g[k] = (y[k] + guv[k]) >> ITUR_BT_601_SHIFT; - b[k] = (y[k] + buv[k]) >> ITUR_BT_601_SHIFT; - } + v_int32 y0, y1, y2, y3, r0, r1, r2, r3, g0, g1, g2, g3, b0, b1, b2, b3; - v_int16 r0, r1, g0, g1, b0, b1; - r0 = v_pack(r[0], r[1]); - r1 = v_pack(r[2], r[3]); - g0 = v_pack(g[0], g[1]); - g1 = v_pack(g[2], g[3]); - b0 = v_pack(b[0], b[1]); - b1 = v_pack(b[2], b[3]); + auto process_yrgb = [&](const v_int32& yy, v_int32& y, v_int32& r, v_int32& g, v_int32& b, + const v_int32& ruv, const v_int32& guv, const v_int32& buv) { + y = v_mul(yy, vcy); + r = v_shr(v_add(y, ruv), ITUR_BT_601_SHIFT); + g = v_shr(v_add(y, guv), ITUR_BT_601_SHIFT); + b = v_shr(v_add(y, buv), ITUR_BT_601_SHIFT); + }; + process_yrgb(yyy0, y0, r0, g0, b0, ruv0, guv0, buv0); + process_yrgb(yyy1, y1, r1, g1, b1, ruv1, guv1, buv1); + process_yrgb(yyy2, y2, r2, g2, b2, ruv2, guv2, buv2); + process_yrgb(yyy3, y3, r3, g3, b3, ruv3, guv3, buv3); - rr = v_pack_u(r0, r1); - gg = v_pack_u(g0, g1); - bb = v_pack_u(b0, b1); + v_int16 _r0, _r1, _g0, _g1, _b0, _b1; + _r0 = v_pack(r0, r1); + _r1 = v_pack(r2, r3); + _g0 = v_pack(g0, g1); + _g1 = v_pack(g2, g3); + _b0 = v_pack(b0, b1); + _b1 = v_pack(b2, b3); + + rr = v_pack_u(_r0, _r1); + gg = v_pack_u(_g0, _g1); + bb = v_pack_u(_b0, _b1); } #endif @@ -1201,8 +1218,8 @@ struct YUV420sp2RGB8Invoker : ParallelLoopBody const uchar* y2 = y1 + my1_step; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 a = vx_setall_u8(uchar(0xff)); for( ; i <= width - 2*vsize; i += 2*vsize, row1 += vsize*dcn*2, row2 += vsize*dcn*2) @@ -1215,36 +1232,50 @@ struct YUV420sp2RGB8Invoker : ParallelLoopBody swap(u, v); } - v_uint8 vy[4]; - v_load_deinterleave(y1 + i, vy[0], vy[1]); - v_load_deinterleave(y2 + i, vy[2], vy[3]); + v_uint8 vy0, vy1, vy2, vy3; + v_load_deinterleave(y1 + i, vy0, vy1); + v_load_deinterleave(y2 + i, vy2, vy3); - v_int32 ruv[4], guv[4], buv[4]; - uvToRGBuv(u, v, ruv, guv, buv); + v_int32 ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3; + uvToRGBuv(u, v, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3); - v_uint8 r[4], g[4], b[4]; + v_uint8 r0, r1, r2, r3, g0, g1, g2, g3, b0, b1, b2, b3; - for(int k = 0; k < 4; k++) - { - yRGBuvToRGBA(vy[k], ruv, guv, buv, r[k], g[k], b[k]); - } + auto call_yRGBuvToRGBA = [&](const v_uint8& vy, v_uint8& r, v_uint8& g, v_uint8& b) { + yRGBuvToRGBA(vy, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3, + r, g, b); + }; + call_yRGBuvToRGBA(vy0, r0, g0, b0); + call_yRGBuvToRGBA(vy1, r1, g1, b1); + call_yRGBuvToRGBA(vy2, r2, g2, b2); + call_yRGBuvToRGBA(vy3, r3, g3, b3); if(bIdx) { - for(int k = 0; k < 4; k++) - swap(r[k], b[k]); + swap(r0, b0); + swap(r1, b1); + swap(r2, b2); + swap(r3, b3); } // [r0...], [r1...] => [r0, r1, r0, r1...], [r0, r1, r0, r1...] v_uint8 r0_0, r0_1, r1_0, r1_1; - v_zip(r[0], r[1], r0_0, r0_1); - v_zip(r[2], r[3], r1_0, r1_1); + v_zip(r0, r1, r0_0, r0_1); + v_zip(r2, r3, r1_0, r1_1); v_uint8 g0_0, g0_1, g1_0, g1_1; - v_zip(g[0], g[1], g0_0, g0_1); - v_zip(g[2], g[3], g1_0, g1_1); + v_zip(g0, g1, g0_0, g0_1); + v_zip(g2, g3, g1_0, g1_1); v_uint8 b0_0, b0_1, b1_0, b1_1; - v_zip(b[0], b[1], b0_0, b0_1); - v_zip(b[2], b[3], b1_0, b1_1); + v_zip(b0, b1, b0_0, b0_1); + v_zip(b2, b3, b1_0, b1_1); if(dcn == 4) { @@ -1319,8 +1350,8 @@ struct YUV420p2RGB8Invoker : ParallelLoopBody const uchar* y2 = y1 + stride; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 a = vx_setall_u8(uchar(0xff)); for( ; i <= width/2 - vsize; i += vsize, row1 += vsize*dcn*2, row2 += vsize*dcn*2) @@ -1329,36 +1360,50 @@ struct YUV420p2RGB8Invoker : ParallelLoopBody u = vx_load(u1 + i); v = vx_load(v1 + i); - v_uint8 vy[4]; - v_load_deinterleave(y1 + 2*i, vy[0], vy[1]); - v_load_deinterleave(y2 + 2*i, vy[2], vy[3]); + v_uint8 vy0, vy1, vy2, vy3; + v_load_deinterleave(y1 + 2*i, vy0, vy1); + v_load_deinterleave(y2 + 2*i, vy2, vy3); - v_int32 ruv[4], guv[4], buv[4]; - uvToRGBuv(u, v, ruv, guv, buv); + v_int32 ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3; + uvToRGBuv(u, v, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3); - v_uint8 r[4], g[4], b[4]; + v_uint8 r0, r1, r2, r3, g0, g1, g2, g3, b0, b1, b2, b3; - for(int k = 0; k < 4; k++) - { - yRGBuvToRGBA(vy[k], ruv, guv, buv, r[k], g[k], b[k]); - } + auto call_yRGBuvToRGBA = [&](const v_uint8& vy, v_uint8& r, v_uint8& g, v_uint8& b) { + yRGBuvToRGBA(vy, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3, + r, g, b); + }; + call_yRGBuvToRGBA(vy0, r0, g0, b0); + call_yRGBuvToRGBA(vy1, r1, g1, b1); + call_yRGBuvToRGBA(vy2, r2, g2, b2); + call_yRGBuvToRGBA(vy3, r3, g3, b3); if(bIdx) { - for(int k = 0; k < 4; k++) - swap(r[k], b[k]); + swap(r0, b0); + swap(r1, b1); + swap(r2, b2); + swap(r3, b3); } // [r0...], [r1...] => [r0, r1, r0, r1...], [r0, r1, r0, r1...] v_uint8 r0_0, r0_1, r1_0, r1_1; - v_zip(r[0], r[1], r0_0, r0_1); - v_zip(r[2], r[3], r1_0, r1_1); + v_zip(r0, r1, r0_0, r0_1); + v_zip(r2, r3, r1_0, r1_1); v_uint8 g0_0, g0_1, g1_0, g1_1; - v_zip(g[0], g[1], g0_0, g0_1); - v_zip(g[2], g[3], g1_0, g1_1); + v_zip(g0, g1, g0_0, g0_1); + v_zip(g2, g3, g1_0, g1_1); v_uint8 b0_0, b0_1, b1_0, b1_1; - v_zip(b[0], b[1], b0_0, b0_1); - v_zip(b[2], b[3], b1_0, b1_1); + v_zip(b0, b1, b0_0, b0_1); + v_zip(b2, b3, b1_0, b1_1); if(dcn == 4) { @@ -1430,7 +1475,7 @@ static inline uchar rgbToY42x(uchar r, uchar g, uchar b) return saturate_cast(yy >> ITUR_BT_601_SHIFT); } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline v_uint8 rgbToY42x(const v_uint8& r, const v_uint8& g, const v_uint8& b) { const int shifted16 = (16 << ITUR_BT_601_SHIFT); @@ -1440,25 +1485,25 @@ static inline v_uint8 rgbToY42x(const v_uint8& r, const v_uint8& g, const v_uint v_expand(g, g0, g1); v_expand(b, b0, b1); - v_uint32 rq[4], gq[4], bq[4]; - v_expand(r0, rq[0], rq[1]); v_expand(r1, rq[2], rq[3]); - v_expand(g0, gq[0], gq[1]); v_expand(g1, gq[2], gq[3]); - v_expand(b0, bq[0], bq[1]); v_expand(b1, bq[2], bq[3]); + v_uint32 rq0, rq1, rq2, rq3, gq0, gq1, gq2, gq3, bq0, bq1, bq2, bq3; + v_expand(r0, rq0, rq1); v_expand(r1, rq2, rq3); + v_expand(g0, gq0, gq1); v_expand(g1, gq2, gq3); + v_expand(b0, bq0, bq1); v_expand(b1, bq2, bq3); v_uint32 ry = vx_setall_u32(ITUR_BT_601_CRY), gy = vx_setall_u32(ITUR_BT_601_CGY); v_uint32 by = vx_setall_u32(ITUR_BT_601_CBY), shift = vx_setall_u32(halfShift + shifted16); - v_uint32 y[4]; - for(int k = 0; k < 4; k++) - { - y[k] = (rq[k]*ry + gq[k]*gy + bq[k]*by + shift) >> ITUR_BT_601_SHIFT; - } + v_uint32 y0, y1, y2, y3; + y0 = v_shr(v_add(v_add(v_add(v_mul(rq0, ry), v_mul(gq0, gy)), v_mul(bq0, by)), shift)); + y1 = v_shr(v_add(v_add(v_add(v_mul(rq1, ry), v_mul(gq1, gy)), v_mul(bq1, by)), shift)); + y2 = v_shr(v_add(v_add(v_add(v_mul(rq2, ry), v_mul(gq2, gy)), v_mul(bq2, by)), shift)); + y3 = v_shr(v_add(v_add(v_add(v_mul(rq3, ry), v_mul(gq3, gy)), v_mul(bq3, by)), shift)); - v_uint16 y0, y1; - y0 = v_pack(y[0], y[1]); - y1 = v_pack(y[2], y[3]); + v_uint16 _y0, _y1; + _y0 = v_pack(y0, y1); + _y1 = v_pack(y2, y3); - return v_pack(y0, y1); + return v_pack(_y0, _y1); } #endif @@ -1473,27 +1518,27 @@ static inline void rgbToUV42x(uchar r, uchar g, uchar b, uchar& u, uchar& v) v = saturate_cast(vv >> ITUR_BT_601_SHIFT); } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) static inline void rgbToUV42x(const v_uint8& r0, const v_uint8& r1, const v_uint8& g0, const v_uint8& g1, const v_uint8& b0, const v_uint8& b1, v_uint8& u, v_uint8& v) { // [r0, r1, r2, r3,..] => [r0, 0, r2, 0,..] v_int16 vlowByte = vx_setall_s16(0x00ff); v_int16 rd0, rd1, gd0, gd1, bd0, bd1; - rd0 = v_reinterpret_as_s16(r0) & vlowByte; - rd1 = v_reinterpret_as_s16(r1) & vlowByte; - gd0 = v_reinterpret_as_s16(g0) & vlowByte; - gd1 = v_reinterpret_as_s16(g1) & vlowByte; - bd0 = v_reinterpret_as_s16(b0) & vlowByte; - bd1 = v_reinterpret_as_s16(b1) & vlowByte; + rd0 = v_and(v_reinterpret_as_s16(r0), vlowByte); + rd1 = v_and(v_reinterpret_as_s16(r1), vlowByte); + gd0 = v_and(v_reinterpret_as_s16(g0), vlowByte); + gd1 = v_and(v_reinterpret_as_s16(g1), vlowByte); + bd0 = v_and(v_reinterpret_as_s16(b0), vlowByte); + bd1 = v_and(v_reinterpret_as_s16(b1), vlowByte); - v_int32 rq[4], gq[4], bq[4]; - v_expand(rd0, rq[0], rq[1]); - v_expand(rd1, rq[2], rq[3]); - v_expand(gd0, gq[0], gq[1]); - v_expand(gd1, gq[2], gq[3]); - v_expand(bd0, bq[0], bq[1]); - v_expand(bd1, bq[2], bq[3]); + v_int32 rq0, rq1, rq2, rq3, gq0, gq1, gq2, gq3, bq0, bq1, bq2, bq3; + v_expand(rd0, rq0, rq1); + v_expand(rd1, rq2, rq3); + v_expand(gd0, gq0, gq1); + v_expand(gd1, gq2, gq3); + v_expand(bd0, bq0, bq1); + v_expand(bd1, bq2, bq3); const int halfShift = (1 << (ITUR_BT_601_SHIFT - 1)); const int shifted128 = (128 << ITUR_BT_601_SHIFT); @@ -1505,18 +1550,21 @@ static inline void rgbToUV42x(const v_uint8& r0, const v_uint8& r1, const v_uint bu = vx_setall_s32(ITUR_BT_601_CBU); bv = vx_setall_s32(ITUR_BT_601_CBV); - v_int32 uq[4], vq[4]; - for(int k = 0; k < 4; k++) - { - uq[k] = (ru*rq[k] + gu*gq[k] + bu*bq[k] + shift) >> ITUR_BT_601_SHIFT; - vq[k] = (bu*rq[k] + gv*gq[k] + bv*bq[k] + shift) >> ITUR_BT_601_SHIFT; - } + v_int32 uq0, uq1, uq2, uq3, vq0, vq1, vq2, vq3; + uq0 = v_shr(v_add(v_add(v_add(v_mul(ru, rq0), v_mul(gu, gq0)), v_mul(bu, bq0)), shift)); + vq0 = v_shr(v_add(v_add(v_add(v_mul(bu, rq0), v_mul(gv, gq0)), v_mul(bv, bq0)), shift)); + uq1 = v_shr(v_add(v_add(v_add(v_mul(ru, rq1), v_mul(gu, gq1)), v_mul(bu, bq1)), shift)); + vq1 = v_shr(v_add(v_add(v_add(v_mul(bu, rq1), v_mul(gv, gq1)), v_mul(bv, bq1)), shift)); + uq2 = v_shr(v_add(v_add(v_add(v_mul(ru, rq2), v_mul(gu, gq2)), v_mul(bu, bq2)), shift)); + vq2 = v_shr(v_add(v_add(v_add(v_mul(bu, rq2), v_mul(gv, gq2)), v_mul(bv, bq2)), shift)); + uq3 = v_shr(v_add(v_add(v_add(v_mul(ru, rq3), v_mul(gu, gq3)), v_mul(bu, bq3)), shift)); + vq3 = v_shr(v_add(v_add(v_add(v_mul(bu, rq3), v_mul(gv, gq3)), v_mul(bv, bq3)), shift)); v_int16 u0, u1, v0, v1; - u0 = v_pack(uq[0], uq[1]); - u1 = v_pack(uq[2], uq[3]); - v0 = v_pack(vq[0], vq[1]); - v1 = v_pack(vq[2], vq[3]); + u0 = v_pack(uq0, uq1); + u1 = v_pack(uq2, uq3); + v0 = v_pack(vq0, vq1); + v1 = v_pack(vq2, vq3); u = v_pack_u(u0, u1); v = v_pack_u(v0, v1); @@ -1559,8 +1607,8 @@ struct RGB8toYUV420pInvoker: public ParallelLoopBody } } int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); for( ; i <= w/2 - vsize; i += vsize) @@ -1708,47 +1756,61 @@ struct YUV422toRGB8Invoker : ParallelLoopBody { uchar* row = dst_data + dst_step * j; int i = 0; -#if CV_SIMD - const int vsize = v_uint8::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int vsize = VTraits::vlanes(); v_uint8 a = vx_setall_u8(uchar(0xff)); for(; i <= 2*width - 4*vsize; i += 4*vsize, row += vsize*dcn*2) { - v_uint8 u, v, vy[2]; + v_uint8 u, v, vy0, vy1; if(yIdx == 1) // UYVY { - v_load_deinterleave(yuv_src + i, u, vy[0], v, vy[1]); + v_load_deinterleave(yuv_src + i, u, vy0, v, vy1); } else // YUYV or YVYU { - v_load_deinterleave(yuv_src + i, vy[0], u, vy[1], v); + v_load_deinterleave(yuv_src + i, vy0, u, vy1, v); if(uIdx == 1) // YVYU { swap(u, v); } } - v_int32 ruv[4], guv[4], buv[4]; - uvToRGBuv(u, v, ruv, guv, buv); + v_int32 ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3; + uvToRGBuv(u, v, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3); - v_uint8 r[2], g[2], b[2]; + v_uint8 r0, r1, g0, g1, b0, b1; - yRGBuvToRGBA(vy[0], ruv, guv, buv, r[0], g[0], b[0]); - yRGBuvToRGBA(vy[1], ruv, guv, buv, r[1], g[1], b[1]); + + yRGBuvToRGBA(vy0, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3, + r0, g0, b0); + yRGBuvToRGBA(vy1, + ruv0, ruv1, ruv2, ruv3, + guv0, guv1, guv2, guv3, + buv0, buv1, buv2, buv3, + r1, g1, b1); if(bIdx) { - swap(r[0], b[0]); - swap(r[1], b[1]); + swap(r0, b0); + swap(r1, b1); } // [r0...], [r1...] => [r0, r1, r0, r1...], [r0, r1, r0, r1...] v_uint8 r0_0, r0_1; - v_zip(r[0], r[1], r0_0, r0_1); + v_zip(r0, r1, r0_0, r0_1); v_uint8 g0_0, g0_1; - v_zip(g[0], g[1], g0_0, g0_1); + v_zip(g0, g1, g0_0, g0_1); v_uint8 b0_0, b0_1; - v_zip(b[0], b[1], b0_0, b0_1); + v_zip(b0, b1, b0_0, b0_1); if(dcn == 4) { diff --git a/modules/imgproc/src/contours.cpp b/modules/imgproc/src/contours.cpp index 25c34f0829..3c5501659a 100644 --- a/modules/imgproc/src/contours.cpp +++ b/modules/imgproc/src/contours.cpp @@ -71,14 +71,14 @@ static Rect pointSetBoundingRect( const Mat& points ) if( npoints == 0 ) return Rect(); -#if CV_SIMD +#if CV_SIMD // TODO: enable for CV_SIMD_SCALABLE, loop tail related. const int64_t* pts = points.ptr(); if( !is_float ) { v_int32 minval, maxval; minval = maxval = v_reinterpret_as_s32(vx_setall_s64(*pts)); //min[0]=pt.x, min[1]=pt.y, min[2]=pt.x, min[3]=pt.y - for( i = 1; i <= npoints - v_int32::nlanes/2; i+= v_int32::nlanes/2 ) + for( i = 1; i <= npoints - VTraits::vlanes()/2; i+= VTraits::vlanes()/2 ) { v_int32 ptXY2 = v_reinterpret_as_s32(vx_load(pts + i)); minval = v_min(ptXY2, minval); @@ -86,22 +86,22 @@ static Rect pointSetBoundingRect( const Mat& points ) } minval = v_min(v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(minval))), v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval)))); maxval = v_max(v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(maxval))), v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval)))); - if( i <= npoints - v_int32::nlanes/4 ) + if( i <= npoints - VTraits::vlanes()/4 ) { v_int32 ptXY = v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(vx_load_low(pts + i)))); minval = v_min(ptXY, minval); maxval = v_max(ptXY, maxval); - i += v_int64::nlanes/2; + i += VTraits::vlanes()/2; } - for(int j = 16; j < CV_SIMD_WIDTH; j*=2) + for(int j = 16; j < VTraits::vlanes(); j*=2) { minval = v_min(v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(minval))), v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval)))); maxval = v_max(v_reinterpret_as_s32(v_expand_low(v_reinterpret_as_u32(maxval))), v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval)))); } - xmin = minval.get0(); - xmax = maxval.get0(); - ymin = v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval))).get0(); - ymax = v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval))).get0(); + xmin = v_get0(minval); + xmax = v_get0(maxval); + ymin = v_get0(v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval)))); + ymax = v_get0(v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval)))); #if CV_SIMD_WIDTH > 16 if( i < npoints ) { @@ -113,18 +113,18 @@ static Rect pointSetBoundingRect( const Mat& points ) minval2 = v_min(ptXY, minval2); maxval2 = v_max(ptXY, maxval2); } - xmin = min(xmin, minval2.get0()); - xmax = max(xmax, maxval2.get0()); - ymin = min(ymin, v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval2))).get0()); - ymax = max(ymax, v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval2))).get0()); + xmin = min(xmin, v_get0(minval2)); + xmax = max(xmax, v_get0(maxval2)); + ymin = min(ymin, v_get0(v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(minval2))))); + ymax = max(ymax, v_get0(v_reinterpret_as_s32(v_expand_high(v_reinterpret_as_u32(maxval2))))); } -#endif +#endif // CV_SIMD } else { v_float32 minval, maxval; minval = maxval = v_reinterpret_as_f32(vx_setall_s64(*pts)); //min[0]=pt.x, min[1]=pt.y, min[2]=pt.x, min[3]=pt.y - for( i = 1; i <= npoints - v_float32::nlanes/2; i+= v_float32::nlanes/2 ) + for( i = 1; i <= npoints - VTraits::vlanes()/2; i+= VTraits::vlanes()/2 ) { v_float32 ptXY2 = v_reinterpret_as_f32(vx_load(pts + i)); minval = v_min(ptXY2, minval); @@ -132,22 +132,22 @@ static Rect pointSetBoundingRect( const Mat& points ) } minval = v_min(v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(minval))), v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval)))); maxval = v_max(v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(maxval))), v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval)))); - if( i <= npoints - v_float32::nlanes/4 ) + if( i <= npoints - VTraits::vlanes()/4 ) { v_float32 ptXY = v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(vx_load_low(pts + i)))); minval = v_min(ptXY, minval); maxval = v_max(ptXY, maxval); - i += v_float32::nlanes/4; + i += VTraits::vlanes()/4; } - for(int j = 16; j < CV_SIMD_WIDTH; j*=2) + for(int j = 16; j < VTraits::vlanes(); j*=2) { minval = v_min(v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(minval))), v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval)))); maxval = v_max(v_reinterpret_as_f32(v_expand_low(v_reinterpret_as_u32(maxval))), v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval)))); } - xmin = cvFloor(minval.get0()); - xmax = cvFloor(maxval.get0()); - ymin = cvFloor(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval))).get0()); - ymax = cvFloor(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval))).get0()); + xmin = cvFloor(v_get0(minval)); + xmax = cvFloor(v_get0(maxval)); + ymin = cvFloor(v_get0(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval))))); + ymax = cvFloor(v_get0(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval))))); #if CV_SIMD_WIDTH > 16 if( i < npoints ) { @@ -159,10 +159,10 @@ static Rect pointSetBoundingRect( const Mat& points ) minval2 = v_min(ptXY, minval2); maxval2 = v_max(ptXY, maxval2); } - xmin = min(xmin, cvFloor(minval2.get0())); - xmax = max(xmax, cvFloor(maxval2.get0())); - ymin = min(ymin, cvFloor(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval2))).get0())); - ymax = max(ymax, cvFloor(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval2))).get0())); + xmin = min(xmin, cvFloor(v_get0(minval2))); + xmax = max(xmax, cvFloor(v_get0(maxval2))); + ymin = min(ymin, cvFloor(v_get0(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(minval2)))))); + ymax = max(ymax, cvFloor(v_get0(v_reinterpret_as_f32(v_expand_high(v_reinterpret_as_u32(maxval2)))))); } #endif } @@ -1258,7 +1258,7 @@ static CvSeq * cvFindNextContour( CvContourScanner scanner ) } else { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if ((p = img[x]) != prev) { goto _next_contour; @@ -1266,9 +1266,9 @@ static CvSeq * cvFindNextContour( CvContourScanner scanner ) else { v_uint8 v_prev = vx_setall_u8((uchar)prev); - for (; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_uint8 vmask = (vx_load((uchar*)(img + x)) != v_prev); + v_uint8 vmask = (v_ne(vx_load((uchar *)(img + x)), v_prev)); if (v_check_any(vmask)) { p = img[(x += v_scan_forward(vmask))]; @@ -1283,7 +1283,7 @@ static CvSeq * cvFindNextContour( CvContourScanner scanner ) if( x >= width ) break; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) _next_contour: #endif { @@ -1530,11 +1530,11 @@ CvLinkedRunPoint; inline int findStartContourPoint(uchar *src_data, CvSize img_size, int j) { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_uint8 v_zero = vx_setzero_u8(); - for (; j <= img_size.width - v_uint8::nlanes; j += v_uint8::nlanes) + for (; j <= img_size.width - VTraits::vlanes(); j += VTraits::vlanes()) { - v_uint8 vmask = (vx_load((uchar*)(src_data + j)) != v_zero); + v_uint8 vmask = (v_ne(vx_load((uchar *)(src_data + j)), v_zero)); if (v_check_any(vmask)) { j += v_scan_forward(vmask); @@ -1549,7 +1549,7 @@ inline int findStartContourPoint(uchar *src_data, CvSize img_size, int j) inline int findEndContourPoint(uchar *src_data, CvSize img_size, int j) { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (j < img_size.width && !src_data[j]) { return j; @@ -1557,9 +1557,9 @@ inline int findEndContourPoint(uchar *src_data, CvSize img_size, int j) else { v_uint8 v_zero = vx_setzero_u8(); - for (; j <= img_size.width - v_uint8::nlanes; j += v_uint8::nlanes) + for (; j <= img_size.width - VTraits::vlanes(); j += VTraits::vlanes()) { - v_uint8 vmask = (vx_load((uchar*)(src_data + j)) == v_zero); + v_uint8 vmask = (v_eq(vx_load((uchar *)(src_data + j)), v_zero)); if (v_check_any(vmask)) { j += v_scan_forward(vmask); diff --git a/modules/imgproc/src/corner.cpp b/modules/imgproc/src/corner.cpp index 31078b9bed..86b843ffc9 100644 --- a/modules/imgproc/src/corner.cpp +++ b/modules/imgproc/src/corner.cpp @@ -74,21 +74,21 @@ static void calcMinEigenVal( const Mat& _cov, Mat& _dst ) #endif // CV_TRY_AVX j = 0; -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) { - v_float32x4 half = v_setall_f32(0.5f); - for( ; j <= size.width - v_float32x4::nlanes; j += v_float32x4::nlanes ) + v_float32 half = vx_setall_f32(0.5f); + for( ; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes() ) { - v_float32x4 v_a, v_b, v_c, v_t; + v_float32 v_a, v_b, v_c, v_t; v_load_deinterleave(cov + j*3, v_a, v_b, v_c); - v_a *= half; - v_c *= half; - v_t = v_a - v_c; - v_t = v_muladd(v_b, v_b, (v_t * v_t)); - v_store(dst + j, (v_a + v_c) - v_sqrt(v_t)); + v_a = v_mul(v_a, half); + v_c = v_mul(v_c, half); + v_t = v_sub(v_a, v_c); + v_t = v_muladd(v_b, v_b, (v_mul(v_t, v_t))); + v_store(dst + j, v_sub(v_add(v_a, v_c), v_sqrt(v_t))); } } -#endif // CV_SIMD128 +#endif // CV_SIMD for( ; j < size.width; j++ ) { @@ -127,18 +127,18 @@ static void calcHarris( const Mat& _cov, Mat& _dst, double k ) #endif // CV_TRY_AVX j = 0; -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) { - v_float32x4 v_k = v_setall_f32((float)k); + v_float32 v_k = vx_setall_f32((float)k); - for( ; j <= size.width - v_float32x4::nlanes; j += v_float32x4::nlanes ) + for( ; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes() ) { - v_float32x4 v_a, v_b, v_c; + v_float32 v_a, v_b, v_c; v_load_deinterleave(cov + j * 3, v_a, v_b, v_c); - v_float32x4 v_ac_bb = v_a * v_c - v_b * v_b; - v_float32x4 v_ac = v_a + v_c; - v_float32x4 v_dst = v_ac_bb - v_k * v_ac * v_ac; + v_float32 v_ac_bb = v_sub(v_mul(v_a, v_c), v_mul(v_b, v_b)); + v_float32 v_ac = v_add(v_a, v_c); + v_float32 v_dst = v_sub(v_ac_bb, v_mul(v_mul(v_k, v_ac), v_ac)); v_store(dst + j, v_dst); } } @@ -282,22 +282,22 @@ cornerEigenValsVecs( const Mat& src, Mat& eigenv, int block_size, #endif // CV_TRY_AVX j = 0; -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) { - for( ; j <= size.width - v_float32x4::nlanes; j += v_float32x4::nlanes ) + for( ; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes() ) { - v_float32x4 v_dx = v_load(dxdata + j); - v_float32x4 v_dy = v_load(dydata + j); + v_float32 v_dx = vx_load(dxdata + j); + v_float32 v_dy = vx_load(dydata + j); - v_float32x4 v_dst0, v_dst1, v_dst2; - v_dst0 = v_dx * v_dx; - v_dst1 = v_dx * v_dy; - v_dst2 = v_dy * v_dy; + v_float32 v_dst0, v_dst1, v_dst2; + v_dst0 = v_mul(v_dx, v_dx); + v_dst1 = v_mul(v_dx, v_dy); + v_dst2 = v_mul(v_dy, v_dy); v_store_interleave(cov_data + j * 3, v_dst0, v_dst1, v_dst2); } } -#endif // CV_SIMD128 +#endif // CV_SIMD for( ; j < size.width; j++ ) { @@ -693,9 +693,9 @@ void cv::preCornerDetect( InputArray _src, OutputArray _dst, int ksize, int bord if( src.depth() == CV_8U ) factor *= 255; factor = 1./(factor * factor * factor); -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) float factor_f = (float)factor; - v_float32x4 v_factor = v_setall_f32(factor_f), v_m2 = v_setall_f32(-2.0f); + v_float32 v_factor = vx_setall_f32(factor_f), v_m2 = vx_setall_f32(-2.0f); #endif Size size = src.size(); @@ -711,18 +711,18 @@ void cv::preCornerDetect( InputArray _src, OutputArray _dst, int ksize, int bord j = 0; -#if CV_SIMD128 +#if (CV_SIMD || CV_SIMD_SCALABLE) { - for( ; j <= size.width - v_float32x4::nlanes; j += v_float32x4::nlanes ) + for( ; j <= size.width - VTraits::vlanes(); j += VTraits::vlanes() ) { - v_float32x4 v_dx = v_load(dxdata + j); - v_float32x4 v_dy = v_load(dydata + j); + v_float32 v_dx = vx_load(dxdata + j); + v_float32 v_dy = vx_load(dydata + j); - v_float32x4 v_s1 = (v_dx * v_dx) * v_load(d2ydata + j); - v_float32x4 v_s2 = v_muladd((v_dy * v_dy), v_load(d2xdata + j), v_s1); - v_float32x4 v_s3 = v_muladd((v_dy * v_dx) * v_load(dxydata + j), v_m2, v_s2); + v_float32 v_s1 = v_mul(v_mul(v_dx, v_dx), vx_load(d2ydata + j)); + v_float32 v_s2 = v_muladd((v_mul(v_dy, v_dy)), vx_load(d2xdata + j), v_s1); + v_float32 v_s3 = v_muladd(v_mul(v_mul(v_dy, v_dx), vx_load(dxydata + j)), v_m2, v_s2); - v_store(dstdata + j, v_s3 * v_factor); + v_store(dstdata + j, v_mul(v_s3, v_factor)); } } #endif diff --git a/modules/imgproc/src/filter.simd.hpp b/modules/imgproc/src/filter.simd.hpp index 8dcf5235af..06053e63fe 100644 --- a/modules/imgproc/src/filter.simd.hpp +++ b/modules/imgproc/src/filter.simd.hpp @@ -349,7 +349,7 @@ struct FilterNoVec }; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) ///////////////////////////////////// 8u-16s & 8u-8u ////////////////////////////////// @@ -383,7 +383,7 @@ struct RowVec_8u32s if( smallValues ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { const uchar* src = _src + i; v_int32 s0 = vx_setzero_s32(); @@ -396,27 +396,27 @@ struct RowVec_8u32s v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16)); v_uint8 x0, x1; v_zip(vx_load(src), vx_load(src + cn), x0, x1); - s0 += v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f)); - s1 += v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f)); - s2 += v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f)); - s3 += v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f)); + s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f))); + s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f))); + s2 = v_add(s2, v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f))); + s3 = v_add(s3, v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f))); } if (k < _ksize) { v_int32 f = vx_setall_s32(_kx[k]); v_uint16 x0, x1; v_expand(vx_load(src), x0, x1); - s0 += v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f)); - s1 += v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f)); - s2 += v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f)); - s3 += v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f)); + s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(v_expand_low(x0)), v_reinterpret_as_s16(f))); + s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(v_expand_high(x0)), v_reinterpret_as_s16(f))); + s2 = v_add(s2, v_dotprod(v_reinterpret_as_s16(v_expand_low(x1)), v_reinterpret_as_s16(f))); + s3 = v_add(s3, v_dotprod(v_reinterpret_as_s16(v_expand_high(x1)), v_reinterpret_as_s16(f))); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - v_store(dst + i + 2*v_int32::nlanes, s2); - v_store(dst + i + 3*v_int32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { const uchar* src = _src + i; v_int32 s0 = vx_setzero_s32(); @@ -427,22 +427,22 @@ struct RowVec_8u32s v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16)); v_uint16 x0, x1; v_zip(vx_load_expand(src), vx_load_expand(src + cn), x0, x1); - s0 += v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f)); - s1 += v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f)); + s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f))); + s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f))); } if( k < _ksize ) { v_int32 f = vx_setall_s32(_kx[k]); v_uint32 x0, x1; v_expand(vx_load_expand(src), x0, x1); - s0 += v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f)); - s1 += v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f)); + s0 = v_add(s0, v_dotprod(v_reinterpret_as_s16(x0), v_reinterpret_as_s16(f))); + s1 = v_add(s1, v_dotprod(v_reinterpret_as_s16(x1), v_reinterpret_as_s16(f))); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - i += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 d = vx_setzero_s32(); k = 0; @@ -452,12 +452,12 @@ struct RowVec_8u32s v_int32 f = vx_setall_s32((_kx[k] & 0xFFFF) | (_kx[k + 1] << 16)); v_uint32 x0, x1; v_zip(vx_load_expand_q(src), vx_load_expand_q(src + cn), x0, x1); - d += v_dotprod(v_pack(v_reinterpret_as_s32(x0), v_reinterpret_as_s32(x1)), v_reinterpret_as_s16(f)); + d = v_add(d, v_dotprod(v_pack(v_reinterpret_as_s32(x0), v_reinterpret_as_s32(x1)), v_reinterpret_as_s16(f))); } if (k < _ksize) - d += v_dotprod(v_reinterpret_as_s16(vx_load_expand_q(src)), v_reinterpret_as_s16(vx_setall_s32(_kx[k]))); + d = v_add(d, v_dotprod(v_reinterpret_as_s16(vx_load_expand_q(src)), v_reinterpret_as_s16(vx_setall_s32(_kx[k])))); v_store(dst + i, d); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } return i; @@ -480,7 +480,7 @@ struct RowVec_8u32f float* dst = (float*)_dst; const float* _kx = kernel.ptr(); width *= cn; - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 s0 = vx_setzero_f32(); v_float32 s1 = vx_setzero_f32(); @@ -492,18 +492,18 @@ struct RowVec_8u32f v_float32 f = vx_setall_f32(_kx[k]); const uchar* src = (const uchar*)_src + i + k * cn; v_float32 vs_ll = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src))); - v_float32 vs_lh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + v_float32::nlanes))); - v_float32 vs_hl = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 2*v_float32::nlanes))); - v_float32 vs_hh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 3*v_float32::nlanes))); + v_float32 vs_lh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + VTraits::vlanes()))); + v_float32 vs_hl = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 2*VTraits::vlanes()))); + v_float32 vs_hh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 3*VTraits::vlanes()))); s0 = v_muladd(vs_ll, f, s0); s1 = v_muladd(vs_lh, f, s1); s2 = v_muladd(vs_hl, f, s2); s3 = v_muladd(vs_hh, f, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } return i; } @@ -553,7 +553,7 @@ struct SymmRowSmallVec_8u32s { if( kx[0] == 2 && kx[1] == 1 ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -562,29 +562,29 @@ struct SymmRowSmallVec_8u32s x1l = v_add_wrap(v_add_wrap(x1l, x1l), v_add_wrap(x0l, x2l)); x1h = v_add_wrap(v_add_wrap(x1h, x1h), v_add_wrap(x0h, x2h)); v_store(dst + i, v_reinterpret_as_s32(v_expand_low(x1l))); - v_store(dst + i + v_int32::nlanes, v_reinterpret_as_s32(v_expand_high(x1l))); - v_store(dst + i + 2*v_int32::nlanes, v_reinterpret_as_s32(v_expand_low(x1h))); - v_store(dst + i + 3*v_int32::nlanes, v_reinterpret_as_s32(v_expand_high(x1h))); + v_store(dst + i + VTraits::vlanes(), v_reinterpret_as_s32(v_expand_high(x1l))); + v_store(dst + i + 2*VTraits::vlanes(), v_reinterpret_as_s32(v_expand_low(x1h))); + v_store(dst + i + 3*VTraits::vlanes(), v_reinterpret_as_s32(v_expand_high(x1h))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint16 x = vx_load_expand(src); x = v_add_wrap(v_add_wrap(x, x), v_add_wrap(vx_load_expand(src - cn), vx_load_expand(src + cn))); v_store(dst + i, v_reinterpret_as_s32(v_expand_low(x))); - v_store(dst + i + v_int32::nlanes, v_reinterpret_as_s32(v_expand_high(x))); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_reinterpret_as_s32(v_expand_high(x))); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint32 x = vx_load_expand_q(src); - x = (x + x) + vx_load_expand_q(src - cn) + vx_load_expand_q(src + cn); + x = v_add(v_add(v_add(x, x), vx_load_expand_q(src - cn)), vx_load_expand_q(src + cn)); v_store(dst + i, v_reinterpret_as_s32(x)); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } else if( kx[0] == -2 && kx[1] == 1 ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -593,31 +593,31 @@ struct SymmRowSmallVec_8u32s x1l = v_sub_wrap(v_add_wrap(x0l, x2l), v_add_wrap(x1l, x1l)); x1h = v_sub_wrap(v_add_wrap(x0h, x2h), v_add_wrap(x1h, x1h)); v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x1l))); - v_store(dst + i + v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x1l))); - v_store(dst + i + 2*v_int32::nlanes, v_expand_low(v_reinterpret_as_s16(x1h))); - v_store(dst + i + 3*v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x1h))); + v_store(dst + i + VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x1l))); + v_store(dst + i + 2*VTraits::vlanes(), v_expand_low(v_reinterpret_as_s16(x1h))); + v_store(dst + i + 3*VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x1h))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint16 x = vx_load_expand(src); x = v_sub_wrap(v_add_wrap(vx_load_expand(src - cn), vx_load_expand(src + cn)), v_add_wrap(x, x)); v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x))); - v_store(dst + i + v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x))); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x))); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 x = v_reinterpret_as_s32(vx_load_expand_q(src)); - x = v_reinterpret_as_s32(vx_load_expand_q(src - cn) + vx_load_expand_q(src + cn)) - (x + x); + x = v_sub(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), v_add(x, x)); v_store(dst + i, x); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } else { v_int16 k0 = vx_setall_s16((short)kx[0]); v_int16 k1 = vx_setall_s16((short)kx[1]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -628,34 +628,34 @@ struct SymmRowSmallVec_8u32s v_int16 x0, x1; v_mul_expand(v_reinterpret_as_s16(x1l), k0, dl, dh); v_zip(v_reinterpret_as_s16(x0l), v_reinterpret_as_s16(x2l), x0, x1); - dl += v_dotprod(x0, k1); - dh += v_dotprod(x1, k1); + dl = v_add(dl, v_dotprod(x0, k1)); + dh = v_add(dh, v_dotprod(x1, k1)); v_store(dst + i, dl); - v_store(dst + i + v_int32::nlanes, dh); + v_store(dst + i + VTraits::vlanes(), dh); v_mul_expand(v_reinterpret_as_s16(x1h), k0, dl, dh); v_zip(v_reinterpret_as_s16(x0h), v_reinterpret_as_s16(x2h), x0, x1); - dl += v_dotprod(x0, k1); - dh += v_dotprod(x1, k1); - v_store(dst + i + 2*v_int32::nlanes, dl); - v_store(dst + i + 3*v_int32::nlanes, dh); + dl = v_add(dl, v_dotprod(x0, k1)); + dh = v_add(dh, v_dotprod(x1, k1)); + v_store(dst + i + 2*VTraits::vlanes(), dl); + v_store(dst + i + 3*VTraits::vlanes(), dh); } - if ( i <= width - v_uint16::nlanes ) + if ( i <= width - VTraits::vlanes() ) { v_int32 dl, dh; v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, dl, dh); v_int16 x0, x1; v_zip(v_reinterpret_as_s16(vx_load_expand(src - cn)), v_reinterpret_as_s16(vx_load_expand(src + cn)), x0, x1); - dl += v_dotprod(x0, k1); - dh += v_dotprod(x1, k1); + dl = v_add(dl, v_dotprod(x0, k1)); + dh = v_add(dh, v_dotprod(x1, k1)); v_store(dst + i, dl); - v_store(dst + i + v_int32::nlanes, dh); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), dh); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if ( i <= width - v_uint32::nlanes ) + if ( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]), v_reinterpret_as_s32(vx_load_expand_q(src - cn) + vx_load_expand_q(src + cn)) * vx_setall_s32(kx[1]))); - i += v_uint32::nlanes; + v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]), v_mul(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), vx_setall_s32(kx[1])))); + i += VTraits::vlanes(); } } } @@ -663,7 +663,7 @@ struct SymmRowSmallVec_8u32s { if( kx[0] == -2 && kx[1] == 0 && kx[2] == 1 ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h; v_expand(vx_load(src - 2*cn), x0l, x0h); @@ -672,31 +672,31 @@ struct SymmRowSmallVec_8u32s x1l = v_sub_wrap(v_add_wrap(x0l, x2l), v_add_wrap(x1l, x1l)); x1h = v_sub_wrap(v_add_wrap(x0h, x2h), v_add_wrap(x1h, x1h)); v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x1l))); - v_store(dst + i + v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x1l))); - v_store(dst + i + 2*v_int32::nlanes, v_expand_low(v_reinterpret_as_s16(x1h))); - v_store(dst + i + 3*v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x1h))); + v_store(dst + i + VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x1l))); + v_store(dst + i + 2*VTraits::vlanes(), v_expand_low(v_reinterpret_as_s16(x1h))); + v_store(dst + i + 3*VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x1h))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint16 x = vx_load_expand(src); x = v_sub_wrap(v_add_wrap(vx_load_expand(src - 2*cn), vx_load_expand(src + 2*cn)), v_add_wrap(x, x)); v_store(dst + i, v_expand_low(v_reinterpret_as_s16(x))); - v_store(dst + i + v_int32::nlanes, v_expand_high(v_reinterpret_as_s16(x))); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_expand_high(v_reinterpret_as_s16(x))); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 x = v_reinterpret_as_s32(vx_load_expand_q(src)); - x = v_reinterpret_as_s32(vx_load_expand_q(src - 2*cn) + vx_load_expand_q(src + 2*cn)) - (x + x); + x = v_sub(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - 2 * cn), vx_load_expand_q(src + 2 * cn))), v_add(x, x)); v_store(dst + i, x); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } else { v_int16 k0 = vx_setall_s16((short)(kx[0])); v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (kx[2] << 16))); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_int32 x0, x1, x2, x3; v_uint16 x0l, x0h, x1l, x1h, x2l, x2h, x3l, x3h; @@ -710,45 +710,45 @@ struct SymmRowSmallVec_8u32s v_expand(vx_load(src + cn), x1l, x1h); v_expand(vx_load(src - 2*cn), x2l, x2h); v_expand(vx_load(src + 2*cn), x3l, x3h); - v_zip(v_reinterpret_as_s16(x0l + x1l), v_reinterpret_as_s16(x2l + x3l), xl, xh); - x0 += v_dotprod(xl, k12); - x1 += v_dotprod(xh, k12); - v_zip(v_reinterpret_as_s16(x0h + x1h), v_reinterpret_as_s16(x2h + x3h), xl, xh); - x2 += v_dotprod(xl, k12); - x3 += v_dotprod(xh, k12); + v_zip(v_reinterpret_as_s16(v_add(x0l, x1l)), v_reinterpret_as_s16(v_add(x2l, x3l)), xl, xh); + x0 = v_add(x0, v_dotprod(xl, k12)); + x1 = v_add(x1, v_dotprod(xh, k12)); + v_zip(v_reinterpret_as_s16(v_add(x0h, x1h)), v_reinterpret_as_s16(v_add(x2h, x3h)), xl, xh); + x2 = v_add(x2, v_dotprod(xl, k12)); + x3 = v_add(x3, v_dotprod(xh, k12)); v_store(dst + i, x0); - v_store(dst + i + v_int32::nlanes, x1); - v_store(dst + i + 2*v_int32::nlanes, x2); - v_store(dst + i + 3*v_int32::nlanes, x3); + v_store(dst + i + VTraits::vlanes(), x1); + v_store(dst + i + 2*VTraits::vlanes(), x2); + v_store(dst + i + 3*VTraits::vlanes(), x3); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 x1, x2; v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, x1, x2); v_int16 xl, xh; - v_zip(v_reinterpret_as_s16(vx_load_expand(src - cn) + vx_load_expand(src + cn)), v_reinterpret_as_s16(vx_load_expand(src - 2*cn) + vx_load_expand(src + 2*cn)), xl, xh); - x1 += v_dotprod(xl, k12); - x2 += v_dotprod(xh, k12); + v_zip(v_reinterpret_as_s16(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn))), v_reinterpret_as_s16(v_add(vx_load_expand(src - 2 * cn), vx_load_expand(src + 2 * cn))), xl, xh); + x1 = v_add(x1, v_dotprod(xl, k12)); + x2 = v_add(x2, v_dotprod(xh, k12)); v_store(dst + i, x1); - v_store(dst + i + v_int32::nlanes, x2); - i += v_uint16::nlanes, src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), x2); + i += VTraits::vlanes(), src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0]), - v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src - cn) + vx_load_expand_q(src + cn)), vx_setall_s32(kx[1]), - v_reinterpret_as_s32(vx_load_expand_q(src - 2*cn) + vx_load_expand_q(src + 2*cn)) * vx_setall_s32(kx[2])))); - i += v_uint32::nlanes; + v_muladd(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - cn), vx_load_expand_q(src + cn))), vx_setall_s32(kx[1]), + v_mul(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - 2 * cn), vx_load_expand_q(src + 2 * cn))), vx_setall_s32(kx[2]))))); + i += VTraits::vlanes(); } } } else { v_int16 k0 = vx_setall_s16((short)(kx[0])); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint8 v_src = vx_load(src); v_int32 s0, s1, s2, s3; @@ -764,12 +764,12 @@ struct SymmRowSmallVec_8u32s v_uint8 v_src3 = vx_load(src + j + cn); v_int16 xl, xh; - v_zip(v_reinterpret_as_s16(v_expand_low(v_src0) + v_expand_low(v_src2)), v_reinterpret_as_s16(v_expand_low(v_src1) + v_expand_low(v_src3)), xl, xh); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); - v_zip(v_reinterpret_as_s16(v_expand_high(v_src0) + v_expand_high(v_src2)), v_reinterpret_as_s16(v_expand_high(v_src1) + v_expand_high(v_src3)), xl, xh); - s2 += v_dotprod(xl, k12); - s3 += v_dotprod(xh, k12); + v_zip(v_reinterpret_as_s16(v_add(v_expand_low(v_src0), v_expand_low(v_src2))), v_reinterpret_as_s16(v_add(v_expand_low(v_src1), v_expand_low(v_src3))), xl, xh); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); + v_zip(v_reinterpret_as_s16(v_add(v_expand_high(v_src0), v_expand_high(v_src2))), v_reinterpret_as_s16(v_add(v_expand_high(v_src1), v_expand_high(v_src3))), xl, xh); + s2 = v_add(s2, v_dotprod(xl, k12)); + s3 = v_add(s3, v_dotprod(xh, k12)); } if( k < _ksize / 2 + 1 ) { @@ -780,48 +780,48 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(v_expand_low(v_src0)), v_reinterpret_as_s16(v_expand_low(v_src1)), xl, xh); - s0 += v_dotprod(xl, k1); - s1 += v_dotprod(xh, k1); + s0 = v_add(s0, v_dotprod(xl, k1)); + s1 = v_add(s1, v_dotprod(xh, k1)); v_zip(v_reinterpret_as_s16(v_expand_high(v_src0)), v_reinterpret_as_s16(v_expand_high(v_src1)), xl, xh); - s2 += v_dotprod(xl, k1); - s3 += v_dotprod(xh, k1); + s2 = v_add(s2, v_dotprod(xl, k1)); + s3 = v_add(s3, v_dotprod(xh, k1)); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - v_store(dst + i + 2*v_int32::nlanes, s2); - v_store(dst + i + 3*v_int32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 s0, s1; v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, s0, s1); for (k = 1, j = cn; k <= _ksize / 2 - 1; k+=2, j += 2*cn) { v_int16 xl, xh; - v_zip(v_reinterpret_as_s16(vx_load_expand(src - j) + vx_load_expand(src + j)), v_reinterpret_as_s16(vx_load_expand(src - j - cn) + vx_load_expand(src + j + cn)), xl, xh); + v_zip(v_reinterpret_as_s16(v_add(vx_load_expand(src - j), vx_load_expand(src + j))), v_reinterpret_as_s16(v_add(vx_load_expand(src - j - cn), vx_load_expand(src + j + cn))), xl, xh); v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (kx[k+1] << 16))); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); } if ( k < _ksize / 2 + 1 ) { v_int16 xl, xh; v_zip(v_reinterpret_as_s16(vx_load_expand(src - j)), v_reinterpret_as_s16(vx_load_expand(src + j)), xl, xh); v_int16 k1 = vx_setall_s16((short)(kx[k])); - s0 += v_dotprod(xl, k1); - s1 += v_dotprod(xh, k1); + s0 = v_add(s0, v_dotprod(xl, k1)); + s1 = v_add(s1, v_dotprod(xh, k1)); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_int32 s0 = v_reinterpret_as_s32(vx_load_expand_q(src)) * vx_setall_s32(kx[0]); + v_int32 s0 = v_mul(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0])); for( k = 1, j = cn; k < _ksize / 2 + 1; k++, j += cn ) - s0 = v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src - j) + vx_load_expand_q(src + j)), vx_setall_s32(kx[k]), s0); + s0 = v_muladd(v_reinterpret_as_s32(v_add(vx_load_expand_q(src - j), vx_load_expand_q(src + j))), vx_setall_s32(kx[k]), s0); v_store(dst + i, s0); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } } @@ -831,7 +831,7 @@ struct SymmRowSmallVec_8u32s { if( kx[0] == 0 && kx[1] == 1 ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -839,27 +839,27 @@ struct SymmRowSmallVec_8u32s v_int16 dl = v_reinterpret_as_s16(v_sub_wrap(x2l, x0l)); v_int16 dh = v_reinterpret_as_s16(v_sub_wrap(x2h, x0h)); v_store(dst + i, v_expand_low(dl)); - v_store(dst + i + v_int32::nlanes, v_expand_high(dl)); - v_store(dst + i + 2*v_int32::nlanes, v_expand_low(dh)); - v_store(dst + i + 3*v_int32::nlanes, v_expand_high(dh)); + v_store(dst + i + VTraits::vlanes(), v_expand_high(dl)); + v_store(dst + i + 2*VTraits::vlanes(), v_expand_low(dh)); + v_store(dst + i + 3*VTraits::vlanes(), v_expand_high(dh)); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int16 dl = v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + cn), vx_load_expand(src - cn))); v_store(dst + i, v_expand_low(dl)); - v_store(dst + i + v_int32::nlanes, v_expand_high(dl)); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_expand_high(dl)); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if (i <= width - v_uint32::nlanes) + if (i <= width - VTraits::vlanes()) { - v_store(dst + i, v_reinterpret_as_s32(vx_load_expand_q(src + cn)) - v_reinterpret_as_s32(vx_load_expand_q(src - cn))); - i += v_uint32::nlanes; + v_store(dst + i, v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), v_reinterpret_as_s32(vx_load_expand_q(src - cn)))); + i += VTraits::vlanes(); } } else { v_int16 k0 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (-kx[1] << 16))); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x2l, x2h; v_expand(vx_load(src - cn), x0l, x0h); @@ -867,30 +867,30 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(x2l), v_reinterpret_as_s16(x0l), xl, xh); v_store(dst + i, v_dotprod(xl, k0)); - v_store(dst + i + v_int32::nlanes, v_dotprod(xh, k0)); + v_store(dst + i + VTraits::vlanes(), v_dotprod(xh, k0)); v_zip(v_reinterpret_as_s16(x2h), v_reinterpret_as_s16(x0h), xl, xh); - v_store(dst + i + 2*v_int32::nlanes, v_dotprod(xl, k0)); - v_store(dst + i + 3*v_int32::nlanes, v_dotprod(xh, k0)); + v_store(dst + i + 2*VTraits::vlanes(), v_dotprod(xl, k0)); + v_store(dst + i + 3*VTraits::vlanes(), v_dotprod(xh, k0)); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int16 xl, xh; v_zip(v_reinterpret_as_s16(vx_load_expand(src + cn)), v_reinterpret_as_s16(vx_load_expand(src - cn)), xl, xh); v_store(dst + i, v_dotprod(xl, k0)); - v_store(dst + i + v_int32::nlanes, v_dotprod(xh, k0)); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_dotprod(xh, k0)); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if (i <= width - v_uint32::nlanes) + if (i <= width - VTraits::vlanes()) { - v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), vx_setall_s32(kx[1]), v_reinterpret_as_s32(vx_load_expand_q(src - cn)) * vx_setall_s32(-kx[1]))); - i += v_uint32::nlanes; + v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), vx_setall_s32(kx[1]), v_mul(v_reinterpret_as_s32(vx_load_expand_q(src - cn)), vx_setall_s32(-kx[1])))); + i += VTraits::vlanes(); } } } else if( _ksize == 5 ) { v_int16 k0 = v_reinterpret_as_s16(vx_setall_s32((kx[1] & 0xFFFF) | (kx[2] << 16))); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint16 x0l, x0h, x1l, x1h, x2l, x2h, x3l, x3h; v_expand(vx_load(src - cn), x0l, x0h); @@ -900,31 +900,31 @@ struct SymmRowSmallVec_8u32s v_int16 x0, x1; v_zip(v_reinterpret_as_s16(v_sub_wrap(x2l, x0l)), v_reinterpret_as_s16(v_sub_wrap(x3l, x1l)), x0, x1); v_store(dst + i, v_dotprod(x0, k0)); - v_store(dst + i + v_int32::nlanes, v_dotprod(x1, k0)); + v_store(dst + i + VTraits::vlanes(), v_dotprod(x1, k0)); v_zip(v_reinterpret_as_s16(v_sub_wrap(x2h, x0h)), v_reinterpret_as_s16(v_sub_wrap(x3h, x1h)), x0, x1); - v_store(dst + i + 2*v_int32::nlanes, v_dotprod(x0, k0)); - v_store(dst + i + 3*v_int32::nlanes, v_dotprod(x1, k0)); + v_store(dst + i + 2*VTraits::vlanes(), v_dotprod(x0, k0)); + v_store(dst + i + 3*VTraits::vlanes(), v_dotprod(x1, k0)); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int16 x0, x1; v_zip(v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + cn), vx_load_expand(src - cn))), v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + 2*cn), vx_load_expand(src - 2*cn))), x0, x1); v_store(dst + i, v_dotprod(x0, k0)); - v_store(dst + i + v_int32::nlanes, v_dotprod(x1, k0)); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), v_dotprod(x1, k0)); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + cn)) - v_reinterpret_as_s32(vx_load_expand_q(src - cn)), vx_setall_s32(kx[1]), - (v_reinterpret_as_s32(vx_load_expand_q(src + 2*cn)) - v_reinterpret_as_s32(vx_load_expand_q(src - 2*cn))) * vx_setall_s32(kx[2]))); - i += v_uint32::nlanes; + v_store(dst + i, v_muladd(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + cn)), v_reinterpret_as_s32(vx_load_expand_q(src - cn))), vx_setall_s32(kx[1]), + v_mul(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + 2 * cn)), v_reinterpret_as_s32(vx_load_expand_q(src - 2 * cn))), vx_setall_s32(kx[2])))); + i += VTraits::vlanes(); } } else { v_int16 k0 = vx_setall_s16((short)(kx[0])); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes, src += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_uint8 v_src = vx_load(src); v_int32 s0, s1, s2, s3; @@ -941,11 +941,11 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(v_sub_wrap(v_expand_low(v_src2), v_expand_low(v_src0))), v_reinterpret_as_s16(v_sub_wrap(v_expand_low(v_src3), v_expand_low(v_src1))), xl, xh); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); v_zip(v_reinterpret_as_s16(v_sub_wrap(v_expand_high(v_src2), v_expand_high(v_src0))), v_reinterpret_as_s16(v_sub_wrap(v_expand_high(v_src3), v_expand_high(v_src1))), xl, xh); - s2 += v_dotprod(xl, k12); - s3 += v_dotprod(xh, k12); + s2 = v_add(s2, v_dotprod(xl, k12)); + s3 = v_add(s3, v_dotprod(xh, k12)); } if( k < _ksize / 2 + 1 ) { @@ -955,18 +955,18 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(v_expand_low(v_src1)), v_reinterpret_as_s16(v_expand_low(v_src0)), xl, xh); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); v_zip(v_reinterpret_as_s16(v_expand_high(v_src1)), v_reinterpret_as_s16(v_expand_high(v_src0)), xl, xh); - s2 += v_dotprod(xl, k12); - s3 += v_dotprod(xh, k12); + s2 = v_add(s2, v_dotprod(xl, k12)); + s3 = v_add(s3, v_dotprod(xh, k12)); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - v_store(dst + i + 2*v_int32::nlanes, s2); - v_store(dst + i + 3*v_int32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 s0, s1; v_mul_expand(v_reinterpret_as_s16(vx_load_expand(src)), k0, s0, s1); @@ -975,28 +975,28 @@ struct SymmRowSmallVec_8u32s v_int16 xl, xh; v_zip(v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + j), vx_load_expand(src - j))), v_reinterpret_as_s16(v_sub_wrap(vx_load_expand(src + j + cn), vx_load_expand(src - j - cn))), xl, xh); v_int16 k12 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (kx[k + 1] << 16))); - s0 += v_dotprod(xl, k12); - s1 += v_dotprod(xh, k12); + s0 = v_add(s0, v_dotprod(xl, k12)); + s1 = v_add(s1, v_dotprod(xh, k12)); } if( k < _ksize / 2 + 1 ) { v_int16 k1 = v_reinterpret_as_s16(vx_setall_s32((kx[k] & 0xFFFF) | (-kx[k] << 16))); v_int16 xl, xh; v_zip(v_reinterpret_as_s16(vx_load_expand(src + j)), v_reinterpret_as_s16(vx_load_expand(src - j)), xl, xh); - s0 += v_dotprod(xl, k1); - s1 += v_dotprod(xh, k1); + s0 = v_add(s0, v_dotprod(xl, k1)); + s1 = v_add(s1, v_dotprod(xh, k1)); } v_store(dst + i, s0); - v_store(dst + i + v_int32::nlanes, s1); - i += v_uint16::nlanes; src += v_uint16::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += VTraits::vlanes(); src += VTraits::vlanes(); } - if( i <= width - v_uint32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_int32 s0 = v_reinterpret_as_s32(vx_load_expand_q(src)) * vx_setall_s32(kx[0]); + v_int32 s0 = v_mul(v_reinterpret_as_s32(vx_load_expand_q(src)), vx_setall_s32(kx[0])); for (k = 1, j = cn; k < _ksize / 2 + 1; k++, j += cn) - s0 = v_muladd(v_reinterpret_as_s32(vx_load_expand_q(src + j)) - v_reinterpret_as_s32(vx_load_expand_q(src - j)), vx_setall_s32(kx[k]), s0); + s0 = v_muladd(v_sub(v_reinterpret_as_s32(vx_load_expand_q(src + j)), v_reinterpret_as_s32(vx_load_expand_q(src - j))), vx_setall_s32(kx[k]), s0); v_store(dst + i, s0); - i += v_uint32::nlanes; + i += VTraits::vlanes(); } } } @@ -1038,120 +1038,120 @@ struct SymmColumnVec_32s8u { v_float32 f0 = vx_setall_f32(ky[0]); v_float32 f1 = vx_setall_f32(ky[1]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { const int* S = src[0] + i; v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S)), f0, d4); - v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + v_int32::nlanes)), f0, d4); - v_float32 s2 = v_muladd(v_cvt_f32(vx_load(S + 2*v_int32::nlanes)), f0, d4); - v_float32 s3 = v_muladd(v_cvt_f32(vx_load(S + 3*v_int32::nlanes)), f0, d4); + v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + VTraits::vlanes())), f0, d4); + v_float32 s2 = v_muladd(v_cvt_f32(vx_load(S + 2*VTraits::vlanes())), f0, d4); + v_float32 s3 = v_muladd(v_cvt_f32(vx_load(S + 3*VTraits::vlanes())), f0, d4); const int* S0 = src[1] + i; const int* S1 = src[-1] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) + vx_load(S1)), f1, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) + vx_load(S1 + v_int32::nlanes)), f1, s1); - s2 = v_muladd(v_cvt_f32(vx_load(S0 + 2 * v_int32::nlanes) + vx_load(S1 + 2 * v_int32::nlanes)), f1, s2); - s3 = v_muladd(v_cvt_f32(vx_load(S0 + 3 * v_int32::nlanes) + vx_load(S1 + 3 * v_int32::nlanes)), f1, s3); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f1, s0); + s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f1, s1); + s2 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes()))), f1, s2); + s3 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes()))), f1, s3); for( k = 2; k <= ksize2; k++ ) { v_float32 f = vx_setall_f32(ky[k]); S0 = src[k] + i; S1 = src[-k] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) + vx_load(S1)), f, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) + vx_load(S1 + v_int32::nlanes)), f, s1); - s2 = v_muladd(v_cvt_f32(vx_load(S0 + 2*v_int32::nlanes) + vx_load(S1 + 2*v_int32::nlanes)), f, s2); - s3 = v_muladd(v_cvt_f32(vx_load(S0 + 3*v_int32::nlanes) + vx_load(S1 + 3*v_int32::nlanes)), f, s3); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f, s0); + s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f, s1); + s2 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes()))), f, s2); + s3 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes()))), f, s3); } v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { const int* S = src[0] + i; v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S)), f0, d4); - v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + v_int32::nlanes)), f0, d4); + v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S + VTraits::vlanes())), f0, d4); const int* S0 = src[1] + i; const int* S1 = src[-1] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) + vx_load(S1)), f1, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) + vx_load(S1 + v_int32::nlanes)), f1, s1); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f1, s0); + s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f1, s1); for( k = 2; k <= ksize2; k++ ) { v_float32 f = vx_setall_f32(ky[k]); S0 = src[k] + i; S1 = src[-k] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) + vx_load(S1)), f, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) + vx_load(S1 + v_int32::nlanes)), f, s1); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(S0), vx_load(S1))), f, s0); + s1 = v_muladd(v_cvt_f32(v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f, s1); } v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_uint16::nlanes; + i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - v_int32x4::nlanes ) + while( i <= width - 4 /*v_int32x4::nlanes*/ ) #else - if( i <= width - v_int32x4::nlanes ) + if( i <= width - v_int32::nlanes ) #endif { - v_float32x4 s0 = v_muladd(v_cvt_f32(v_load(src[0] + i)), v_setall_f32(ky[0]), v_setall_f32(delta)); - s0 = v_muladd(v_cvt_f32(v_load(src[1] + i) + v_load(src[-1] + i)), v_setall_f32(ky[1]), s0); + v_float32 s0 = v_muladd(v_cvt_f32(vx_load(src[0] + i)), vx_setall_f32(ky[0]), vx_setall_f32(delta)); + s0 = v_muladd(v_cvt_f32(v_add(vx_load(src[1] + i), vx_load(src[-1] + i))), vx_setall_f32(ky[1]), s0); for( k = 2; k <= ksize2; k++ ) - s0 = v_muladd(v_cvt_f32(v_load(src[k] + i) + v_load(src[-k] + i)), v_setall_f32(ky[k]), s0); - v_int32x4 s32 = v_round(s0); - v_int16x8 s16 = v_pack(s32, s32); - *(unaligned_int*)(dst + i) = v_reinterpret_as_s32(v_pack_u(s16, s16)).get0(); - i += v_int32x4::nlanes; + s0 = v_muladd(v_cvt_f32(v_add(vx_load(src[k] + i), vx_load(src[-k] + i))), vx_setall_f32(ky[k]), s0); + v_int32 s32 = v_round(s0); + v_int16 s16 = v_pack(s32, s32); + *(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16))); + i += 4 /*v_int32x4::nlanes*/ ; } } else { v_float32 f1 = vx_setall_f32(ky[1]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { const int* S0 = src[1] + i; const int* S1 = src[-1] + i; - v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S0) - vx_load(S1)), f1, d4); - v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) - vx_load(S1 + v_int32::nlanes)), f1, d4); - v_float32 s2 = v_muladd(v_cvt_f32(vx_load(S0 + 2 * v_int32::nlanes) - vx_load(S1 + 2 * v_int32::nlanes)), f1, d4); - v_float32 s3 = v_muladd(v_cvt_f32(vx_load(S0 + 3 * v_int32::nlanes) - vx_load(S1 + 3 * v_int32::nlanes)), f1, d4); + v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f1, d4); + v_float32 s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f1, d4); + v_float32 s2 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes()))), f1, d4); + v_float32 s3 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes()))), f1, d4); for ( k = 2; k <= ksize2; k++ ) { v_float32 f = vx_setall_f32(ky[k]); S0 = src[k] + i; S1 = src[-k] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) - vx_load(S1)), f, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) - vx_load(S1 + v_int32::nlanes)), f, s1); - s2 = v_muladd(v_cvt_f32(vx_load(S0 + 2*v_int32::nlanes) - vx_load(S1 + 2*v_int32::nlanes)), f, s2); - s3 = v_muladd(v_cvt_f32(vx_load(S0 + 3*v_int32::nlanes) - vx_load(S1 + 3*v_int32::nlanes)), f, s3); + s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f, s0); + s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f, s1); + s2 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes()))), f, s2); + s3 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes()))), f, s3); } v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { const int* S0 = src[1] + i; const int* S1 = src[-1] + i; - v_float32 s0 = v_muladd(v_cvt_f32(vx_load(S0) - vx_load(S1)), f1, d4); - v_float32 s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) - vx_load(S1 + v_int32::nlanes)), f1, d4); + v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f1, d4); + v_float32 s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f1, d4); for ( k = 2; k <= ksize2; k++ ) { v_float32 f = vx_setall_f32(ky[k]); S0 = src[k] + i; S1 = src[-k] + i; - s0 = v_muladd(v_cvt_f32(vx_load(S0) - vx_load(S1)), f, s0); - s1 = v_muladd(v_cvt_f32(vx_load(S0 + v_int32::nlanes) - vx_load(S1 + v_int32::nlanes)), f, s1); + s0 = v_muladd(v_cvt_f32(v_sub(vx_load(S0), vx_load(S1))), f, s0); + s1 = v_muladd(v_cvt_f32(v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes()))), f, s1); } v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_uint16::nlanes; + i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - v_int32x4::nlanes ) + while( i <= width - 4 /*v_int32x4::nlanes*/ ) #else - if( i <= width - v_int32x4::nlanes ) + if( i <= width - v_int32::nlanes ) #endif { - v_float32x4 s0 = v_muladd(v_cvt_f32(v_load(src[1] + i) - v_load(src[-1] + i)), v_setall_f32(ky[1]), v_setall_f32(delta)); + v_float32 s0 = v_muladd(v_cvt_f32(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i))), vx_setall_f32(ky[1]), vx_setall_f32(delta)); for (k = 2; k <= ksize2; k++) - s0 = v_muladd(v_cvt_f32(v_load(src[k] + i) - v_load(src[-k] + i)), v_setall_f32(ky[k]), s0); - v_int32x4 s32 = v_round(s0); - v_int16x8 s16 = v_pack(s32, s32); - *(unaligned_int*)(dst + i) = v_reinterpret_as_s32(v_pack_u(s16, s16)).get0(); - i += v_int32x4::nlanes; + s0 = v_muladd(v_cvt_f32(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i))), vx_setall_f32(ky[k]), s0); + v_int32 s32 = v_round(s0); + v_int16 s16 = v_pack(s32, s32); + *(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16))); + i += 4 /*v_int32x4::nlanes*/ ; } } return i; @@ -1187,31 +1187,31 @@ struct SymmColumnVec_32f8u if( symmetrical ) { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 v_ky0 = vx_setall_f32(ky[0]); v_float32 v32_delta = vx_setall_f32(delta); const float* S = src[0] + i; v_float32 s0 = v_muladd(v_ky0, vx_load(S), v32_delta); - v_float32 s1 = v_muladd(v_ky0, vx_load(S + v_float32::nlanes), v32_delta); - v_float32 s2 = v_muladd(v_ky0, vx_load(S + 2*v_float32::nlanes), v32_delta); - v_float32 s3 = v_muladd(v_ky0, vx_load(S + 3*v_float32::nlanes), v32_delta); + v_float32 s1 = v_muladd(v_ky0, vx_load(S + VTraits::vlanes()), v32_delta); + v_float32 s2 = v_muladd(v_ky0, vx_load(S + 2*VTraits::vlanes()), v32_delta); + v_float32 s3 = v_muladd(v_ky0, vx_load(S + 3*VTraits::vlanes()), v32_delta); for( k = 1; k <= ksize2; k++ ) { v_float32 v_kyk = vx_setall_f32(ky[k]); const float* S0 = src[k] + i; const float* S1 = src[-k] + i; - s0 = v_muladd(v_kyk, vx_load(S0) + vx_load(S1), s0); - s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) + vx_load(S1 + v_float32::nlanes), s1); - s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) + vx_load(S1 + 2*v_float32::nlanes), s2); - s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) + vx_load(S1 + 3*v_float32::nlanes), s3); + s0 = v_muladd(v_kyk, v_add(vx_load(S0), vx_load(S1)), s0); + s1 = v_muladd(v_kyk, v_add(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes())), s1); + s2 = v_muladd(v_kyk, v_add(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes())), s2); + s3 = v_muladd(v_kyk, v_add(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes())), s3); } v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } } else { - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 s0 = vx_setall_f32(delta); v_float32 s1 = vx_setall_f32(delta); @@ -1222,10 +1222,10 @@ struct SymmColumnVec_32f8u v_float32 v_kyk = vx_setall_f32(ky[k]); const float* S0 = src[k] + i; const float* S1 = src[-k] + i; - s0 = v_muladd(v_kyk, vx_load(S0) - vx_load(S1), s0); - s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) - vx_load(S1 + v_float32::nlanes), s1); - s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) - vx_load(S1 + 2*v_float32::nlanes), s2); - s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) - vx_load(S1 + 3*v_float32::nlanes), s3); + s0 = v_muladd(v_kyk, v_sub(vx_load(S0), vx_load(S1)), s0); + s1 = v_muladd(v_kyk, v_sub(vx_load(S0 + VTraits::vlanes()), vx_load(S1 + VTraits::vlanes())), s1); + s2 = v_muladd(v_kyk, v_sub(vx_load(S0 + 2 * VTraits::vlanes()), vx_load(S1 + 2 * VTraits::vlanes())), s2); + s3 = v_muladd(v_kyk, v_sub(vx_load(S0 + 3 * VTraits::vlanes()), vx_load(S1 + 3 * VTraits::vlanes())), s3); } v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } @@ -1268,55 +1268,52 @@ struct SymmColumnSmallVec_32s16s { if( ky[0] == 2 && ky[1] == 1 ) { - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { v_int32 s0 = vx_load(S1 + i); - v_int32 s1 = vx_load(S1 + i + v_int32::nlanes); - v_int32 s2 = vx_load(S1 + i + 2*v_int32::nlanes); - v_int32 s3 = vx_load(S1 + i + 3*v_int32::nlanes); - v_store(dst + i, v_pack(vx_load(S0 + i) + vx_load(S2 + i) + (s0 + s0), vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes) + (s1 + s1)) + d8); - v_store(dst + i + v_int16::nlanes, v_pack(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes) + (s2 + s2), - vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes) + (s3 + s3)) + d8); + v_int32 s1 = vx_load(S1 + i + VTraits::vlanes()); + v_int32 s2 = vx_load(S1 + i + 2*VTraits::vlanes()); + v_int32 s3 = vx_load(S1 + i + 3*VTraits::vlanes()); + v_store(dst + i, v_add(v_pack(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(s0, s0)), v_add(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), v_add(s1, s1))), d8)); + v_store(dst + i + VTraits::vlanes(), v_add(v_pack(v_add(v_add(vx_load(S0 + i + 2 * VTraits::vlanes()), vx_load(S2 + i + 2 * VTraits::vlanes())), v_add(s2, s2)), v_add(v_add(vx_load(S0 + i + 3 * VTraits::vlanes()), vx_load(S2 + i + 3 * VTraits::vlanes())), v_add(s3, s3))), d8)); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 sl = vx_load(S1 + i); - v_int32 sh = vx_load(S1 + i + v_int32::nlanes); - v_store(dst + i, v_pack(vx_load(S0 + i) + vx_load(S2 + i) + (sl + sl), vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes) + (sh + sh)) + d8); - i += v_int16::nlanes; + v_int32 sh = vx_load(S1 + i + VTraits::vlanes()); + v_store(dst + i, v_add(v_pack(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(sl, sl)), v_add(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), v_add(sh, sh))), d8)); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 s = vx_load(S1 + i); - v_pack_store(dst + i, vx_load(S0 + i) + vx_load(S2 + i) + vx_setall_s32(d) + (s + s)); - i += v_int32::nlanes; + v_pack_store(dst + i, v_add(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), vx_setall_s32(d)), v_add(s, s))); + i += VTraits::vlanes(); } } else if( ky[0] == -2 && ky[1] == 1 ) { - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { v_int32 s0 = vx_load(S1 + i); - v_int32 s1 = vx_load(S1 + i + v_int32::nlanes); - v_int32 s2 = vx_load(S1 + i + 2*v_int32::nlanes); - v_int32 s3 = vx_load(S1 + i + 3*v_int32::nlanes); - v_store(dst + i, v_pack(vx_load(S0 + i) + vx_load(S2 + i) - (s0 + s0), - vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes) - (s1 + s1)) + d8); - v_store(dst + i + v_int16::nlanes, v_pack(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes) - (s2 + s2), - vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes) - (s3 + s3)) + d8); + v_int32 s1 = vx_load(S1 + i + VTraits::vlanes()); + v_int32 s2 = vx_load(S1 + i + 2*VTraits::vlanes()); + v_int32 s3 = vx_load(S1 + i + 3*VTraits::vlanes()); + v_store(dst + i, v_add(v_pack(v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(s0, s0)), v_sub(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), v_add(s1, s1))), d8)); + v_store(dst + i + VTraits::vlanes(), v_add(v_pack(v_sub(v_add(vx_load(S0 + i + 2 * VTraits::vlanes()), vx_load(S2 + i + 2 * VTraits::vlanes())), v_add(s2, s2)), v_sub(v_add(vx_load(S0 + i + 3 * VTraits::vlanes()), vx_load(S2 + i + 3 * VTraits::vlanes())), v_add(s3, s3))), d8)); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 sl = vx_load(S1 + i); - v_int32 sh = vx_load(S1 + i + v_int32::nlanes); - v_store(dst + i, v_pack(vx_load(S0 + i) + vx_load(S2 + i) - (sl + sl), vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes) - (sh + sh)) + d8); - i += v_int16::nlanes; + v_int32 sh = vx_load(S1 + i + VTraits::vlanes()); + v_store(dst + i, v_add(v_pack(v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i)), v_add(sl, sl)), v_sub(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes())), v_add(sh, sh))), d8)); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_int32 s = vx_load(S1 + i); - v_pack_store(dst + i, vx_load(S0 + i) + vx_load(S2 + i) + vx_setall_s32(d) - (s + s)); - i += v_int32::nlanes; + v_pack_store(dst + i, v_sub(v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), vx_setall_s32(d)), v_add(s, s))); + i += VTraits::vlanes(); } } #if CV_NEON @@ -1347,23 +1344,23 @@ struct SymmColumnSmallVec_32s16s else { v_float32 k0 = vx_setall_f32(ky[0]), k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S0 + i) + vx_load(S2 + i)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))), - v_round(v_muladd(v_cvt_f32(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + v_int32::nlanes)), k0, df4))))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S0 + i + 2*v_int32::nlanes) + vx_load(S2 + i + 2*v_int32::nlanes)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 2*v_int32::nlanes)), k0, df4))), - v_round(v_muladd(v_cvt_f32(vx_load(S0 + i + 3*v_int32::nlanes) + vx_load(S2 + i + 3*v_int32::nlanes)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 3*v_int32::nlanes)), k0, df4))))); + v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))), + v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + VTraits::vlanes())), k0, df4))))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + 2 * VTraits::vlanes()), vx_load(S2 + i + 2 * VTraits::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 2*VTraits::vlanes())), k0, df4))), + v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + 3 * VTraits::vlanes()), vx_load(S2 + i + 3 * VTraits::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + 3*VTraits::vlanes())), k0, df4))))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S0 + i) + vx_load(S2 + i)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))), - v_round(v_muladd(v_cvt_f32(vx_load(S0 + i + v_int32::nlanes) + vx_load(S2 + i + v_int32::nlanes)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + v_int32::nlanes)), k0, df4))))); - i += v_int16::nlanes; + v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4))), + v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i + VTraits::vlanes()), vx_load(S2 + i + VTraits::vlanes()))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i + VTraits::vlanes())), k0, df4))))); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(vx_load(S0 + i) + vx_load(S2 + i)), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4)))); - i += v_int32::nlanes; + v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(v_add(vx_load(S0 + i), vx_load(S2 + i))), k1, v_muladd(v_cvt_f32(vx_load(S1 + i)), k0, df4)))); + i += VTraits::vlanes(); } } } @@ -1373,42 +1370,42 @@ struct SymmColumnSmallVec_32s16s { if( ky[1] < 0 ) std::swap(S0, S2); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_store(dst + i, v_pack(vx_load(S2 + i) - vx_load(S0 + i), vx_load(S2 + i + v_int32::nlanes) - vx_load(S0 + i + v_int32::nlanes)) + d8); - v_store(dst + i + v_int16::nlanes, v_pack(vx_load(S2 + i + 2*v_int32::nlanes) - vx_load(S0 + i + 2*v_int32::nlanes), vx_load(S2 + i + 3*v_int32::nlanes) - vx_load(S0 + i + 3*v_int32::nlanes)) + d8); + v_store(dst + i, v_add(v_pack(v_sub(vx_load(S2 + i), vx_load(S0 + i)), v_sub(vx_load(S2 + i + VTraits::vlanes()), vx_load(S0 + i + VTraits::vlanes()))), d8)); + v_store(dst + i + VTraits::vlanes(), v_add(v_pack(v_sub(vx_load(S2 + i + 2 * VTraits::vlanes()), vx_load(S0 + i + 2 * VTraits::vlanes())), v_sub(vx_load(S2 + i + 3 * VTraits::vlanes()), vx_load(S0 + i + 3 * VTraits::vlanes()))), d8)); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_pack(vx_load(S2 + i) - vx_load(S0 + i), vx_load(S2 + i + v_int32::nlanes) - vx_load(S0 + i + v_int32::nlanes)) + d8); - i += v_int16::nlanes; + v_store(dst + i, v_add(v_pack(v_sub(vx_load(S2 + i), vx_load(S0 + i)), v_sub(vx_load(S2 + i + VTraits::vlanes()), vx_load(S0 + i + VTraits::vlanes()))), d8)); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_pack_store(dst + i, vx_load(S2 + i) - vx_load(S0 + i) + vx_setall_s32(d)); - i += v_int32::nlanes; + v_pack_store(dst + i, v_add(v_sub(vx_load(S2 + i), vx_load(S0 + i)), vx_setall_s32(d))); + i += VTraits::vlanes(); } } else { v_float32 k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S2 + i) - vx_load(S0 + i)), k1, df4)), - v_round(v_muladd(v_cvt_f32(vx_load(S2 + i + v_int32::nlanes) - vx_load(S0 + i + v_int32::nlanes)), k1, df4)))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S2 + i + 2*v_int32::nlanes) - vx_load(S0 + i + 2*v_int32::nlanes)), k1, df4)), - v_round(v_muladd(v_cvt_f32(vx_load(S2 + i + 3*v_int32::nlanes) - vx_load(S0 + i + 3*v_int32::nlanes)), k1, df4)))); + v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4)), + v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + VTraits::vlanes()), vx_load(S0 + i + VTraits::vlanes()))), k1, df4)))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + 2 * VTraits::vlanes()), vx_load(S0 + i + 2 * VTraits::vlanes()))), k1, df4)), + v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + 3 * VTraits::vlanes()), vx_load(S0 + i + 3 * VTraits::vlanes()))), k1, df4)))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S2 + i) - vx_load(S0 + i)), k1, df4)), - v_round(v_muladd(v_cvt_f32(vx_load(S2 + i + v_int32::nlanes) - vx_load(S0 + i + v_int32::nlanes)), k1, df4)))); - i += v_int16::nlanes; + v_store(dst + i, v_pack(v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4)), + v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i + VTraits::vlanes()), vx_load(S0 + i + VTraits::vlanes()))), k1, df4)))); + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(vx_load(S2 + i) - vx_load(S0 + i)), k1, df4))); - i += v_int32::nlanes; + v_pack_store(dst + i, v_round(v_muladd(v_cvt_f32(v_sub(vx_load(S2 + i), vx_load(S0 + i))), k1, df4))); + i += VTraits::vlanes(); } } } @@ -1440,7 +1437,7 @@ struct RowVec_16s32f const float* _kx = kernel.ptr(); width *= cn; - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { const short* src = (const short*)_src + i; v_float32 s0 = vx_setzero_f32(); @@ -1451,18 +1448,18 @@ struct RowVec_16s32f { v_float32 f = vx_setall_f32(_kx[k]); v_int16 xl = vx_load(src); - v_int16 xh = vx_load(src + v_int16::nlanes); + v_int16 xh = vx_load(src + VTraits::vlanes()); s0 = v_muladd(v_cvt_f32(v_expand_low(xl)), f, s0); s1 = v_muladd(v_cvt_f32(v_expand_high(xl)), f, s1); s2 = v_muladd(v_cvt_f32(v_expand_low(xh)), f, s2); s3 = v_muladd(v_cvt_f32(v_expand_high(xh)), f, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { const short* src = (const short*)_src + i; v_float32 s0 = vx_setzero_f32(); @@ -1475,17 +1472,17 @@ struct RowVec_16s32f s1 = v_muladd(v_cvt_f32(v_expand_high(x)), f, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += v_int16::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { const short* src = (const short*)_src + i; v_float32 s0 = vx_setzero_f32(); for( k = 0; k < _ksize; k++, src += cn ) s0 = v_muladd(v_cvt_f32(vx_load_expand(src)), vx_setall_f32(_kx[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } return i; } @@ -1524,92 +1521,92 @@ struct SymmColumnVec_32f16s { v_float32 k0 = vx_setall_f32(ky[0]); v_float32 k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), k0, d4); - v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*v_float32::nlanes), k0, d4); - v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*v_float32::nlanes), k0, d4); - s0 = v_muladd(vx_load(src[1] + i) + vx_load(src[-1] + i), k1, s0); - s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) + vx_load(src[-1] + i + v_float32::nlanes), k1, s1); - s2 = v_muladd(vx_load(src[1] + i + 2*v_float32::nlanes) + vx_load(src[-1] + i + 2*v_float32::nlanes), k1, s2); - s3 = v_muladd(vx_load(src[1] + i + 3*v_float32::nlanes) + vx_load(src[-1] + i + 3*v_float32::nlanes), k1, s3); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), k0, d4); + v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits::vlanes()), k0, d4); + v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits::vlanes()), k0, d4); + s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0); + s1 = v_muladd(v_add(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, s1); + s2 = v_muladd(v_add(vx_load(src[1] + i + 2 * VTraits::vlanes()), vx_load(src[-1] + i + 2 * VTraits::vlanes())), k1, s2); + s3 = v_muladd(v_add(vx_load(src[1] + i + 3 * VTraits::vlanes()), vx_load(src[-1] + i + 3 * VTraits::vlanes())), k1, s3); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) + vx_load(src[-k] + i + v_float32::nlanes), k2, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes) + vx_load(src[-k] + i + 2*v_float32::nlanes), k2, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes) + vx_load(src[-k] + i + 3*v_float32::nlanes), k2, s3); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); + s2 = v_muladd(v_add(vx_load(src[k] + i + 2 * VTraits::vlanes()), vx_load(src[-k] + i + 2 * VTraits::vlanes())), k2, s2); + s3 = v_muladd(v_add(vx_load(src[k] + i + 3 * VTraits::vlanes()), vx_load(src[-k] + i + 3 * VTraits::vlanes())), k2, s3); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(s2), v_round(s3))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(s2), v_round(s3))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), k0, d4); - s0 = v_muladd(vx_load(src[1] + i) + vx_load(src[-1] + i), k1, s0); - s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) + vx_load(src[-1] + i + v_float32::nlanes), k1, s1); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), k0, d4); + s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0); + s1 = v_muladd(v_add(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, s1); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) + vx_load(src[-k] + i + v_float32::nlanes), k2, s1); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_int16::nlanes; + i += VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - s0 = v_muladd(vx_load(src[1] + i) + vx_load(src[-1] + i), k1, s0); + s0 = v_muladd(v_add(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, s0); for( k = 2; k <= ksize2; k++ ) - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), vx_setall_f32(ky[k]), s0); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0); v_pack_store(dst + i, v_round(s0)); - i += v_float32::nlanes; + i += VTraits::vlanes(); } } else { v_float32 k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 2*v_int16::nlanes; i += 2*v_int16::nlanes ) + for( ; i <= width - 2*VTraits::vlanes(); i += 2*VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); - v_float32 s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) - vx_load(src[-1] + i + v_float32::nlanes), k1, d4); - v_float32 s2 = v_muladd(vx_load(src[1] + i + 2*v_float32::nlanes) - vx_load(src[-1] + i + 2*v_float32::nlanes), k1, d4); - v_float32 s3 = v_muladd(vx_load(src[1] + i + 3*v_float32::nlanes) - vx_load(src[-1] + i + 3*v_float32::nlanes), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); + v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, d4); + v_float32 s2 = v_muladd(v_sub(vx_load(src[1] + i + 2 * VTraits::vlanes()), vx_load(src[-1] + i + 2 * VTraits::vlanes())), k1, d4); + v_float32 s3 = v_muladd(v_sub(vx_load(src[1] + i + 3 * VTraits::vlanes()), vx_load(src[-1] + i + 3 * VTraits::vlanes())), k1, d4); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) - vx_load(src[-k] + i + v_float32::nlanes), k2, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes) - vx_load(src[-k] + i + 2*v_float32::nlanes), k2, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes) - vx_load(src[-k] + i + 3*v_float32::nlanes), k2, s3); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); + s2 = v_muladd(v_sub(vx_load(src[k] + i + 2 * VTraits::vlanes()), vx_load(src[-k] + i + 2 * VTraits::vlanes())), k2, s2); + s3 = v_muladd(v_sub(vx_load(src[k] + i + 3 * VTraits::vlanes()), vx_load(src[-k] + i + 3 * VTraits::vlanes())), k2, s3); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(s2), v_round(s3))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(s2), v_round(s3))); } - if( i <= width - v_int16::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); - v_float32 s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) - vx_load(src[-1] + i + v_float32::nlanes), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); + v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, d4); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) - vx_load(src[-k] + i + v_float32::nlanes), k2, s1); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_int16::nlanes; + i += VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); for( k = 2; k <= ksize2; k++ ) - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), vx_setall_f32(ky[k]), s0); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0); v_pack_store(dst + i, v_round(s0)); - i += v_float32::nlanes; + i += VTraits::vlanes(); } } @@ -1682,52 +1679,52 @@ struct RowVec_32f } #endif v_float32 k0 = vx_setall_f32(_kx[0]); - for( ; i <= width - 4*v_float32::nlanes; i += 4*v_float32::nlanes ) + for( ; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { const float* src = src0 + i; - v_float32 s0 = vx_load(src) * k0; - v_float32 s1 = vx_load(src + v_float32::nlanes) * k0; - v_float32 s2 = vx_load(src + 2*v_float32::nlanes) * k0; - v_float32 s3 = vx_load(src + 3*v_float32::nlanes) * k0; + v_float32 s0 = v_mul(vx_load(src), k0); + v_float32 s1 = v_mul(vx_load(src + VTraits::vlanes()), k0); + v_float32 s2 = v_mul(vx_load(src + 2 * VTraits::vlanes()), k0); + v_float32 s3 = v_mul(vx_load(src + 3 * VTraits::vlanes()), k0); src += cn; for( k = 1; k < _ksize; k++, src += cn ) { v_float32 k1 = vx_setall_f32(_kx[k]); s0 = v_muladd(vx_load(src), k1, s0); - s1 = v_muladd(vx_load(src + v_float32::nlanes), k1, s1); - s2 = v_muladd(vx_load(src + 2*v_float32::nlanes), k1, s2); - s3 = v_muladd(vx_load(src + 3*v_float32::nlanes), k1, s3); + s1 = v_muladd(vx_load(src + VTraits::vlanes()), k1, s1); + s2 = v_muladd(vx_load(src + 2*VTraits::vlanes()), k1, s2); + s3 = v_muladd(vx_load(src + 3*VTraits::vlanes()), k1, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*v_float32::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { const float* src = src0 + i; - v_float32 s0 = vx_load(src) * k0; - v_float32 s1 = vx_load(src + v_float32::nlanes) * k0; + v_float32 s0 = v_mul(vx_load(src), k0); + v_float32 s1 = v_mul(vx_load(src + VTraits::vlanes()), k0); src += cn; for( k = 1; k < _ksize; k++, src += cn ) { v_float32 k1 = vx_setall_f32(_kx[k]); s0 = v_muladd(vx_load(src), k1, s0); - s1 = v_muladd(vx_load(src + v_float32::nlanes), k1, s1); + s1 = v_muladd(vx_load(src + VTraits::vlanes()), k1, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += 2*v_float32::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { const float* src = src0 + i; - v_float32 s0 = vx_load(src) * k0; + v_float32 s0 = v_mul(vx_load(src), k0); src += cn; for( k = 1; k < _ksize; k++, src += cn ) s0 = v_muladd(vx_load(src), vx_setall_f32(_kx[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } return i; } @@ -1806,28 +1803,28 @@ struct SymmRowSmallVec_32f { #if CV_FMA3 || CV_AVX2 v_float32 k0 = vx_setall_f32(kx[0]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src), k0, vx_load(src - cn) + vx_load(src + cn))); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(vx_load(src), k0, v_add(vx_load(src - cn), vx_load(src + cn)))); #else if( kx[0] > 0 ) - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_float32 x = vx_load(src); - v_store(dst + i, vx_load(src - cn) + vx_load(src + cn) + (x + x)); + v_store(dst + i, v_add(vx_load(src - cn), vx_load(src + cn), x , x)); } else - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_float32 x = vx_load(src); - v_store(dst + i, vx_load(src - cn) + vx_load(src + cn) - (x + x)); + v_store(dst + i, v_sub(v_add(vx_load(src - cn), vx_load(src + cn)), v_add(x, x))); } #endif } else { v_float32 k0 = vx_setall_f32(kx[0]), k1 = vx_setall_f32(kx[1]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src), k0, (vx_load(src - cn) + vx_load(src + cn)) * k1)); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(vx_load(src), k0, v_mul(v_add(vx_load(src - cn), vx_load(src + cn)), k1))); } } else if( _ksize == 5 ) @@ -1836,21 +1833,21 @@ struct SymmRowSmallVec_32f { #if CV_FMA3 || CV_AVX2 v_float32 k0 = vx_setall_f32(-2); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src), k0, vx_load(src - 2*cn) + vx_load(src + 2*cn))); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(vx_load(src), k0, v_add(vx_load(src - 2 * cn), vx_load(src + 2 * cn)))); #else - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) { v_float32 x = vx_load(src); - v_store(dst + i, vx_load(src - 2*cn) + vx_load(src + 2*cn) - (x + x)); + v_store(dst + i, v_sub(v_add(vx_load(src - 2*cn), vx_load(src + 2*cn)), v_add(x, x))); } #endif } else { v_float32 k0 = vx_setall_f32(kx[0]), k1 = vx_setall_f32(kx[1]), k2 = vx_setall_f32(kx[2]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src + 2*cn) + vx_load(src - 2*cn), k2, v_muladd(vx_load(src), k0, (vx_load(src - cn) + vx_load(src + cn)) * k1))); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(v_add(vx_load(src + 2 * cn), vx_load(src - 2 * cn)), k2, v_muladd(vx_load(src), k0, v_mul(v_add(vx_load(src - cn), vx_load(src + cn)), k1)))); } } } @@ -1859,20 +1856,20 @@ struct SymmRowSmallVec_32f if( _ksize == 3 ) { if( kx[0] == 0 && kx[1] == 1 ) - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, vx_load(src + cn) - vx_load(src - cn)); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_sub(vx_load(src + cn), vx_load(src - cn))); else { v_float32 k1 = vx_setall_f32(kx[1]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, (vx_load(src + cn) - vx_load(src - cn)) * k1); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_mul(v_sub(vx_load(src + cn), vx_load(src - cn)), k1)); } } else if( _ksize == 5 ) { v_float32 k1 = vx_setall_f32(kx[1]), k2 = vx_setall_f32(kx[2]); - for ( ; i <= width - v_float32::nlanes; i += v_float32::nlanes, src += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(src + 2*cn) - vx_load(src - 2*cn), k2, (vx_load(src + cn) - vx_load(src - cn)) * k1)); + for ( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes(), src += VTraits::vlanes() ) + v_store(dst + i, v_muladd(v_sub(vx_load(src + 2 * cn), vx_load(src - 2 * cn)), k2, v_mul(v_sub(vx_load(src + cn), vx_load(src - cn)), k1))); } } return i; @@ -1961,46 +1958,46 @@ struct SymmColumnVec_32f #endif const v_float32 d4 = vx_setall_f32(delta); const v_float32 k0 = vx_setall_f32(ky[0]); - for( ; i <= width - 4*v_float32::nlanes; i += 4*v_float32::nlanes ) + for( ; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), k0, d4); - v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*v_float32::nlanes), k0, d4); - v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*v_float32::nlanes), k0, d4); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), k0, d4); + v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits::vlanes()), k0, d4); + v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits::vlanes()), k0, d4); for( k = 1; k <= ksize2; k++ ) { v_float32 k1 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), k1, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) + vx_load(src[-k] + i + v_float32::nlanes), k1, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes) + vx_load(src[-k] + i + 2*v_float32::nlanes), k1, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes) + vx_load(src[-k] + i + 3*v_float32::nlanes), k1, s3); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k1, s0); + s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k1, s1); + s2 = v_muladd(v_add(vx_load(src[k] + i + 2 * VTraits::vlanes()), vx_load(src[-k] + i + 2 * VTraits::vlanes())), k1, s2); + s3 = v_muladd(v_add(vx_load(src[k] + i + 3 * VTraits::vlanes()), vx_load(src[-k] + i + 3 * VTraits::vlanes())), k1, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*v_float32::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), k0, d4); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), k0, d4); for( k = 1; k <= ksize2; k++ ) { v_float32 k1 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), k1, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) + vx_load(src[-k] + i + v_float32::nlanes), k1, s1); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), k1, s0); + s1 = v_muladd(v_add(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k1, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += 2*v_float32::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), k0, d4); for( k = 1; k <= ksize2; k++ ) - s0 = v_muladd(vx_load(src[k] + i) + vx_load(src[-k] + i), vx_setall_f32(ky[k]), s0); + s0 = v_muladd(v_add(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } } else @@ -2042,46 +2039,46 @@ struct SymmColumnVec_32f #endif const v_float32 d4 = vx_setall_f32(delta); const v_float32 k1 = vx_setall_f32(ky[1]); - for( ; i <= width - 4*v_float32::nlanes; i += 4*v_float32::nlanes ) + for( ; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); - v_float32 s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) - vx_load(src[-1] + i + v_float32::nlanes), k1, d4); - v_float32 s2 = v_muladd(vx_load(src[1] + i + 2*v_float32::nlanes) - vx_load(src[-1] + i + 2*v_float32::nlanes), k1, d4); - v_float32 s3 = v_muladd(vx_load(src[1] + i + 3*v_float32::nlanes) - vx_load(src[-1] + i + 3*v_float32::nlanes), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); + v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, d4); + v_float32 s2 = v_muladd(v_sub(vx_load(src[1] + i + 2 * VTraits::vlanes()), vx_load(src[-1] + i + 2 * VTraits::vlanes())), k1, d4); + v_float32 s3 = v_muladd(v_sub(vx_load(src[1] + i + 3 * VTraits::vlanes()), vx_load(src[-1] + i + 3 * VTraits::vlanes())), k1, d4); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) - vx_load(src[-k] + i + v_float32::nlanes), k2, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes) - vx_load(src[-k] + i + 2*v_float32::nlanes), k2, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes) - vx_load(src[-k] + i + 3*v_float32::nlanes), k2, s3); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); + s2 = v_muladd(v_sub(vx_load(src[k] + i + 2 * VTraits::vlanes()), vx_load(src[-k] + i + 2 * VTraits::vlanes())), k2, s2); + s3 = v_muladd(v_sub(vx_load(src[k] + i + 3 * VTraits::vlanes()), vx_load(src[-k] + i + 3 * VTraits::vlanes())), k2, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*v_float32::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); - v_float32 s1 = v_muladd(vx_load(src[1] + i + v_float32::nlanes) - vx_load(src[-1] + i + v_float32::nlanes), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); + v_float32 s1 = v_muladd(v_sub(vx_load(src[1] + i + VTraits::vlanes()), vx_load(src[-1] + i + VTraits::vlanes())), k1, d4); for( k = 2; k <= ksize2; k++ ) { v_float32 k2 = vx_setall_f32(ky[k]); - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), k2, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes) - vx_load(src[-k] + i + v_float32::nlanes), k2, s1); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), k2, s0); + s1 = v_muladd(v_sub(vx_load(src[k] + i + VTraits::vlanes()), vx_load(src[-k] + i + VTraits::vlanes())), k2, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += 2*v_float32::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { - v_float32 s0 = v_muladd(vx_load(src[1] + i) - vx_load(src[-1] + i), k1, d4); + v_float32 s0 = v_muladd(v_sub(vx_load(src[1] + i), vx_load(src[-1] + i)), k1, d4); for( k = 2; k <= ksize2; k++ ) - s0 = v_muladd(vx_load(src[k] + i) - vx_load(src[-k] + i), vx_setall_f32(ky[k]), s0); + s0 = v_muladd(v_sub(vx_load(src[k] + i), vx_load(src[-k] + i)), vx_setall_f32(ky[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } } return i; @@ -2123,28 +2120,28 @@ struct SymmColumnSmallVec_32f { #if CV_FMA3 || CV_AVX2 v_float32 k0 = vx_setall_f32(ky[0]); - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(S1 + i), k0, vx_load(S0 + i) + vx_load(S2 + i) + d4)); + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) + v_store(dst + i, v_muladd(vx_load(S1 + i), k0, v_add(v_add(vx_load(S0 + i), vx_load(S2 + i)), d4))); #else if(ky[0] > 0) - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 x = vx_load(S1 + i); - v_store(dst + i, vx_load(S0 + i) + vx_load(S2 + i) + d4 + (x + x)); + v_store(dst + i, v_add(vx_load(S0 + i), vx_load(S2 + i), d4, x, x)); } else - for( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_float32 x = vx_load(S1 + i); - v_store(dst + i, vx_load(S0 + i) + vx_load(S2 + i) + d4 - (x + x)); + v_store(dst + i, v_sub(v_add(vx_load(S0 + i), vx_load(S2 + i), d4), v_add(x, x))); } #endif } else { v_float32 k0 = vx_setall_f32(ky[0]), k1 = vx_setall_f32(ky[1]); - for ( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(S0 + i) + vx_load(S2 + i), k1, v_muladd(vx_load(S1 + i), k0, d4))); + for ( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) + v_store(dst + i, v_muladd(v_add(vx_load(S0 + i), vx_load(S2 + i)), k1, v_muladd(vx_load(S1 + i), k0, d4))); } } else @@ -2153,14 +2150,14 @@ struct SymmColumnSmallVec_32f { if( ky[1] < 0 ) std::swap(S0, S2); - for ( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) - v_store(dst + i, vx_load(S2 + i) - vx_load(S0 + i) + d4); + for ( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) + v_store(dst + i, v_add(v_sub(vx_load(S2 + i), vx_load(S0 + i)), d4)); } else { v_float32 k1 = vx_setall_f32(ky[1]); - for ( ; i <= width - v_float32::nlanes; i += v_float32::nlanes ) - v_store(dst + i, v_muladd(vx_load(S2 + i) - vx_load(S0 + i), k1, d4)); + for ( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) + v_store(dst + i, v_muladd(v_sub(vx_load(S2 + i), vx_load(S0 + i)), k1, d4)); } } return i; @@ -2199,7 +2196,7 @@ struct FilterVec_8u v_float32 d4 = vx_setall_f32(delta); v_float32 f0 = vx_setall_f32(kf[0]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_uint16 xl, xh; v_expand(vx_load(src[0] + i), xl, xh); @@ -2223,7 +2220,7 @@ struct FilterVec_8u } v_store(dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint32 x0, x1; v_expand(vx_load_expand(src[0] + i), x0, x1); @@ -2237,21 +2234,21 @@ struct FilterVec_8u s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(x1)), f, s1); } v_pack_u_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_uint16::nlanes; + i += VTraits::vlanes(); } #if CV_SIMD_WIDTH > 16 - while( i <= width - v_int32x4::nlanes ) + while( i <= width - 4 /*v_int32x4::nlanes*/ ) #else - if( i <= width - v_int32x4::nlanes ) + if( i <= width - v_int32::nlanes ) #endif { - v_float32x4 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand_q(src[0] + i))), v_setall_f32(kf[0]), v_setall_f32(delta)); + v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[0] + i))), vx_setall_f32(kf[0]), vx_setall_f32(delta)); for( k = 1; k < nz; k++ ) - s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_load_expand_q(src[k] + i))), v_setall_f32(kf[k]), s0); - v_int32x4 s32 = v_round(s0); - v_int16x8 s16 = v_pack(s32, s32); - *(unaligned_int*)(dst + i) = v_reinterpret_as_s32(v_pack_u(s16, s16)).get0(); - i += v_int32x4::nlanes; + s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[k] + i))), vx_setall_f32(kf[k]), s0); + v_int32 s32 = v_round(s0); + v_int16 s16 = v_pack(s32, s32); + *(unaligned_int*)(dst + i) = v_get0(v_reinterpret_as_s32(v_pack_u(s16, s16))); + i += 4 /*v_int32x4::nlanes*/ ; } return i; } @@ -2286,7 +2283,7 @@ struct FilterVec_8u16s v_float32 d4 = vx_setall_f32(delta); v_float32 f0 = vx_setall_f32(kf[0]); - for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + for( ; i <= width - VTraits::vlanes(); i += VTraits::vlanes() ) { v_uint16 xl, xh; v_expand(vx_load(src[0] + i), xl, xh); @@ -2304,9 +2301,9 @@ struct FilterVec_8u16s s3 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(xh))), f, s3); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - v_store(dst + i + v_int16::nlanes, v_pack(v_round(s2), v_round(s3))); + v_store(dst + i + VTraits::vlanes(), v_pack(v_round(s2), v_round(s3))); } - if( i <= width - v_uint16::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_uint16 x = vx_load_expand(src[0] + i); v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_low(x))), f0, d4); @@ -2319,15 +2316,15 @@ struct FilterVec_8u16s s1 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(v_expand_high(x))), f, s1); } v_store(dst + i, v_pack(v_round(s0), v_round(s1))); - i += v_uint16::nlanes; + i += VTraits::vlanes(); } - if( i <= width - v_int32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[0] + i))), f0, d4); for( k = 1; k < nz; k++ ) s0 = v_muladd(v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src[k] + i))), vx_setall_f32(kf[k]), s0); v_pack_store(dst + i, v_round(s0)); - i += v_int32::nlanes; + i += VTraits::vlanes(); } return i; } @@ -2360,46 +2357,46 @@ struct FilterVec_32f v_float32 d4 = vx_setall_f32(delta); v_float32 f0 = vx_setall_f32(kf[0]); - for( ; i <= width - 4*v_float32::nlanes; i += 4*v_float32::nlanes ) + for( ; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), f0, d4); - v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*v_float32::nlanes), f0, d4); - v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*v_float32::nlanes), f0, d4); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), f0, d4); + v_float32 s2 = v_muladd(vx_load(src[0] + i + 2*VTraits::vlanes()), f0, d4); + v_float32 s3 = v_muladd(vx_load(src[0] + i + 3*VTraits::vlanes()), f0, d4); for( k = 1; k < nz; k++ ) { v_float32 f1 = vx_setall_f32(kf[k]); s0 = v_muladd(vx_load(src[k] + i), f1, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes), f1, s1); - s2 = v_muladd(vx_load(src[k] + i + 2*v_float32::nlanes), f1, s2); - s3 = v_muladd(vx_load(src[k] + i + 3*v_float32::nlanes), f1, s3); + s1 = v_muladd(vx_load(src[k] + i + VTraits::vlanes()), f1, s1); + s2 = v_muladd(vx_load(src[k] + i + 2*VTraits::vlanes()), f1, s2); + s3 = v_muladd(vx_load(src[k] + i + 3*VTraits::vlanes()), f1, s3); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - v_store(dst + i + 2*v_float32::nlanes, s2); - v_store(dst + i + 3*v_float32::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*v_float32::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4); - v_float32 s1 = v_muladd(vx_load(src[0] + i + v_float32::nlanes), f0, d4); + v_float32 s1 = v_muladd(vx_load(src[0] + i + VTraits::vlanes()), f0, d4); for( k = 1; k < nz; k++ ) { v_float32 f1 = vx_setall_f32(kf[k]); s0 = v_muladd(vx_load(src[k] + i), f1, s0); - s1 = v_muladd(vx_load(src[k] + i + v_float32::nlanes), f1, s1); + s1 = v_muladd(vx_load(src[k] + i + VTraits::vlanes()), f1, s1); } v_store(dst + i, s0); - v_store(dst + i + v_float32::nlanes, s1); - i += 2*v_float32::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - v_float32::nlanes ) + if( i <= width - VTraits::vlanes() ) { v_float32 s0 = v_muladd(vx_load(src[0] + i), f0, d4); for( k = 1; k < nz; k++ ) s0 = v_muladd(vx_load(src[k] + i), vx_setall_f32(kf[k]), s0); v_store(dst + i, s0); - i += v_float32::nlanes; + i += VTraits::vlanes(); } return i; } diff --git a/modules/imgproc/src/histogram.cpp b/modules/imgproc/src/histogram.cpp index 426cbcb3ca..23fe2bb42a 100644 --- a/modules/imgproc/src/histogram.cpp +++ b/modules/imgproc/src/histogram.cpp @@ -2066,13 +2066,13 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) } else if( method == cv::HISTCMP_CORREL ) { -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) v_float64 v_s1 = vx_setzero_f64(); v_float64 v_s2 = vx_setzero_f64(); v_float64 v_s11 = vx_setzero_f64(); v_float64 v_s12 = vx_setzero_f64(); v_float64 v_s22 = vx_setzero_f64(); - for ( ; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for ( ; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_a = vx_load(h1 + j); v_float32 v_b = vx_load(h2 + j); @@ -2083,8 +2083,8 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) v_s12 = v_muladd(v_ad, v_bd, v_s12); v_s11 = v_muladd(v_ad, v_ad, v_s11); v_s22 = v_muladd(v_bd, v_bd, v_s22); - v_s1 += v_ad; - v_s2 += v_bd; + v_s1 = v_add(v_s1, v_ad); + v_s2 = v_add(v_s2, v_bd); // 2-3 v_ad = v_cvt_f64_high(v_a); @@ -2092,8 +2092,8 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) v_s12 = v_muladd(v_ad, v_bd, v_s12); v_s11 = v_muladd(v_ad, v_ad, v_s11); v_s22 = v_muladd(v_bd, v_bd, v_s22); - v_s1 += v_ad; - v_s2 += v_bd; + v_s1 = v_add(v_s1, v_ad); + v_s2 = v_add(v_s2, v_bd); } s12 += v_reduce_sum(v_s12); s11 += v_reduce_sum(v_s11); @@ -2137,12 +2137,12 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) } else if( method == cv::HISTCMP_INTERSECT ) { -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) v_float64 v_result = vx_setzero_f64(); - for ( ; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for ( ; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_src = v_min(vx_load(h1 + j), vx_load(h2 + j)); - v_result += v_cvt_f64(v_src) + v_cvt_f64_high(v_src); + v_result = v_add(v_result, v_add(v_cvt_f64(v_src), v_cvt_f64_high(v_src))); } result += v_reduce_sum(v_result); #elif CV_SIMD @@ -2159,26 +2159,26 @@ double cv::compareHist( InputArray _H1, InputArray _H2, int method ) } else if( method == cv::HISTCMP_BHATTACHARYYA ) { -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) v_float64 v_s1 = vx_setzero_f64(); v_float64 v_s2 = vx_setzero_f64(); v_float64 v_result = vx_setzero_f64(); - for ( ; j <= len - v_float32::nlanes; j += v_float32::nlanes) + for ( ; j <= len - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_a = vx_load(h1 + j); v_float32 v_b = vx_load(h2 + j); v_float64 v_ad = v_cvt_f64(v_a); v_float64 v_bd = v_cvt_f64(v_b); - v_s1 += v_ad; - v_s2 += v_bd; - v_result += v_sqrt(v_ad * v_bd); + v_s1 = v_add(v_s1, v_ad); + v_s2 = v_add(v_s2, v_bd); + v_result = v_add(v_result, v_sqrt(v_mul(v_ad, v_bd))); v_ad = v_cvt_f64_high(v_a); v_bd = v_cvt_f64_high(v_b); - v_s1 += v_ad; - v_s2 += v_bd; - v_result += v_sqrt(v_ad * v_bd); + v_s1 = v_add(v_s1, v_ad); + v_s2 = v_add(v_s2, v_bd); + v_result = v_add(v_result, v_sqrt(v_mul(v_ad, v_bd))); } s1 += v_reduce_sum(v_s1); s2 += v_reduce_sum(v_s2); diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index 44fe25bdd9..8b2269253f 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -1156,13 +1156,13 @@ public: for(; x < numCols; ++x ) { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { v_uint8 v_zero = vx_setzero_u8(); - for(; x <= numCols - 2*v_uint8::nlanes; x += 2*v_uint8::nlanes) { - v_uint8 v_edge1 = (vx_load(edgeData + x ) != v_zero); - v_uint8 v_edge2 = (vx_load(edgeData + x + v_uint8::nlanes) != v_zero); + for(; x <= numCols - 2*VTraits::vlanes(); x += 2*VTraits::vlanes()) { + v_uint8 v_edge1 = (v_ne(vx_load(edgeData + x), v_zero)); + v_uint8 v_edge2 = (v_ne(vx_load(edgeData + x + VTraits::vlanes()), v_zero)); if(v_check_any(v_edge1)) { @@ -1172,7 +1172,7 @@ public: if(v_check_any(v_edge2)) { - x += v_uint8::nlanes + v_scan_forward(v_edge2); + x += VTraits::vlanes() + v_scan_forward(v_edge2); goto _next_step; } } @@ -1183,7 +1183,7 @@ public: if(x == numCols) continue; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) _next_step: #endif float vx, vy; @@ -1514,7 +1514,7 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Po int nzCount = 0; const Point* nz_ = &nz[0]; int j = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { const v_float32 v_minRadius2 = vx_setall_f32(minRadius2); const v_float32 v_maxRadius2 = vx_setall_f32(maxRadius2); @@ -1522,9 +1522,9 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Po v_float32 v_curCenterX = vx_setall_f32(curCenter.x); v_float32 v_curCenterY = vx_setall_f32(curCenter.y); - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rbuf[v_float32::nlanes]; - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) rmask[v_int32::nlanes]; - for(; j <= nzSz - v_float32::nlanes; j += v_float32::nlanes) + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rbuf[VTraits::max_nlanes]; + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) rmask[VTraits::max_nlanes]; + for(; j <= nzSz - VTraits::vlanes(); j += VTraits::vlanes()) { v_float32 v_nzX, v_nzY; v_load_deinterleave((const float*)&nz_[j], v_nzX, v_nzY); // FIXIT use proper datatype @@ -1532,16 +1532,16 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Po v_float32 v_x = v_cvt_f32(v_reinterpret_as_s32(v_nzX)); v_float32 v_y = v_cvt_f32(v_reinterpret_as_s32(v_nzY)); - v_float32 v_dx = v_x - v_curCenterX; - v_float32 v_dy = v_y - v_curCenterY; + v_float32 v_dx = v_sub(v_x, v_curCenterX); + v_float32 v_dy = v_sub(v_y, v_curCenterY); - v_float32 v_r2 = (v_dx * v_dx) + (v_dy * v_dy); - v_float32 vmask = (v_minRadius2 <= v_r2) & (v_r2 <= v_maxRadius2); + v_float32 v_r2 = v_add(v_mul(v_dx, v_dx), v_mul(v_dy, v_dy)); + v_float32 vmask = v_and(v_le(v_minRadius2, v_r2), v_le(v_r2, v_maxRadius2)); if (v_check_any(vmask)) { v_store_aligned(rmask, v_reinterpret_as_s32(vmask)); v_store_aligned(rbuf, v_r2); - for (int i = 0; i < v_int32::nlanes; ++i) + for (int i = 0; i < VTraits::vlanes(); ++i) if (rmask[i]) ddata[nzCount++] = rbuf[i]; } } @@ -1573,13 +1573,13 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Poi const Range xOuter = Range(std::max(int(curCenter.x - rOuter), 0), std::min(int(curCenter.x + rOuter), positions.cols)); const Range yOuter = Range(std::max(int(curCenter.y - rOuter), 0), std::min(int(curCenter.y + rOuter), positions.rows)); -#if CV_SIMD - float v_seq[v_float32::nlanes]; - for (int i = 0; i < v_float32::nlanes; ++i) +#if (CV_SIMD || CV_SIMD_SCALABLE) + float v_seq[VTraits::max_nlanes]; + for (int i = 0; i < VTraits::vlanes(); ++i) v_seq[i] = (float)i; const v_float32 v_minRadius2 = vx_setall_f32(minRadius2); const v_float32 v_maxRadius2 = vx_setall_f32(maxRadius2); - const v_float32 v_curCenterX_0123 = vx_setall_f32(curCenter.x) - vx_load(v_seq); + const v_float32 v_curCenterX_0123 = v_sub(vx_setall_f32(curCenter.x), vx_load(v_seq)); #endif for (int y = yOuter.start; y < yOuter.end; y++) @@ -1589,27 +1589,27 @@ inline int HoughCircleEstimateRadiusInvoker::filterCircles(const Poi float dy2 = dy * dy; int x = xOuter.start; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { const v_float32 v_dy2 = vx_setall_f32(dy2); const v_uint32 v_zero_u32 = vx_setall_u32(0); - float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rbuf[v_float32::nlanes]; - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) rmask[v_int32::nlanes]; - for (; x <= xOuter.end - v_float32::nlanes; x += v_float32::nlanes) + float CV_DECL_ALIGNED(CV_SIMD_WIDTH) rbuf[VTraits::max_nlanes]; + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) rmask[VTraits::max_nlanes]; + for (; x <= xOuter.end - VTraits::vlanes(); x += VTraits::vlanes()) { v_uint32 v_mask = vx_load_expand_q(ptr + x); - v_mask = v_mask != v_zero_u32; + v_mask = v_ne(v_mask, v_zero_u32); v_float32 v_x = v_cvt_f32(vx_setall_s32(x)); - v_float32 v_dx = v_x - v_curCenterX_0123; + v_float32 v_dx = v_sub(v_x, v_curCenterX_0123); - v_float32 v_r2 = (v_dx * v_dx) + v_dy2; - v_float32 vmask = (v_minRadius2 <= v_r2) & (v_r2 <= v_maxRadius2) & v_reinterpret_as_f32(v_mask); + v_float32 v_r2 = v_add(v_mul(v_dx, v_dx), v_dy2); + v_float32 vmask = v_and(v_and(v_le(v_minRadius2, v_r2), v_le(v_r2, v_maxRadius2)), v_reinterpret_as_f32(v_mask)); if (v_check_any(vmask)) { v_store_aligned(rmask, v_reinterpret_as_s32(vmask)); v_store_aligned(rbuf, v_r2); - for (int i = 0; i < v_int32::nlanes; ++i) + for (int i = 0; i < VTraits::vlanes(); ++i) if (rmask[i]) ddata[nzCount++] = rbuf[i]; } } diff --git a/modules/imgproc/src/morph.simd.hpp b/modules/imgproc/src/morph.simd.hpp index 9b3023f8f0..ef813dccec 100644 --- a/modules/imgproc/src/morph.simd.hpp +++ b/modules/imgproc/src/morph.simd.hpp @@ -106,12 +106,12 @@ struct MorphNoVec int operator()(uchar**, int, uchar*, int) const { return 0; } }; -#if CV_SIMD +#if CV_SIMD // TODO: enable for CV_SIMD_SCALABLE, GCC 13 related template struct MorphRowVec { typedef typename VecUpdate::vtype vtype; - typedef typename vtype::lane_type stype; + typedef typename VTraits::lane_type stype; MorphRowVec(int _ksize, int _anchor) : ksize(_ksize), anchor(_anchor) {} int operator()(const uchar* src, uchar* dst, int width, int cn) const { @@ -121,52 +121,52 @@ template struct MorphRowVec width *= cn; VecUpdate updateOp; - for( i = 0; i <= width - 4*vtype::nlanes; i += 4*vtype::nlanes ) + for( i = 0; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { vtype s0 = vx_load((const stype*)src + i); - vtype s1 = vx_load((const stype*)src + i + vtype::nlanes); - vtype s2 = vx_load((const stype*)src + i + 2*vtype::nlanes); - vtype s3 = vx_load((const stype*)src + i + 3*vtype::nlanes); + vtype s1 = vx_load((const stype*)src + i + VTraits::vlanes()); + vtype s2 = vx_load((const stype*)src + i + 2*VTraits::vlanes()); + vtype s3 = vx_load((const stype*)src + i + 3*VTraits::vlanes()); for (k = cn; k < _ksize; k += cn) { s0 = updateOp(s0, vx_load((const stype*)src + i + k)); - s1 = updateOp(s1, vx_load((const stype*)src + i + k + vtype::nlanes)); - s2 = updateOp(s2, vx_load((const stype*)src + i + k + 2*vtype::nlanes)); - s3 = updateOp(s3, vx_load((const stype*)src + i + k + 3*vtype::nlanes)); + s1 = updateOp(s1, vx_load((const stype*)src + i + k + VTraits::vlanes())); + s2 = updateOp(s2, vx_load((const stype*)src + i + k + 2*VTraits::vlanes())); + s3 = updateOp(s3, vx_load((const stype*)src + i + k + 3*VTraits::vlanes())); } v_store((stype*)dst + i, s0); - v_store((stype*)dst + i + vtype::nlanes, s1); - v_store((stype*)dst + i + 2*vtype::nlanes, s2); - v_store((stype*)dst + i + 3*vtype::nlanes, s3); + v_store((stype*)dst + i + VTraits::vlanes(), s1); + v_store((stype*)dst + i + 2*VTraits::vlanes(), s2); + v_store((stype*)dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*vtype::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { vtype s0 = vx_load((const stype*)src + i); - vtype s1 = vx_load((const stype*)src + i + vtype::nlanes); + vtype s1 = vx_load((const stype*)src + i + VTraits::vlanes()); for( k = cn; k < _ksize; k += cn ) { s0 = updateOp(s0, vx_load((const stype*)src + i + k)); - s1 = updateOp(s1, vx_load((const stype*)src + i + k + vtype::nlanes)); + s1 = updateOp(s1, vx_load((const stype*)src + i + k + VTraits::vlanes())); } v_store((stype*)dst + i, s0); - v_store((stype*)dst + i + vtype::nlanes, s1); - i += 2*vtype::nlanes; + v_store((stype*)dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - vtype::nlanes ) + if( i <= width - VTraits::vlanes() ) { vtype s = vx_load((const stype*)src + i); for( k = cn; k < _ksize; k += cn ) s = updateOp(s, vx_load((const stype*)src + i + k)); v_store((stype*)dst + i, s); - i += vtype::nlanes; + i += VTraits::vlanes(); } - if( i <= width - vtype::nlanes/2 ) + if( i <= width - VTraits::vlanes()/2 ) { vtype s = vx_load_low((const stype*)src + i); for( k = cn; k < _ksize; k += cn ) s = updateOp(s, vx_load_low((const stype*)src + i + k)); v_store_low((stype*)dst + i, s); - i += vtype::nlanes/2; + i += VTraits::vlanes()/2; } return i - i % cn; @@ -179,7 +179,7 @@ template struct MorphRowVec template struct MorphColumnVec { typedef typename VecUpdate::vtype vtype; - typedef typename vtype::lane_type stype; + typedef typename VTraits::lane_type stype; MorphColumnVec(int _ksize, int _anchor) : ksize(_ksize), anchor(_anchor) {} int operator()(const uchar** _src, uchar* _dst, int dststep, int count, int width) const { @@ -189,7 +189,7 @@ template struct MorphColumnVec VecUpdate updateOp; for( i = 0; i < count + ksize - 1; i++ ) - CV_Assert( ((size_t)_src[i] & (CV_SIMD_WIDTH-1)) == 0 ); + CV_Assert( ((size_t)_src[i] & (VTraits::vlanes()-1)) == 0 ); const stype** src = (const stype**)_src; stype* dst = (stype*)_dst; @@ -197,58 +197,58 @@ template struct MorphColumnVec for( ; _ksize > 1 && count > 1; count -= 2, dst += dststep*2, src += 2 ) { - for( i = 0; i <= width - 4*vtype::nlanes; i += 4*vtype::nlanes) + for( i = 0; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes()) { const stype* sptr = src[1] + i; vtype s0 = vx_load_aligned(sptr); - vtype s1 = vx_load_aligned(sptr + vtype::nlanes); - vtype s2 = vx_load_aligned(sptr + 2*vtype::nlanes); - vtype s3 = vx_load_aligned(sptr + 3*vtype::nlanes); + vtype s1 = vx_load_aligned(sptr + VTraits::vlanes()); + vtype s2 = vx_load_aligned(sptr + 2*VTraits::vlanes()); + vtype s3 = vx_load_aligned(sptr + 3*VTraits::vlanes()); for( k = 2; k < _ksize; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load_aligned(sptr)); - s1 = updateOp(s1, vx_load_aligned(sptr + vtype::nlanes)); - s2 = updateOp(s2, vx_load_aligned(sptr + 2*vtype::nlanes)); - s3 = updateOp(s3, vx_load_aligned(sptr + 3*vtype::nlanes)); + s1 = updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes())); + s2 = updateOp(s2, vx_load_aligned(sptr + 2*VTraits::vlanes())); + s3 = updateOp(s3, vx_load_aligned(sptr + 3*VTraits::vlanes())); } sptr = src[0] + i; v_store(dst + i, updateOp(s0, vx_load_aligned(sptr))); - v_store(dst + i + vtype::nlanes, updateOp(s1, vx_load_aligned(sptr + vtype::nlanes))); - v_store(dst + i + 2*vtype::nlanes, updateOp(s2, vx_load_aligned(sptr + 2*vtype::nlanes))); - v_store(dst + i + 3*vtype::nlanes, updateOp(s3, vx_load_aligned(sptr + 3*vtype::nlanes))); + v_store(dst + i + VTraits::vlanes(), updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes()))); + v_store(dst + i + 2*VTraits::vlanes(), updateOp(s2, vx_load_aligned(sptr + 2*VTraits::vlanes()))); + v_store(dst + i + 3*VTraits::vlanes(), updateOp(s3, vx_load_aligned(sptr + 3*VTraits::vlanes()))); sptr = src[k] + i; v_store(dst + dststep + i, updateOp(s0, vx_load_aligned(sptr))); - v_store(dst + dststep + i + vtype::nlanes, updateOp(s1, vx_load_aligned(sptr + vtype::nlanes))); - v_store(dst + dststep + i + 2*vtype::nlanes, updateOp(s2, vx_load_aligned(sptr + 2*vtype::nlanes))); - v_store(dst + dststep + i + 3*vtype::nlanes, updateOp(s3, vx_load_aligned(sptr + 3*vtype::nlanes))); + v_store(dst + dststep + i + VTraits::vlanes(), updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes()))); + v_store(dst + dststep + i + 2*VTraits::vlanes(), updateOp(s2, vx_load_aligned(sptr + 2*VTraits::vlanes()))); + v_store(dst + dststep + i + 3*VTraits::vlanes(), updateOp(s3, vx_load_aligned(sptr + 3*VTraits::vlanes()))); } - if( i <= width - 2*vtype::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { const stype* sptr = src[1] + i; vtype s0 = vx_load_aligned(sptr); - vtype s1 = vx_load_aligned(sptr + vtype::nlanes); + vtype s1 = vx_load_aligned(sptr + VTraits::vlanes()); for( k = 2; k < _ksize; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load_aligned(sptr)); - s1 = updateOp(s1, vx_load_aligned(sptr + vtype::nlanes)); + s1 = updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes())); } sptr = src[0] + i; v_store(dst + i, updateOp(s0, vx_load_aligned(sptr))); - v_store(dst + i + vtype::nlanes, updateOp(s1, vx_load_aligned(sptr + vtype::nlanes))); + v_store(dst + i + VTraits::vlanes(), updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes()))); sptr = src[k] + i; v_store(dst + dststep + i, updateOp(s0, vx_load_aligned(sptr))); - v_store(dst + dststep + i + vtype::nlanes, updateOp(s1, vx_load_aligned(sptr + vtype::nlanes))); - i += 2*vtype::nlanes; + v_store(dst + dststep + i + VTraits::vlanes(), updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes()))); + i += 2*VTraits::vlanes(); } - if( i <= width - vtype::nlanes ) + if( i <= width - VTraits::vlanes() ) { vtype s0 = vx_load_aligned(src[1] + i); @@ -257,9 +257,9 @@ template struct MorphColumnVec v_store(dst + i, updateOp(s0, vx_load_aligned(src[0] + i))); v_store(dst + dststep + i, updateOp(s0, vx_load_aligned(src[k] + i))); - i += vtype::nlanes; + i += VTraits::vlanes(); } - if( i <= width - vtype::nlanes/2 ) + if( i <= width - VTraits::vlanes()/2 ) { vtype s0 = vx_load_low(src[1] + i); @@ -268,66 +268,66 @@ template struct MorphColumnVec v_store_low(dst + i, updateOp(s0, vx_load_low(src[0] + i))); v_store_low(dst + dststep + i, updateOp(s0, vx_load_low(src[k] + i))); - i += vtype::nlanes/2; + i += VTraits::vlanes()/2; } } for( ; count > 0; count--, dst += dststep, src++ ) { - for( i = 0; i <= width - 4*vtype::nlanes; i += 4*vtype::nlanes) + for( i = 0; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes()) { const stype* sptr = src[0] + i; vtype s0 = vx_load_aligned(sptr); - vtype s1 = vx_load_aligned(sptr + vtype::nlanes); - vtype s2 = vx_load_aligned(sptr + 2*vtype::nlanes); - vtype s3 = vx_load_aligned(sptr + 3*vtype::nlanes); + vtype s1 = vx_load_aligned(sptr + VTraits::vlanes()); + vtype s2 = vx_load_aligned(sptr + 2*VTraits::vlanes()); + vtype s3 = vx_load_aligned(sptr + 3*VTraits::vlanes()); for( k = 1; k < _ksize; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load_aligned(sptr)); - s1 = updateOp(s1, vx_load_aligned(sptr + vtype::nlanes)); - s2 = updateOp(s2, vx_load_aligned(sptr + 2*vtype::nlanes)); - s3 = updateOp(s3, vx_load_aligned(sptr + 3*vtype::nlanes)); + s1 = updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes())); + s2 = updateOp(s2, vx_load_aligned(sptr + 2*VTraits::vlanes())); + s3 = updateOp(s3, vx_load_aligned(sptr + 3*VTraits::vlanes())); } v_store(dst + i, s0); - v_store(dst + i + vtype::nlanes, s1); - v_store(dst + i + 2*vtype::nlanes, s2); - v_store(dst + i + 3*vtype::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*vtype::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { const stype* sptr = src[0] + i; vtype s0 = vx_load_aligned(sptr); - vtype s1 = vx_load_aligned(sptr + vtype::nlanes); + vtype s1 = vx_load_aligned(sptr + VTraits::vlanes()); for( k = 1; k < _ksize; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load_aligned(sptr)); - s1 = updateOp(s1, vx_load_aligned(sptr + vtype::nlanes)); + s1 = updateOp(s1, vx_load_aligned(sptr + VTraits::vlanes())); } v_store(dst + i, s0); - v_store(dst + i + vtype::nlanes, s1); - i += 2*vtype::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - vtype::nlanes ) + if( i <= width - VTraits::vlanes() ) { vtype s0 = vx_load_aligned(src[0] + i); for( k = 1; k < _ksize; k++ ) s0 = updateOp(s0, vx_load_aligned(src[k] + i)); v_store(dst + i, s0); - i += vtype::nlanes; + i += VTraits::vlanes(); } - if( i <= width - vtype::nlanes/2 ) + if( i <= width - VTraits::vlanes()/2 ) { vtype s0 = vx_load_low(src[0] + i); for( k = 1; k < _ksize; k++ ) s0 = updateOp(s0, vx_load_low(src[k] + i)); v_store_low(dst + i, s0); - i += vtype::nlanes/2; + i += VTraits::vlanes()/2; } } @@ -341,7 +341,7 @@ template struct MorphColumnVec template struct MorphVec { typedef typename VecUpdate::vtype vtype; - typedef typename vtype::lane_type stype; + typedef typename VTraits::lane_type stype; int operator()(uchar** _src, int nz, uchar* _dst, int width) const { CV_INSTRUMENT_REGION(); @@ -351,56 +351,56 @@ template struct MorphVec int i, k; VecUpdate updateOp; - for( i = 0; i <= width - 4*vtype::nlanes; i += 4*vtype::nlanes ) + for( i = 0; i <= width - 4*VTraits::vlanes(); i += 4*VTraits::vlanes() ) { const stype* sptr = src[0] + i; vtype s0 = vx_load(sptr); - vtype s1 = vx_load(sptr + vtype::nlanes); - vtype s2 = vx_load(sptr + 2*vtype::nlanes); - vtype s3 = vx_load(sptr + 3*vtype::nlanes); + vtype s1 = vx_load(sptr + VTraits::vlanes()); + vtype s2 = vx_load(sptr + 2*VTraits::vlanes()); + vtype s3 = vx_load(sptr + 3*VTraits::vlanes()); for( k = 1; k < nz; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load(sptr)); - s1 = updateOp(s1, vx_load(sptr + vtype::nlanes)); - s2 = updateOp(s2, vx_load(sptr + 2*vtype::nlanes)); - s3 = updateOp(s3, vx_load(sptr + 3*vtype::nlanes)); + s1 = updateOp(s1, vx_load(sptr + VTraits::vlanes())); + s2 = updateOp(s2, vx_load(sptr + 2*VTraits::vlanes())); + s3 = updateOp(s3, vx_load(sptr + 3*VTraits::vlanes())); } v_store(dst + i, s0); - v_store(dst + i + vtype::nlanes, s1); - v_store(dst + i + 2*vtype::nlanes, s2); - v_store(dst + i + 3*vtype::nlanes, s3); + v_store(dst + i + VTraits::vlanes(), s1); + v_store(dst + i + 2*VTraits::vlanes(), s2); + v_store(dst + i + 3*VTraits::vlanes(), s3); } - if( i <= width - 2*vtype::nlanes ) + if( i <= width - 2*VTraits::vlanes() ) { const stype* sptr = src[0] + i; vtype s0 = vx_load(sptr); - vtype s1 = vx_load(sptr + vtype::nlanes); + vtype s1 = vx_load(sptr + VTraits::vlanes()); for( k = 1; k < nz; k++ ) { sptr = src[k] + i; s0 = updateOp(s0, vx_load(sptr)); - s1 = updateOp(s1, vx_load(sptr + vtype::nlanes)); + s1 = updateOp(s1, vx_load(sptr + VTraits::vlanes())); } v_store(dst + i, s0); - v_store(dst + i + vtype::nlanes, s1); - i += 2*vtype::nlanes; + v_store(dst + i + VTraits::vlanes(), s1); + i += 2*VTraits::vlanes(); } - if( i <= width - vtype::nlanes ) + if( i <= width - VTraits::vlanes() ) { vtype s0 = vx_load(src[0] + i); for( k = 1; k < nz; k++ ) s0 = updateOp(s0, vx_load(src[k] + i)); v_store(dst + i, s0); - i += vtype::nlanes; + i += VTraits::vlanes(); } - if( i <= width - vtype::nlanes/2 ) + if( i <= width - VTraits::vlanes()/2 ) { vtype s0 = vx_load_low(src[0] + i); for( k = 1; k < nz; k++ ) s0 = updateOp(s0, vx_load_low(src[k] + i)); v_store_low(dst + i, s0); - i += vtype::nlanes/2; + i += VTraits::vlanes()/2; } return i; } diff --git a/modules/imgproc/src/pyramids.cpp b/modules/imgproc/src/pyramids.cpp index 45dfb4e0ca..f1544bd5e7 100644 --- a/modules/imgproc/src/pyramids.cpp +++ b/modules/imgproc/src/pyramids.cpp @@ -84,7 +84,7 @@ template int PyrUpVecV(T1**, T2**, int) { return 0; } template int PyrUpVecVOneRow(T1**, T2*, int) { return 0; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template<> int PyrDownVecH(const uchar* src, int* row, int width) { @@ -93,10 +93,8 @@ template<> int PyrDownVecH(const uchar* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_reinterpret_as_s16(vx_load_expand(src01)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(vx_load_expand(src23)), v_6_4) + - (v_reinterpret_as_s32(vx_load_expand(src4)) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(v_reinterpret_as_s16(vx_load_expand(src01)), v_1_4), v_dotprod(v_reinterpret_as_s16(vx_load_expand(src23)), v_6_4)), v_shr<16>(v_reinterpret_as_s32(vx_load_expand(src4))))); vx_cleanup(); return x; @@ -108,42 +106,40 @@ template<> int PyrDownVecH(const uchar* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(vx_load_expand(src01))), v_1_4) + - v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(vx_load_expand(src23))), v_6_4) + - (v_reinterpret_as_s32(v_interleave_pairs(vx_load_expand(src4))) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(vx_load_expand(src01))), v_1_4), v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(vx_load_expand(src23))), v_6_4)), v_shr<16>(v_reinterpret_as_s32(v_interleave_pairs(vx_load_expand(src4)))))); vx_cleanup(); return x; } template<> int PyrDownVecH(const uchar* src, int* row, int width) { - int idx[v_int8::nlanes/2 + 4]; - for (int i = 0; i < v_int8::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 6*i; - idx[i + v_int8::nlanes/4 + 2] = 6*i + 3; + idx[i + VTraits::vlanes()/4 + 2] = 6*i + 3; } int x = 0; v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int8::nlanes; x += 3*v_int8::nlanes/4, src += 6*v_int8::nlanes/4, row += 3*v_int8::nlanes/4) + for (; x <= width - VTraits::vlanes(); x += 3*VTraits::vlanes()/4, src += 6*VTraits::vlanes()/4, row += 3*VTraits::vlanes()/4) { v_uint16 r0l, r0h, r1l, r1h, r2l, r2h, r3l, r3h, r4l, r4h; v_expand(vx_lut_quads(src, idx ), r0l, r0h); - v_expand(vx_lut_quads(src, idx + v_int8::nlanes/4 + 2), r1l, r1h); + v_expand(vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r1l, r1h); v_expand(vx_lut_quads(src, idx + 1 ), r2l, r2h); - v_expand(vx_lut_quads(src, idx + v_int8::nlanes/4 + 3), r3l, r3h); + v_expand(vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r3l, r3h); v_expand(vx_lut_quads(src, idx + 2 ), r4l, r4h); - v_zip(r2l, r1l + r3l, r1l, r3l); - v_zip(r2h, r1h + r3h, r1h, r3h); - r0l += r4l; r0h += r4h; + v_zip(r2l, v_add(r1l, r3l), r1l, r3l); + v_zip(r2h, v_add(r1h, r3h), r1h, r3h); + r0l = v_add(r0l, r4l); r0h = v_add(r0h, r4h); - v_store(row , v_pack_triplets(v_dotprod(v_reinterpret_as_s16(r1l), v_6_4) + v_reinterpret_as_s32(v_expand_low( r0l)))); - v_store(row + 3*v_int32::nlanes/4, v_pack_triplets(v_dotprod(v_reinterpret_as_s16(r3l), v_6_4) + v_reinterpret_as_s32(v_expand_high(r0l)))); - v_store(row + 6*v_int32::nlanes/4, v_pack_triplets(v_dotprod(v_reinterpret_as_s16(r1h), v_6_4) + v_reinterpret_as_s32(v_expand_low( r0h)))); - v_store(row + 9*v_int32::nlanes/4, v_pack_triplets(v_dotprod(v_reinterpret_as_s16(r3h), v_6_4) + v_reinterpret_as_s32(v_expand_high(r0h)))); + v_store(row , v_pack_triplets(v_add(v_dotprod(v_reinterpret_as_s16(r1l), v_6_4), v_reinterpret_as_s32(v_expand_low(r0l))))); + v_store(row + 3*VTraits::vlanes()/4, v_pack_triplets(v_add(v_dotprod(v_reinterpret_as_s16(r3l), v_6_4), v_reinterpret_as_s32(v_expand_high(r0l))))); + v_store(row + 6*VTraits::vlanes()/4, v_pack_triplets(v_add(v_dotprod(v_reinterpret_as_s16(r1h), v_6_4), v_reinterpret_as_s32(v_expand_low(r0h))))); + v_store(row + 9*VTraits::vlanes()/4, v_pack_triplets(v_add(v_dotprod(v_reinterpret_as_s16(r3h), v_6_4), v_reinterpret_as_s32(v_expand_high(r0h))))); } vx_cleanup(); @@ -156,10 +152,8 @@ template<> int PyrDownVecH(const uchar* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_interleave_quads(v_reinterpret_as_s16(vx_load_expand(src01))), v_1_4) + - v_dotprod(v_interleave_quads(v_reinterpret_as_s16(vx_load_expand(src23))), v_6_4) + - (v_reinterpret_as_s32(v_interleave_quads(vx_load_expand(src4))) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(v_interleave_quads(v_reinterpret_as_s16(vx_load_expand(src01))), v_1_4), v_dotprod(v_interleave_quads(v_reinterpret_as_s16(vx_load_expand(src23))), v_6_4)), v_shr<16>(v_reinterpret_as_s32(v_interleave_quads(vx_load_expand(src4)))))); vx_cleanup(); return x; @@ -172,10 +166,8 @@ template<> int PyrDownVecH(const short* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(vx_load(src01), v_1_4) + - v_dotprod(vx_load(src23), v_6_4) + - (v_reinterpret_as_s32(vx_load(src4)) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(vx_load(src01), v_1_4), v_dotprod(vx_load(src23), v_6_4)), v_shr<16>(v_reinterpret_as_s32(vx_load(src4))))); vx_cleanup(); return x; @@ -187,34 +179,32 @@ template<> int PyrDownVecH(const short* src, int* row, int width) v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_interleave_pairs(vx_load(src01)), v_1_4) + - v_dotprod(v_interleave_pairs(vx_load(src23)), v_6_4) + - (v_reinterpret_as_s32(v_interleave_pairs(vx_load(src4))) >> 16)); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_dotprod(v_interleave_pairs(vx_load(src01)), v_1_4), v_dotprod(v_interleave_pairs(vx_load(src23)), v_6_4)), v_shr<16>(v_reinterpret_as_s32(v_interleave_pairs(vx_load(src4)))))); vx_cleanup(); return x; } template<> int PyrDownVecH(const short* src, int* row, int width) { - int idx[v_int16::nlanes/2 + 4]; - for (int i = 0; i < v_int16::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 6*i; - idx[i + v_int16::nlanes/4 + 2] = 6*i + 3; + idx[i + VTraits::vlanes()/4 + 2] = 6*i + 3; } int x = 0; v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int16::nlanes; x += 3*v_int16::nlanes/4, src += 6*v_int16::nlanes/4, row += 3*v_int16::nlanes/4) + for (; x <= width - VTraits::vlanes(); x += 3*VTraits::vlanes()/4, src += 6*VTraits::vlanes()/4, row += 3*VTraits::vlanes()/4) { v_int16 r0, r1, r2, r3, r4; - v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + v_int16::nlanes/4 + 2), r0, r1); - v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + v_int16::nlanes/4 + 3), r2, r3); + v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r0, r1); + v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r2, r3); r4 = vx_lut_quads(src, idx + 2); - v_store(row, v_pack_triplets(v_dotprod(r0, v_1_4) + v_dotprod(r2, v_6_4) + v_expand_low(r4))); - v_store(row + 3*v_int32::nlanes/4, v_pack_triplets(v_dotprod(r1, v_1_4) + v_dotprod(r3, v_6_4) + v_expand_high(r4))); + v_store(row, v_pack_triplets(v_add(v_add(v_dotprod(r0, v_1_4), v_dotprod(r2, v_6_4)), v_expand_low(r4)))); + v_store(row + 3*VTraits::vlanes()/4, v_pack_triplets(v_add(v_add(v_dotprod(r1, v_1_4), v_dotprod(r3, v_6_4)), v_expand_high(r4)))); } vx_cleanup(); @@ -222,24 +212,24 @@ template<> int PyrDownVecH(const short* src, int* row, int width) } template<> int PyrDownVecH(const short* src, int* row, int width) { - int idx[v_int16::nlanes/2 + 4]; - for (int i = 0; i < v_int16::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 8*i; - idx[i + v_int16::nlanes/4 + 2] = 8*i + 4; + idx[i + VTraits::vlanes()/4 + 2] = 8*i + 4; } int x = 0; v_int16 v_1_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040001)); v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes, src += 2*v_int16::nlanes, row += v_int16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src += 2*VTraits::vlanes(), row += VTraits::vlanes()) { v_int16 r0, r1, r2, r3, r4; - v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + v_int16::nlanes/4 + 2), r0, r1); - v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + v_int16::nlanes/4 + 3), r2, r3); + v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r0, r1); + v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r2, r3); r4 = vx_lut_quads(src, idx + 2); - v_store(row, v_dotprod(r0, v_1_4) + v_dotprod(r2, v_6_4) + v_expand_low(r4)); - v_store(row + v_int32::nlanes, v_dotprod(r1, v_1_4) + v_dotprod(r3, v_6_4) + v_expand_high(r4)); + v_store(row, v_add(v_add(v_dotprod(r0, v_1_4), v_dotprod(r2, v_6_4)), v_expand_low(r4))); + v_store(row + VTraits::vlanes(), v_add(v_add(v_dotprod(r1, v_1_4), v_dotprod(r3, v_6_4)), v_expand_high(r4))); } vx_cleanup(); @@ -255,10 +245,8 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); v_uint16 v_half = vx_setall_u16(0x8000); v_int32 v_half15 = vx_setall_s32(0x00078000); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_reinterpret_as_s16(v_sub_wrap(vx_load(src01), v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(vx_load(src23), v_half)), v_6_4) + - v_reinterpret_as_s32(v_reinterpret_as_u32(vx_load(src4)) >> 16) + v_half15); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(vx_load(src01), v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(vx_load(src23), v_half)), v_6_4)), v_reinterpret_as_s32(v_shr<16>(v_reinterpret_as_u32(vx_load(src4))))), v_half15)); vx_cleanup(); return x; @@ -272,21 +260,19 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); v_uint16 v_half = vx_setall_u16(0x8000); v_int32 v_half15 = vx_setall_s32(0x00078000); - for (; x <= width - v_int32::nlanes; x += v_int32::nlanes, src01 += v_int16::nlanes, src23 += v_int16::nlanes, src4 += v_int16::nlanes, row += v_int32::nlanes) - v_store(row, v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(v_sub_wrap(vx_load(src01), v_half))), v_1_4) + - v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(v_sub_wrap(vx_load(src23), v_half))), v_6_4) + - v_reinterpret_as_s32(v_reinterpret_as_u32(v_interleave_pairs(vx_load(src4))) >> 16) + v_half15); + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += VTraits::vlanes(), src23 += VTraits::vlanes(), src4 += VTraits::vlanes(), row += VTraits::vlanes()) + v_store(row, v_add(v_add(v_add(v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(v_sub_wrap(vx_load(src01), v_half))), v_1_4), v_dotprod(v_interleave_pairs(v_reinterpret_as_s16(v_sub_wrap(vx_load(src23), v_half))), v_6_4)), v_reinterpret_as_s32(v_shr<16>(v_reinterpret_as_u32(v_interleave_pairs(vx_load(src4)))))), v_half15)); vx_cleanup(); return x; } template<> int PyrDownVecH(const ushort* src, int* row, int width) { - int idx[v_int16::nlanes/2 + 4]; - for (int i = 0; i < v_int16::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 6*i; - idx[i + v_int16::nlanes/4 + 2] = 6*i + 3; + idx[i + VTraits::vlanes()/4 + 2] = 6*i + 3; } int x = 0; @@ -294,18 +280,14 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); v_uint16 v_half = vx_setall_u16(0x8000); v_int32 v_half15 = vx_setall_s32(0x00078000); - for (; x <= width - v_int16::nlanes; x += 3*v_int16::nlanes/4, src += 6*v_int16::nlanes/4, row += 3*v_int16::nlanes/4) + for (; x <= width - VTraits::vlanes(); x += 3*VTraits::vlanes()/4, src += 6*VTraits::vlanes()/4, row += 3*VTraits::vlanes()/4) { v_uint16 r0, r1, r2, r3, r4; - v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + v_int16::nlanes/4 + 2), r0, r1); - v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + v_int16::nlanes/4 + 3), r2, r3); + v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r0, r1); + v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r2, r3); r4 = vx_lut_quads(src, idx + 2); - v_store(row , v_pack_triplets(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r0, v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r2, v_half)), v_6_4) + - v_reinterpret_as_s32(v_expand_low(r4)) + v_half15)); - v_store(row + 3*v_int32::nlanes/4, v_pack_triplets(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r1, v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r3, v_half)), v_6_4) + - v_reinterpret_as_s32(v_expand_high(r4)) + v_half15)); + v_store(row , v_pack_triplets(v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r0, v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r2, v_half)), v_6_4)), v_reinterpret_as_s32(v_expand_low(r4))), v_half15))); + v_store(row + 3*VTraits::vlanes()/4, v_pack_triplets(v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r1, v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r3, v_half)), v_6_4)), v_reinterpret_as_s32(v_expand_high(r4))), v_half15))); } vx_cleanup(); @@ -313,11 +295,11 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt } template<> int PyrDownVecH(const ushort* src, int* row, int width) { - int idx[v_int16::nlanes/2 + 4]; - for (int i = 0; i < v_int16::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 8*i; - idx[i + v_int16::nlanes/4 + 2] = 8*i + 4; + idx[i + VTraits::vlanes()/4 + 2] = 8*i + 4; } int x = 0; @@ -325,18 +307,14 @@ template<> int PyrDownVecH(const ushort* src, int* row, int widt v_int16 v_6_4 = v_reinterpret_as_s16(vx_setall_u32(0x00040006)); v_uint16 v_half = vx_setall_u16(0x8000); v_int32 v_half15 = vx_setall_s32(0x00078000); - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes, src += 2*v_int16::nlanes, row += v_int16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src += 2*VTraits::vlanes(), row += VTraits::vlanes()) { v_uint16 r0, r1, r2, r3, r4; - v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + v_int16::nlanes/4 + 2), r0, r1); - v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + v_int16::nlanes/4 + 3), r2, r3); + v_zip(vx_lut_quads(src, idx), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2), r0, r1); + v_zip(vx_lut_quads(src, idx + 1), vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3), r2, r3); r4 = vx_lut_quads(src, idx + 2); - v_store(row , v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r0, v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r2, v_half)), v_6_4) + - v_reinterpret_as_s32(v_expand_low(r4)) + v_half15); - v_store(row + v_int32::nlanes, v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r1, v_half)), v_1_4) + - v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r3, v_half)), v_6_4) + - v_reinterpret_as_s32(v_expand_high(r4)) + v_half15); + v_store(row , v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r0, v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r2, v_half)), v_6_4)), v_reinterpret_as_s32(v_expand_low(r4))), v_half15)); + v_store(row + VTraits::vlanes(), v_add(v_add(v_add(v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r1, v_half)), v_1_4), v_dotprod(v_reinterpret_as_s16(v_sub_wrap(r3, v_half)), v_6_4)), v_reinterpret_as_s32(v_expand_high(r4))), v_half15)); } vx_cleanup(); @@ -349,13 +327,13 @@ template<> int PyrDownVecH(const float* src, float* row, int wi const float *src01 = src, *src23 = src + 2, *src4 = src + 3; v_float32 _4 = vx_setall_f32(4.f), _6 = vx_setall_f32(6.f); - for (; x <= width - v_float32::nlanes; x += v_float32::nlanes, src01 += 2*v_float32::nlanes, src23 += 2*v_float32::nlanes, src4 += 2*v_float32::nlanes, row+=v_float32::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += 2*VTraits::vlanes(), src23 += 2*VTraits::vlanes(), src4 += 2*VTraits::vlanes(), row+=VTraits::vlanes()) { v_float32 r0, r1, r2, r3, r4, rtmp; v_load_deinterleave(src01, r0, r1); v_load_deinterleave(src23, r2, r3); v_load_deinterleave(src4, rtmp, r4); - v_store(row, v_muladd(r2, _6, v_muladd(r1 + r3, _4, r0 + r4))); + v_store(row, v_muladd(r2, _6, v_muladd(v_add(r1, r3), _4, v_add(r0, r4)))); } vx_cleanup(); @@ -367,13 +345,13 @@ template<> int PyrDownVecH(const float* src, float* row, int wi const float *src01 = src, *src23 = src + 4, *src4 = src + 6; v_float32 _4 = vx_setall_f32(4.f), _6 = vx_setall_f32(6.f); - for (; x <= width - 2*v_float32::nlanes; x += 2*v_float32::nlanes, src01 += 4*v_float32::nlanes, src23 += 4*v_float32::nlanes, src4 += 4*v_float32::nlanes, row += 2*v_float32::nlanes) + for (; x <= width - 2*VTraits::vlanes(); x += 2*VTraits::vlanes(), src01 += 4*VTraits::vlanes(), src23 += 4*VTraits::vlanes(), src4 += 4*VTraits::vlanes(), row += 2*VTraits::vlanes()) { v_float32 r0a, r0b, r1a, r1b, r2a, r2b, r3a, r3b, r4a, r4b, rtmpa, rtmpb; v_load_deinterleave(src01, r0a, r0b, r1a, r1b); v_load_deinterleave(src23, r2a, r2b, r3a, r3b); v_load_deinterleave(src4, rtmpa, rtmpb, r4a, r4b); - v_store_interleave(row, v_muladd(r2a, _6, v_muladd(r1a + r3a, _4, r0a + r4a)), v_muladd(r2b, _6, v_muladd(r1b + r3b, _4, r0b + r4b))); + v_store_interleave(row, v_muladd(r2a, _6, v_muladd(v_add(r1a, r3a), _4, v_add(r0a, r4a))), v_muladd(r2b, _6, v_muladd(v_add(r1b, r3b), _4, v_add(r0b, r4b)))); } vx_cleanup(); @@ -381,23 +359,23 @@ template<> int PyrDownVecH(const float* src, float* row, int wi } template<> int PyrDownVecH(const float* src, float* row, int width) { - int idx[v_float32::nlanes/2 + 4]; - for (int i = 0; i < v_float32::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 6*i; - idx[i + v_float32::nlanes/4 + 2] = 6*i + 3; + idx[i + VTraits::vlanes()/4 + 2] = 6*i + 3; } int x = 0; v_float32 _4 = vx_setall_f32(4.f), _6 = vx_setall_f32(6.f); - for (; x <= width - v_float32::nlanes; x += 3*v_float32::nlanes/4, src += 6*v_float32::nlanes/4, row += 3*v_float32::nlanes/4) + for (; x <= width - VTraits::vlanes(); x += 3*VTraits::vlanes()/4, src += 6*VTraits::vlanes()/4, row += 3*VTraits::vlanes()/4) { v_float32 r0 = vx_lut_quads(src, idx); - v_float32 r1 = vx_lut_quads(src, idx + v_float32::nlanes/4 + 2); + v_float32 r1 = vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2); v_float32 r2 = vx_lut_quads(src, idx + 1); - v_float32 r3 = vx_lut_quads(src, idx + v_float32::nlanes/4 + 3); + v_float32 r3 = vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3); v_float32 r4 = vx_lut_quads(src, idx + 2); - v_store(row, v_pack_triplets(v_muladd(r2, _6, v_muladd(r1 + r3, _4, r0 + r4)))); + v_store(row, v_pack_triplets(v_muladd(r2, _6, v_muladd(v_add(r1, r3), _4, v_add(r0, r4))))); } vx_cleanup(); @@ -405,43 +383,43 @@ template<> int PyrDownVecH(const float* src, float* row, int wi } template<> int PyrDownVecH(const float* src, float* row, int width) { - int idx[v_float32::nlanes/2 + 4]; - for (int i = 0; i < v_float32::nlanes/4 + 2; i++) + int idx[VTraits::max_nlanes/2 + 4]; + for (int i = 0; i < VTraits::vlanes()/4 + 2; i++) { idx[i] = 8*i; - idx[i + v_float32::nlanes/4 + 2] = 8*i + 4; + idx[i + VTraits::vlanes()/4 + 2] = 8*i + 4; } int x = 0; v_float32 _4 = vx_setall_f32(4.f), _6 = vx_setall_f32(6.f); - for (; x <= width - v_float32::nlanes; x += v_float32::nlanes, src += 2*v_float32::nlanes, row += v_float32::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src += 2*VTraits::vlanes(), row += VTraits::vlanes()) { v_float32 r0 = vx_lut_quads(src, idx); - v_float32 r1 = vx_lut_quads(src, idx + v_float32::nlanes/4 + 2); + v_float32 r1 = vx_lut_quads(src, idx + VTraits::vlanes()/4 + 2); v_float32 r2 = vx_lut_quads(src, idx + 1); - v_float32 r3 = vx_lut_quads(src, idx + v_float32::nlanes/4 + 3); + v_float32 r3 = vx_lut_quads(src, idx + VTraits::vlanes()/4 + 3); v_float32 r4 = vx_lut_quads(src, idx + 2); - v_store(row, v_muladd(r2, _6, v_muladd(r1 + r3, _4, r0 + r4))); + v_store(row, v_muladd(r2, _6, v_muladd(v_add(r1, r3), _4, v_add(r0, r4)))); } vx_cleanup(); return x; } -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) template<> int PyrDownVecH(const double* src, double* row, int width) { int x = 0; const double *src01 = src, *src23 = src + 2, *src4 = src + 3; v_float64 _4 = vx_setall_f64(4.f), _6 = vx_setall_f64(6.f); - for (; x <= width - v_float64::nlanes; x += v_float64::nlanes, src01 += 2*v_float64::nlanes, src23 += 2*v_float64::nlanes, src4 += 2*v_float64::nlanes, row += v_float64::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes(), src01 += 2*VTraits::vlanes(), src23 += 2*VTraits::vlanes(), src4 += 2*VTraits::vlanes(), row += VTraits::vlanes()) { v_float64 r0, r1, r2, r3, r4, rtmp; v_load_deinterleave(src01, r0, r1); v_load_deinterleave(src23, r2, r3); v_load_deinterleave(src4, rtmp, r4); - v_store(row, v_muladd(r2, _6, v_muladd(r1 + r3, _4, r0 + r4))); + v_store(row, v_muladd(r2, _6, v_muladd(v_add(r1, r3), _4, v_add(r0, r4)))); } vx_cleanup(); @@ -454,35 +432,36 @@ template<> int PyrDownVecV(int** src, uchar* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4]; - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes() ) { v_uint16 r0, r1, r2, r3, r4, t0, t1; - r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes))); - r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes))); - r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes))); - r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x), vx_load(row3 + x + v_int32::nlanes))); - r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x), vx_load(row4 + x + v_int32::nlanes))); - t0 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); - r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x + 2*v_int32::nlanes), vx_load(row0 + x + 3*v_int32::nlanes))); - r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x + 2*v_int32::nlanes), vx_load(row1 + x + 3*v_int32::nlanes))); - r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x + 2*v_int32::nlanes), vx_load(row2 + x + 3*v_int32::nlanes))); - r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x + 2*v_int32::nlanes), vx_load(row3 + x + 3*v_int32::nlanes))); - r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x + 2*v_int32::nlanes), vx_load(row4 + x + 3*v_int32::nlanes))); - t1 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); + r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes()))); + r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes()))); + r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes()))); + r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x), vx_load(row3 + x + VTraits::vlanes()))); + r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x), vx_load(row4 + x + VTraits::vlanes()))); + t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); + r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x + 2*VTraits::vlanes()), vx_load(row0 + x + 3*VTraits::vlanes()))); + r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x + 2*VTraits::vlanes()), vx_load(row1 + x + 3*VTraits::vlanes()))); + r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x + 2*VTraits::vlanes()), vx_load(row2 + x + 3*VTraits::vlanes()))); + r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x + 2*VTraits::vlanes()), vx_load(row3 + x + 3*VTraits::vlanes()))); + r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x + 2*VTraits::vlanes()), vx_load(row4 + x + 3*VTraits::vlanes()))); + t1 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); v_store(dst + x, v_rshr_pack<8>(t0, t1)); } - if (x <= width - v_int16::nlanes) + if (x <= width - VTraits::vlanes()) { v_uint16 r0, r1, r2, r3, r4, t0; - r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes))); - r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes))); - r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes))); - r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x), vx_load(row3 + x + v_int32::nlanes))); - r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x), vx_load(row4 + x + v_int32::nlanes))); - t0 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); + r0 = v_reinterpret_as_u16(v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes()))); + r1 = v_reinterpret_as_u16(v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes()))); + r2 = v_reinterpret_as_u16(v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes()))); + r3 = v_reinterpret_as_u16(v_pack(vx_load(row3 + x), vx_load(row3 + x + VTraits::vlanes()))); + r4 = v_reinterpret_as_u16(v_pack(vx_load(row4 + x), vx_load(row4 + x + VTraits::vlanes()))); + t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); v_rshr_pack_store<8>(dst + x, t0); - x += v_uint16::nlanes; + x += VTraits::vlanes(); } + #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; for ( ; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) { @@ -492,10 +471,23 @@ template<> int PyrDownVecV(int** src, uchar* dst, int width) r2 = v_load(row2 + x); r3 = v_load(row3 + x); r4 = v_load(row4 + x); - t0 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); + t0 = v_add(v_add(v_add(r0, r4), v_add(r2, r2)), v_shl<2>(v_add(v_add(r1, r3), r2))); *((unaligned_int*) (dst + x)) = v_reinterpret_as_s32(v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16())).get0(); } + #else + for (; x <= width - 1; x += 1) + { + int r0 = *(row0 + x); + int r1 = *(row1 + x); + int r2 = *(row2 + x); + int r3 = *(row3 + x); + int r4 = *(row4 + x); + int t0 = r0 + r4 + (r2 + r2) + ((r1 + r3 + r2) << 2); + // Similar to v_rshr_pack<8>(v_pack_u(t0, t0), v_setzero_u16()).get0() + *(dst + x) = (int)((((unsigned int)t0) + ((1 << (8 - 1)))) >> 8); + } + #endif //CV_SIMD128 vx_cleanup(); return x; @@ -508,7 +500,7 @@ int PyrDownVecV(float** src, float* dst, int width) const float *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4]; v_float32 _4 = vx_setall_f32(4.f), _scale = vx_setall_f32(1.f/256); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_float32 r0, r1, r2, r3, r4; r0 = vx_load(row0 + x); @@ -516,7 +508,7 @@ int PyrDownVecV(float** src, float* dst, int width) r2 = vx_load(row2 + x); r3 = vx_load(row3 + x); r4 = vx_load(row4 + x); - v_store(dst + x, v_muladd(r1 + r3 + r2, _4, r0 + r4 + (r2 + r2)) * _scale); + v_store(dst + x, v_mul(v_muladd(v_add(v_add(r1, r3), r2), _4, v_add(v_add(r0, r4), v_add(r2, r2))), _scale)); } vx_cleanup(); @@ -528,30 +520,30 @@ template <> int PyrDownVecV(int** src, ushort* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4]; - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 r00 = vx_load(row0 + x), - r01 = vx_load(row0 + x + v_int32::nlanes), + r01 = vx_load(row0 + x + VTraits::vlanes()), r10 = vx_load(row1 + x), - r11 = vx_load(row1 + x + v_int32::nlanes), + r11 = vx_load(row1 + x + VTraits::vlanes()), r20 = vx_load(row2 + x), - r21 = vx_load(row2 + x + v_int32::nlanes), + r21 = vx_load(row2 + x + VTraits::vlanes()), r30 = vx_load(row3 + x), - r31 = vx_load(row3 + x + v_int32::nlanes), + r31 = vx_load(row3 + x + VTraits::vlanes()), r40 = vx_load(row4 + x), - r41 = vx_load(row4 + x + v_int32::nlanes); - v_store(dst + x, v_rshr_pack_u<8>(r00 + r40 + (r20 + r20) + ((r10 + r20 + r30) << 2), - r01 + r41 + (r21 + r21) + ((r11 + r21 + r31) << 2))); + r41 = vx_load(row4 + x + VTraits::vlanes()); + v_store(dst + x, v_rshr_pack_u<8>(v_add(v_add(v_add(r00, r40), v_add(r20, r20)), v_shl<2>(v_add(v_add(r10, r20), r30))), + v_add(v_add(v_add(r01, r41), v_add(r21, r21)), v_shl<2>(v_add(v_add(r11, r21), r31))))); } - if (x <= width - v_int32::nlanes) + if (x <= width - VTraits::vlanes()) { v_int32 r00 = vx_load(row0 + x), r10 = vx_load(row1 + x), r20 = vx_load(row2 + x), r30 = vx_load(row3 + x), r40 = vx_load(row4 + x); - v_rshr_pack_u_store<8>(dst + x, r00 + r40 + (r20 + r20) + ((r10 + r20 + r30) << 2)); - x += v_int32::nlanes; + v_rshr_pack_u_store<8>(dst + x, v_add(v_add(v_add(r00, r40), v_add(r20, r20)), v_shl<2>(v_add(v_add(r10, r20), r30)))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -563,30 +555,30 @@ template <> int PyrDownVecV(int** src, short* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2], *row3 = src[3], *row4 = src[4]; - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 r00 = vx_load(row0 + x), - r01 = vx_load(row0 + x + v_int32::nlanes), + r01 = vx_load(row0 + x + VTraits::vlanes()), r10 = vx_load(row1 + x), - r11 = vx_load(row1 + x + v_int32::nlanes), + r11 = vx_load(row1 + x + VTraits::vlanes()), r20 = vx_load(row2 + x), - r21 = vx_load(row2 + x + v_int32::nlanes), + r21 = vx_load(row2 + x + VTraits::vlanes()), r30 = vx_load(row3 + x), - r31 = vx_load(row3 + x + v_int32::nlanes), + r31 = vx_load(row3 + x + VTraits::vlanes()), r40 = vx_load(row4 + x), - r41 = vx_load(row4 + x + v_int32::nlanes); - v_store(dst + x, v_rshr_pack<8>(r00 + r40 + (r20 + r20) + ((r10 + r20 + r30) << 2), - r01 + r41 + (r21 + r21) + ((r11 + r21 + r31) << 2))); + r41 = vx_load(row4 + x + VTraits::vlanes()); + v_store(dst + x, v_rshr_pack<8>(v_add(v_add(v_add(r00, r40), v_add(r20, r20)), v_shl<2>(v_add(v_add(r10, r20), r30))), + v_add(v_add(v_add(r01, r41), v_add(r21, r21)), v_shl<2>(v_add(v_add(r11, r21), r31))))); } - if (x <= width - v_int32::nlanes) + if (x <= width - VTraits::vlanes()) { v_int32 r00 = vx_load(row0 + x), r10 = vx_load(row1 + x), r20 = vx_load(row2 + x), r30 = vx_load(row3 + x), r40 = vx_load(row4 + x); - v_rshr_pack_store<8>(dst + x, r00 + r40 + (r20 + r20) + ((r10 + r20 + r30) << 2)); - x += v_int32::nlanes; + v_rshr_pack_store<8>(dst + x, v_add(v_add(v_add(r00, r40), v_add(r20, r20)), v_shl<2>(v_add(v_add(r10, r20), r30)))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -599,39 +591,55 @@ template <> int PyrUpVecV(int** src, uchar** dst, int width) uchar *dst0 = dst[0], *dst1 = dst[1]; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes)), - v_r01 = v_pack(vx_load(row0 + x + 2 * v_int32::nlanes), vx_load(row0 + x + 3 * v_int32::nlanes)), - v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes)), - v_r11 = v_pack(vx_load(row1 + x + 2 * v_int32::nlanes), vx_load(row1 + x + 3 * v_int32::nlanes)), - v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes)), - v_r21 = v_pack(vx_load(row2 + x + 2 * v_int32::nlanes), vx_load(row2 + x + 3 * v_int32::nlanes)); - v_int16 v_2r10 = v_r10 + v_r10, v_2r11 = (v_r11 + v_r11); - v_store(dst0 + x, v_rshr_pack_u<6>(v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10), v_r01 + v_r21 + (v_2r11 + v_2r11 + v_2r11))); - v_store(dst1 + x, v_rshr_pack_u<6>((v_r10 + v_r20) << 2, (v_r11 + v_r21) << 2)); + v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes())), + v_r01 = v_pack(vx_load(row0 + x + 2 * VTraits::vlanes()), vx_load(row0 + x + 3 * VTraits::vlanes())), + v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes())), + v_r11 = v_pack(vx_load(row1 + x + 2 * VTraits::vlanes()), vx_load(row1 + x + 3 * VTraits::vlanes())), + v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes())), + v_r21 = v_pack(vx_load(row2 + x + 2 * VTraits::vlanes()), vx_load(row2 + x + 3 * VTraits::vlanes())); + v_int16 v_2r10 = v_add(v_r10, v_r10), v_2r11 = (v_add(v_r11, v_r11)); + v_store(dst0 + x, v_rshr_pack_u<6>(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_add(v_add(v_r01, v_r21), v_add(v_add(v_2r11, v_2r11), v_2r11)))); + v_store(dst1 + x, v_rshr_pack_u<6>(v_shl<2>(v_add(v_r10, v_r20)), v_shl<2>(v_add(v_r11, v_r21)))); } - if(x <= width - v_uint16::nlanes) + if(x <= width - VTraits::vlanes()) { - v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes)), - v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes)), - v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes)); - v_int16 v_2r10 = v_r10 + v_r10; - v_rshr_pack_u_store<6>(dst0 + x, v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10)); - v_rshr_pack_u_store<6>(dst1 + x, (v_r10 + v_r20) << 2); - x += v_uint16::nlanes; + v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes())), + v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes())), + v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes())); + v_int16 v_2r10 = v_add(v_r10, v_r10); + v_rshr_pack_u_store<6>(dst0 + x, v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10))); + v_rshr_pack_u_store<6>(dst1 + x, v_shl<2>(v_add(v_r10, v_r20))); + x += VTraits::vlanes(); } + #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_int32 v_2r10 = v_r10 + v_r10; - v_int16 d = v_pack(v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10), (v_r10 + v_r20) << 2); + v_int32 v_2r10 = v_add(v_r10, v_r10); + v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20))); *(unaligned_int*)(dst0 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0(); *(unaligned_int*)(dst1 + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16())).get0(); } + #else + for (; x <= width - 1; x += 1) + { + int r00 = *(row0 + x), + r10 = *(row1 + x), + r20 = *(row2 + x); + int _2r10 = r10 + r10; + int d = r00 + r20 + (_2r10 + _2r10 + _2r10); + int d_shifted = (r10 + r20) << 2; + // Similar to v_rshr_pack_u<6>(d, vx_setzero_s16()).get0() + *(dst0 + x) = (int)((((unsigned int)d) + ((1 << (6 - 1)))) >> 6); + // Similar to v_rshr_pack_u<6>(v_combine_high(d, d), vx_setzero_s16()).get0() + *(dst1 + x) = (int)((((unsigned int)d_shifted) + ((1 << (6 - 1)))) >> 6); + } + #endif //CV_SIMD128 vx_cleanup(); return x; @@ -643,25 +651,25 @@ template <> int PyrUpVecV(int** src, short** dst, int width) short *dst0 = dst[0], *dst1 = dst[1]; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), - v_r01 = vx_load(row0 + x + v_int32::nlanes), + v_r01 = vx_load(row0 + x + VTraits::vlanes()), v_r10 = vx_load(row1 + x), - v_r11 = vx_load(row1 + x + v_int32::nlanes), + v_r11 = vx_load(row1 + x + VTraits::vlanes()), v_r20 = vx_load(row2 + x), - v_r21 = vx_load(row2 + x + v_int32::nlanes); - v_store(dst0 + x, v_rshr_pack<6>(v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2)), v_r01 + v_r21 + ((v_r11 << 1) + (v_r11 << 2)))); - v_store(dst1 + x, v_rshr_pack<6>((v_r10 + v_r20) << 2, (v_r11 + v_r21) << 2)); + v_r21 = vx_load(row2 + x + VTraits::vlanes()); + v_store(dst0 + x, v_rshr_pack<6>(v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10))), v_add(v_add(v_r01, v_r21), v_add(v_shl<1>(v_r11), v_shl<2>(v_r11))))); + v_store(dst1 + x, v_rshr_pack<6>(v_shl<2>(v_add(v_r10, v_r20)), v_shl<2>(v_add(v_r11, v_r21)))); } - if(x <= width - v_int32::nlanes) + if(x <= width - VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_rshr_pack_store<6>(dst0 + x, v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2))); - v_rshr_pack_store<6>(dst1 + x, (v_r10 + v_r20) << 2); - x += v_int32::nlanes; + v_rshr_pack_store<6>(dst0 + x, v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10)))); + v_rshr_pack_store<6>(dst1 + x, v_shl<2>(v_add(v_r10, v_r20))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -674,25 +682,25 @@ template <> int PyrUpVecV(int** src, ushort** dst, int width) ushort *dst0 = dst[0], *dst1 = dst[1]; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), - v_r01 = vx_load(row0 + x + v_int32::nlanes), + v_r01 = vx_load(row0 + x + VTraits::vlanes()), v_r10 = vx_load(row1 + x), - v_r11 = vx_load(row1 + x + v_int32::nlanes), + v_r11 = vx_load(row1 + x + VTraits::vlanes()), v_r20 = vx_load(row2 + x), - v_r21 = vx_load(row2 + x + v_int32::nlanes); - v_store(dst0 + x, v_rshr_pack_u<6>(v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2)), v_r01 + v_r21 + ((v_r11 << 1) + (v_r11 << 2)))); - v_store(dst1 + x, v_rshr_pack_u<6>((v_r10 + v_r20) << 2, (v_r11 + v_r21) << 2)); + v_r21 = vx_load(row2 + x + VTraits::vlanes()); + v_store(dst0 + x, v_rshr_pack_u<6>(v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10))), v_add(v_add(v_r01, v_r21), v_add(v_shl<1>(v_r11), v_shl<2>(v_r11))))); + v_store(dst1 + x, v_rshr_pack_u<6>(v_shl<2>(v_add(v_r10, v_r20)), v_shl<2>(v_add(v_r11, v_r21)))); } - if(x <= width - v_int32::nlanes) + if(x <= width - VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_rshr_pack_u_store<6>(dst0 + x, v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2))); - v_rshr_pack_u_store<6>(dst1 + x, (v_r10 + v_r20) << 2); - x += v_int32::nlanes; + v_rshr_pack_u_store<6>(dst0 + x, v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10)))); + v_rshr_pack_u_store<6>(dst1 + x, v_shl<2>(v_add(v_r10, v_r20))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -706,13 +714,13 @@ template <> int PyrUpVecV(float** src, float** dst, int width) float *dst0 = dst[0], *dst1 = dst[1]; v_float32 v_6 = vx_setall_f32(6.0f), v_scale = vx_setall_f32(1.f/64.f), v_scale4 = vx_setall_f32(1.f/16.f); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_float32 v_r0 = vx_load(row0 + x), v_r1 = vx_load(row1 + x), v_r2 = vx_load(row2 + x); - v_store(dst1 + x, v_scale4 * (v_r1 + v_r2)); - v_store(dst0 + x, v_scale * (v_muladd(v_6, v_r1, v_r0) + v_r2)); + v_store(dst1 + x, v_mul(v_scale4, v_add(v_r1, v_r2))); + v_store(dst0 + x, v_mul(v_scale, v_add(v_muladd(v_6, v_r1, v_r0), v_r2))); } vx_cleanup(); @@ -724,36 +732,50 @@ template <> int PyrUpVecVOneRow(int** src, uchar* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes)), - v_r01 = v_pack(vx_load(row0 + x + 2 * v_int32::nlanes), vx_load(row0 + x + 3 * v_int32::nlanes)), - v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes)), - v_r11 = v_pack(vx_load(row1 + x + 2 * v_int32::nlanes), vx_load(row1 + x + 3 * v_int32::nlanes)), - v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes)), - v_r21 = v_pack(vx_load(row2 + x + 2 * v_int32::nlanes), vx_load(row2 + x + 3 * v_int32::nlanes)); - v_int16 v_2r10 = v_r10 + v_r10, v_2r11 = (v_r11 + v_r11); - v_store(dst + x, v_rshr_pack_u<6>(v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10), v_r01 + v_r21 + (v_2r11 + v_2r11 + v_2r11))); + v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes())), + v_r01 = v_pack(vx_load(row0 + x + 2 * VTraits::vlanes()), vx_load(row0 + x + 3 * VTraits::vlanes())), + v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes())), + v_r11 = v_pack(vx_load(row1 + x + 2 * VTraits::vlanes()), vx_load(row1 + x + 3 * VTraits::vlanes())), + v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes())), + v_r21 = v_pack(vx_load(row2 + x + 2 * VTraits::vlanes()), vx_load(row2 + x + 3 * VTraits::vlanes())); + v_int16 v_2r10 = v_add(v_r10, v_r10), v_2r11 = (v_add(v_r11, v_r11)); + v_store(dst + x, v_rshr_pack_u<6>(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_add(v_add(v_r01, v_r21), v_add(v_add(v_2r11, v_2r11), v_2r11)))); } - if(x <= width - v_uint16::nlanes) + if(x <= width - VTraits::vlanes()) { - v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + v_int32::nlanes)), - v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + v_int32::nlanes)), - v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + v_int32::nlanes)); - v_int16 v_2r10 = v_r10 + v_r10; - v_rshr_pack_u_store<6>(dst + x, v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10)); - x += v_uint16::nlanes; + v_int16 v_r00 = v_pack(vx_load(row0 + x), vx_load(row0 + x + VTraits::vlanes())), + v_r10 = v_pack(vx_load(row1 + x), vx_load(row1 + x + VTraits::vlanes())), + v_r20 = v_pack(vx_load(row2 + x), vx_load(row2 + x + VTraits::vlanes())); + v_int16 v_2r10 = v_add(v_r10, v_r10); + v_rshr_pack_u_store<6>(dst + x, v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10))); + x += VTraits::vlanes(); } + #if CV_SIMD128 typedef int CV_DECL_ALIGNED(1) unaligned_int; for (; x <= width - v_int32x4::nlanes; x += v_int32x4::nlanes) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_int32 v_2r10 = v_r10 + v_r10; - v_int16 d = v_pack(v_r00 + v_r20 + (v_2r10 + v_2r10 + v_2r10), (v_r10 + v_r20) << 2); + v_int32 v_2r10 = v_add(v_r10, v_r10); + v_int16 d = v_pack(v_add(v_add(v_r00, v_r20), v_add(v_add(v_2r10, v_2r10), v_2r10)), v_shl<2>(v_add(v_r10, v_r20))); *(unaligned_int*)(dst + x) = v_reinterpret_as_s32(v_rshr_pack_u<6>(d, vx_setzero_s16())).get0(); } + #else + for (; x <= width - 1; x += 1) + { + int r00 = *(row0 + x), + r10 = *(row1 + x), + r20 = *(row2 + x); + int _2r10 = r10 + r10; + int d = r00 + r20 + (_2r10 + _2r10 + _2r10); + int d_shifted = (r10 + r20) << 2; + // Similar to v_rshr_pack_u<6>(d, vx_setzero_s16()).get0() + *(dst + x) = (int)((((unsigned int)d) + ((1 << (6 - 1)))) >> 6); + } + #endif //CV_SIMD128 vx_cleanup(); return x; @@ -764,23 +786,23 @@ template <> int PyrUpVecVOneRow(int** src, short* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), - v_r01 = vx_load(row0 + x + v_int32::nlanes), + v_r01 = vx_load(row0 + x + VTraits::vlanes()), v_r10 = vx_load(row1 + x), - v_r11 = vx_load(row1 + x + v_int32::nlanes), + v_r11 = vx_load(row1 + x + VTraits::vlanes()), v_r20 = vx_load(row2 + x), - v_r21 = vx_load(row2 + x + v_int32::nlanes); - v_store(dst + x, v_rshr_pack<6>(v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2)), v_r01 + v_r21 + ((v_r11 << 1) + (v_r11 << 2)))); + v_r21 = vx_load(row2 + x + VTraits::vlanes()); + v_store(dst + x, v_rshr_pack<6>(v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10))), v_add(v_add(v_r01, v_r21), v_add(v_shl<1>(v_r11), v_shl<2>(v_r11))))); } - if(x <= width - v_int32::nlanes) + if(x <= width - VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_rshr_pack_store<6>(dst + x, v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2))); - x += v_int32::nlanes; + v_rshr_pack_store<6>(dst + x, v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10)))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -792,23 +814,23 @@ template <> int PyrUpVecVOneRow(int** src, ushort* dst, int width) int x = 0; const int *row0 = src[0], *row1 = src[1], *row2 = src[2]; - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), - v_r01 = vx_load(row0 + x + v_int32::nlanes), + v_r01 = vx_load(row0 + x + VTraits::vlanes()), v_r10 = vx_load(row1 + x), - v_r11 = vx_load(row1 + x + v_int32::nlanes), + v_r11 = vx_load(row1 + x + VTraits::vlanes()), v_r20 = vx_load(row2 + x), - v_r21 = vx_load(row2 + x + v_int32::nlanes); - v_store(dst + x, v_rshr_pack_u<6>(v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2)), v_r01 + v_r21 + ((v_r11 << 1) + (v_r11 << 2)))); + v_r21 = vx_load(row2 + x + VTraits::vlanes()); + v_store(dst + x, v_rshr_pack_u<6>(v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10))), v_add(v_add(v_r01, v_r21), v_add(v_shl<1>(v_r11), v_shl<2>(v_r11))))); } - if(x <= width - v_int32::nlanes) + if(x <= width - VTraits::vlanes()) { v_int32 v_r00 = vx_load(row0 + x), v_r10 = vx_load(row1 + x), v_r20 = vx_load(row2 + x); - v_rshr_pack_u_store<6>(dst + x, v_r00 + v_r20 + ((v_r10 << 1) + (v_r10 << 2))); - x += v_int32::nlanes; + v_rshr_pack_u_store<6>(dst + x, v_add(v_add(v_r00, v_r20), v_add(v_shl<1>(v_r10), v_shl<2>(v_r10)))); + x += VTraits::vlanes(); } vx_cleanup(); @@ -821,12 +843,12 @@ template <> int PyrUpVecVOneRow(float** src, float* dst, int width const float *row0 = src[0], *row1 = src[1], *row2 = src[2]; v_float32 v_6 = vx_setall_f32(6.0f), v_scale = vx_setall_f32(1.f/64.f); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_float32 v_r0 = vx_load(row0 + x), v_r1 = vx_load(row1 + x), v_r2 = vx_load(row2 + x); - v_store(dst + x, v_scale * (v_muladd(v_6, v_r1, v_r0) + v_r2)); + v_store(dst + x, v_mul(v_scale, v_add(v_muladd(v_6, v_r1, v_r0), v_r2))); } vx_cleanup(); diff --git a/modules/imgproc/src/resize.cpp b/modules/imgproc/src/resize.cpp index af91ca8d45..bba8cda4f7 100644 --- a/modules/imgproc/src/resize.cpp +++ b/modules/imgproc/src/resize.cpp @@ -346,8 +346,8 @@ void hlineResizeCn(uint8_t* src, int, int *o { int i = 0; ufixedpoint16 src_0(src[0]); -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_src_0 = vx_setall_u16(*((uint16_t*)&src_0)); for (; i <= dst_min - VECSZ; i += VECSZ, m += 2*VECSZ, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -358,7 +358,7 @@ void hlineResizeCn(uint8_t* src, int, int *o { *(dst++) = src_0; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) for (; i <= dst_max - 2*VECSZ; i += 2*VECSZ, m += 4*VECSZ, dst += 2*VECSZ) { v_uint16 v_src0, v_src1; @@ -384,7 +384,7 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = m[0] * px[0] + m[1] * px[1]; } src_0 = (src + ofst[dst_width - 1])[0]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_src_0 = vx_setall_u16(*((uint16_t*)&src_0)); for (; i <= dst_width - VECSZ; i += VECSZ, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -406,8 +406,8 @@ void hlineResizeCn(uint8_t* src, int, int *o } srccn; ((ufixedpoint16*)(srccn.w))[0] = src[0]; ((ufixedpoint16*)(srccn.w))[1] = src[1]; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_srccn = v_reinterpret_as_u16(vx_setall_u32(srccn.d)); for (; i <= dst_min - VECSZ/2; i += VECSZ/2, m += VECSZ, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -419,7 +419,7 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = ((ufixedpoint16*)(srccn.w))[0]; *(dst++) = ((ufixedpoint16*)(srccn.w))[1]; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) for (; i <= dst_max - VECSZ/2; i += VECSZ/2, m += VECSZ, dst += VECSZ) { v_uint16 v_src0, v_src1; @@ -440,7 +440,7 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = m[0] * px[1] + m[1] * px[3]; } ((ufixedpoint16*)(srccn.w))[0] = (src + 2 * ofst[dst_width - 1])[0]; ((ufixedpoint16*)(srccn.w))[1] = (src + 2 * ofst[dst_width - 1])[1]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_srccn = v_reinterpret_as_u16(vx_setall_u32(srccn.d)); for (; i <= dst_width - VECSZ/2; i += VECSZ/2, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -465,8 +465,8 @@ void hlineResizeCn(uint8_t* src, int, int *o ((ufixedpoint16*)(srccn.w))[1] = src[1]; ((ufixedpoint16*)(srccn.w))[2] = src[2]; ((ufixedpoint16*)(srccn.w))[3] = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_srccn = v_pack_triplets(v_reinterpret_as_u16(vx_setall_u64(srccn.q))); for (; i <= dst_min - (VECSZ+2)/3; i += VECSZ/4, m += VECSZ/2, dst += 3*VECSZ/4) // Points that fall left from src image so became equal to leftmost src point { @@ -479,14 +479,14 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = ((ufixedpoint16*)(srccn.w))[1]; *(dst++) = ((ufixedpoint16*)(srccn.w))[2]; } -#if CV_SIMD - CV_DECL_ALIGNED(CV_SIMD_WIDTH) int ofst3[VECSZ/2]; +#if (CV_SIMD || CV_SIMD_SCALABLE) + CV_DECL_ALIGNED(CV_SIMD_WIDTH) int ofst3[VTraits::max_nlanes/2]; for (; i <= dst_max - (3*VECSZ/4 + (VECSZ+2)/3); i += VECSZ/2, m += VECSZ, dst += 3*VECSZ/2) { - v_store(ofst3, vx_load(ofst + i) * vx_setall_s32(3)); + v_store(ofst3, v_mul(vx_load(ofst + i), vx_setall_s32(3))); v_uint8 v_src01, v_src23; v_uint16 v_src0, v_src1, v_src2, v_src3; - v_zip(vx_lut_quads(src, ofst3), v_reinterpret_as_u8(v_reinterpret_as_u32(vx_lut_quads(src+2, ofst3)) >> 8), v_src01, v_src23); + v_zip(vx_lut_quads(src, ofst3), v_reinterpret_as_u8(v_shr<8>(v_reinterpret_as_u32(vx_lut_quads(src+2, ofst3)))), v_src01, v_src23); v_expand(v_src01, v_src0, v_src1); v_expand(v_src23, v_src2, v_src3); @@ -514,7 +514,7 @@ void hlineResizeCn(uint8_t* src, int, int *o ((ufixedpoint16*)(srccn.w))[0] = (src + 3*ofst[dst_width - 1])[0]; ((ufixedpoint16*)(srccn.w))[1] = (src + 3*ofst[dst_width - 1])[1]; ((ufixedpoint16*)(srccn.w))[2] = (src + 3*ofst[dst_width - 1])[2]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_srccn = v_pack_triplets(v_reinterpret_as_u16(vx_setall_u64(srccn.q))); for (; i <= dst_width - (VECSZ+2)/3; i += VECSZ/4, dst += 3*VECSZ/4) // Points that fall right from src image so became equal to rightmost src point { @@ -540,8 +540,8 @@ void hlineResizeCn(uint8_t* src, int, int *o ((ufixedpoint16*)(srccn.w))[1] = src[1]; ((ufixedpoint16*)(srccn.w))[2] = src[2]; ((ufixedpoint16*)(srccn.w))[3] = src[3]; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_srccn = v_reinterpret_as_u16(vx_setall_u64(srccn.q)); for (; i <= dst_min - VECSZ/4; i += VECSZ/4, m += VECSZ/2, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -555,7 +555,7 @@ void hlineResizeCn(uint8_t* src, int, int *o *(dst++) = ((ufixedpoint16*)(srccn.w))[2]; *(dst++) = ((ufixedpoint16*)(srccn.w))[3]; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) for (; i <= dst_max - VECSZ/2; i += VECSZ/2, m += VECSZ, dst += 2*VECSZ) { v_uint16 v_src0, v_src1, v_src2, v_src3; @@ -586,7 +586,7 @@ void hlineResizeCn(uint8_t* src, int, int *o } ((ufixedpoint16*)(srccn.w))[0] = (src + 4 * ofst[dst_width - 1])[0]; ((ufixedpoint16*)(srccn.w))[1] = (src + 4 * ofst[dst_width - 1])[1]; ((ufixedpoint16*)(srccn.w))[2] = (src + 4 * ofst[dst_width - 1])[2]; ((ufixedpoint16*)(srccn.w))[3] = (src + 4 * ofst[dst_width - 1])[3]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_srccn = v_reinterpret_as_u16(vx_setall_u64(srccn.q)); for (; i <= dst_width - VECSZ/4; i += VECSZ/4, dst += VECSZ) // Points that fall right from src image so became equal to rightmost src point { @@ -606,8 +606,8 @@ void hlineResizeCn(uint16_t* src, int, int { int i = 0; ufixedpoint32 src_0(src[0]); -#if CV_SIMD - const int VECSZ = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint32 v_src_0 = vx_setall_u32(*((uint32_t*)&src_0)); for (; i <= dst_min - VECSZ; i += VECSZ, m += 2*VECSZ, dst += VECSZ) // Points that fall left from src image so became equal to leftmost src point { @@ -618,16 +618,16 @@ void hlineResizeCn(uint16_t* src, int, int { *(dst++) = src_0; } -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) for (; i <= dst_max - VECSZ; i += VECSZ, m += 2*VECSZ, dst += VECSZ) { v_uint32 v_src0, v_src1; v_expand(vx_lut_pairs(src, ofst + i), v_src0, v_src1); - v_uint64 v_res0 = v_reinterpret_as_u64(v_src0 * vx_load((uint32_t*)m)); - v_uint64 v_res1 = v_reinterpret_as_u64(v_src1 * vx_load((uint32_t*)m + VECSZ)); - v_store((uint32_t*)dst, v_pack((v_res0 & vx_setall_u64(0xFFFFFFFF)) + (v_res0 >> 32), - (v_res1 & vx_setall_u64(0xFFFFFFFF)) + (v_res1 >> 32))); + v_uint64 v_res0 = v_reinterpret_as_u64(v_mul(v_src0, vx_load((uint32_t *)m))); + v_uint64 v_res1 = v_reinterpret_as_u64(v_mul(v_src1, vx_load((uint32_t *)m + VECSZ))); + v_store((uint32_t*)dst, v_pack(v_add(v_and(v_res0, vx_setall_u64(0xFFFFFFFF)), v_shr<32>(v_res0)), + v_add(v_and(v_res1, vx_setall_u64(0xFFFFFFFF)), v_shr<32>(v_res1)))); } #endif for (; i < dst_max; i += 1, m += 2) @@ -636,7 +636,7 @@ void hlineResizeCn(uint16_t* src, int, int *(dst++) = m[0] * px[0] + m[1] * px[1]; } src_0 = (src + ofst[dst_width - 1])[0]; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_src_0 = vx_setall_u32(*((uint32_t*)&src_0)); for (; i <= dst_width - VECSZ; i += VECSZ, dst += VECSZ) { @@ -659,16 +659,16 @@ template <> void vlineSet(ufixedpoint16* src, uint8_t* dst, int dst_width) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint8::nlanes; - static const v_uint16 v_fixedRound = vx_setall_u16((uint16_t)((1U << 8) >> 1)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); + const v_uint16 v_fixedRound = vx_setall_u16((uint16_t)((1U << 8) >> 1)); for (; i <= dst_width - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) { v_uint16 v_src0 = vx_load((uint16_t*)src); v_uint16 v_src1 = vx_load((uint16_t*)src + VECSZ/2); - v_uint16 v_res0 = (v_src0 + v_fixedRound) >> 8; - v_uint16 v_res1 = (v_src1 + v_fixedRound) >> 8; + v_uint16 v_res0 = v_shr<8>(v_add(v_src0, v_fixedRound)); + v_uint16 v_res1 = v_shr<8>(v_add(v_src1, v_fixedRound)); v_store(dst, v_pack(v_res0, v_res1)); } @@ -693,11 +693,11 @@ void vlineResize(ufixedpoint16* src, size_t src_step, { int i = 0; ufixedpoint16* src1 = src + src_step; -#if CV_SIMD - const int VECSZ = v_uint8::nlanes; - static const v_int32 v_fixedRound = vx_setall_s32((int32_t)((1 << 16) >> 1)); - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1<<15)); - static const v_int8 v_128_16 = v_reinterpret_as_s8 (vx_setall_u8 ((uint8_t) 1<<7)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); + const v_int32 v_fixedRound = vx_setall_s32((int32_t)((1 << 16) >> 1)); + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1<<15)); + const v_int8 v_128_16 = v_reinterpret_as_s8 (vx_setall_u8 ((uint8_t) 1<<7)); v_int16 v_mul = v_reinterpret_as_s16(vx_setall_u32(((uint32_t*)m)[0])); for (; i <= dst_width - VECSZ; i += VECSZ, src += VECSZ, src1 += VECSZ, dst += VECSZ) @@ -716,10 +716,10 @@ void vlineResize(ufixedpoint16* src, size_t src_step, v_int32 v_res2 = v_dotprod(v_tmp0, v_mul); v_int32 v_res3 = v_dotprod(v_tmp1, v_mul); - v_int8 v_res = v_pack(v_pack((v_res0 + v_fixedRound) >> 16, - (v_res1 + v_fixedRound) >> 16), - v_pack((v_res2 + v_fixedRound) >> 16, - (v_res3 + v_fixedRound) >> 16)); + v_int8 v_res = v_pack(v_pack(v_shr<16>(v_add(v_res0, v_fixedRound)), + v_shr<16>(v_add(v_res1, v_fixedRound))), + v_pack(v_shr<16>(v_add(v_res2, v_fixedRound)), + v_shr<16>(v_add(v_res3, v_fixedRound)))); v_store(dst, v_reinterpret_as_u8(v_sub_wrap(v_res, v_128_16))); } @@ -828,7 +828,7 @@ public: hResize((ET*)(src + (src_height - 1) * src_step), cn, xoffsets, xcoeffs, endline, min_x, max_x, dst_width); for (; dy < range.end; dy++) vlineSet(endline, (ET*)(dst + dst_step * dy), dst_width*cn); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) vx_cleanup(); #endif } @@ -1136,16 +1136,16 @@ public: switch( pix_size ) { case 1: -#if CV_SIMD - for( ; x <= dsize.width - v_uint8::nlanes; x += v_uint8::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; x <= dsize.width - VTraits::vlanes(); x += VTraits::vlanes() ) v_store(D + x, vx_lut(S, x_ofse + x)); #endif for( ; x < dsize.width; x++ ) D[x] = S[x_ofse[x]]; break; case 2: -#if CV_SIMD - for( ; x <= dsize.width - v_uint16::nlanes; x += v_uint16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; x <= dsize.width - VTraits::vlanes(); x += VTraits::vlanes() ) v_store((ushort*)D + x, vx_lut((ushort*)S, x_ofse + x)); #endif for( ; x < dsize.width; x++ ) @@ -1159,8 +1159,8 @@ public: } break; case 4: -#if CV_SIMD - for( ; x <= dsize.width - v_uint32::nlanes; x += v_uint32::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; x <= dsize.width - VTraits::vlanes(); x += VTraits::vlanes() ) v_store((uint32_t*)D + x, vx_lut((uint32_t*)S, x_ofse + x)); #endif for( ; x < dsize.width; x++ ) @@ -1175,8 +1175,8 @@ public: } break; case 8: -#if CV_SIMD - for( ; x <= dsize.width - v_uint64::nlanes; x += v_uint64::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( ; x <= dsize.width - VTraits::vlanes(); x += VTraits::vlanes() ) v_store((uint64_t*)D + x, vx_lut((uint64_t*)S, x_ofse + x)); #endif for( ; x < dsize.width; x++ ) @@ -1250,7 +1250,7 @@ struct HResizeNoVec } }; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) struct VResizeLinearVec_32s8u { @@ -1260,22 +1260,17 @@ struct VResizeLinearVec_32s8u int x = 0; v_int16 b0 = vx_setall_s16(beta[0]), b1 = vx_setall_s16(beta[1]); - if( (((size_t)S0|(size_t)S1)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) - v_store(dst + x, v_rshr_pack_u<2>(v_mul_hi(v_pack(vx_load_aligned(S0 + x ) >> 4, vx_load_aligned(S0 + x + v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load_aligned(S1 + x ) >> 4, vx_load_aligned(S1 + x + v_int32::nlanes) >> 4), b1), - v_mul_hi(v_pack(vx_load_aligned(S0 + x + 2 * v_int32::nlanes) >> 4, vx_load_aligned(S0 + x + 3 * v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load_aligned(S1 + x + 2 * v_int32::nlanes) >> 4, vx_load_aligned(S1 + x + 3 * v_int32::nlanes) >> 4), b1))); + if( (((size_t)S0|(size_t)S1)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_rshr_pack_u<2>(v_add(v_mul_hi(v_pack(v_shr<4>(vx_load_aligned(S0 + x)), v_shr<4>(vx_load_aligned(S0 + x + VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load_aligned(S1 + x)), v_shr<4>(vx_load_aligned(S1 + x + VTraits::vlanes()))), b1)), + v_add(v_mul_hi(v_pack(v_shr<4>(vx_load_aligned(S0 + x + 2 * VTraits::vlanes())), v_shr<4>(vx_load_aligned(S0 + x + 3 * VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load_aligned(S1 + x + 2 * VTraits::vlanes())), v_shr<4>(vx_load_aligned(S1 + x + 3 * VTraits::vlanes()))), b1)))); else - for( ; x <= width - v_uint8::nlanes; x += v_uint8::nlanes) - v_store(dst + x, v_rshr_pack_u<2>(v_mul_hi(v_pack(vx_load(S0 + x ) >> 4, vx_load(S0 + x + v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load(S1 + x ) >> 4, vx_load(S1 + x + v_int32::nlanes) >> 4), b1), - v_mul_hi(v_pack(vx_load(S0 + x + 2 * v_int32::nlanes) >> 4, vx_load(S0 + x + 3 * v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load(S1 + x + 2 * v_int32::nlanes) >> 4, vx_load(S1 + x + 3 * v_int32::nlanes) >> 4), b1))); + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_rshr_pack_u<2>(v_add(v_mul_hi(v_pack(v_shr<4>(vx_load(S0 + x)), v_shr<4>(vx_load(S0 + x + VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load(S1 + x)), v_shr<4>(vx_load(S1 + x + VTraits::vlanes()))), b1)), + v_add(v_mul_hi(v_pack(v_shr<4>(vx_load(S0 + x + 2 * VTraits::vlanes())), v_shr<4>(vx_load(S0 + x + 3 * VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load(S1 + x + 2 * VTraits::vlanes())), v_shr<4>(vx_load(S1 + x + 3 * VTraits::vlanes()))), b1)))); - for( ; x < width - v_int16::nlanes; x += v_int16::nlanes) - v_rshr_pack_u_store<2>(dst + x, v_mul_hi(v_pack(vx_load(S0 + x) >> 4, vx_load(S0 + x + v_int32::nlanes) >> 4), b0) + - v_mul_hi(v_pack(vx_load(S1 + x) >> 4, vx_load(S1 + x + v_int32::nlanes) >> 4), b1)); + for( ; x < width - VTraits::vlanes(); x += VTraits::vlanes()) + v_rshr_pack_u_store<2>(dst + x, v_add(v_mul_hi(v_pack(v_shr<4>(vx_load(S0 + x)), v_shr<4>(vx_load(S0 + x + VTraits::vlanes()))), b0), v_mul_hi(v_pack(v_shr<4>(vx_load(S1 + x)), v_shr<4>(vx_load(S1 + x + VTraits::vlanes()))), b1))); return x; } @@ -1290,17 +1285,17 @@ struct VResizeLinearVec_32f16u v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]); - if( (((size_t)S0|(size_t)S1)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) - v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load_aligned(S0 + x ), b0, vx_load_aligned(S1 + x ) * b1)), - v_round(v_muladd(vx_load_aligned(S0 + x + v_float32::nlanes), b0, vx_load_aligned(S1 + x + v_float32::nlanes) * b1)))); + if( (((size_t)S0|(size_t)S1)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load_aligned(S0 + x ), b0, v_mul(vx_load_aligned(S1 + x), b1))), + v_round(v_muladd(vx_load_aligned(S0 + x + VTraits::vlanes()), b0, v_mul(vx_load_aligned(S1 + x + VTraits::vlanes()), b1))))); else - for (; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) - v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load(S0 + x ), b0, vx_load(S1 + x ) * b1)), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, vx_load(S1 + x + v_float32::nlanes) * b1)))); - for( ; x < width - v_float32::nlanes; x += v_float32::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load(S0 + x ), b0, v_mul(vx_load(S1 + x), b1))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, v_mul(vx_load(S1 + x + VTraits::vlanes()), b1))))); + for( ; x < width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_int32 t0 = v_round(v_muladd(vx_load(S0 + x), b0, vx_load(S1 + x) * b1)); + v_int32 t0 = v_round(v_muladd(vx_load(S0 + x), b0, v_mul(vx_load(S1 + x), b1))); v_store_low(dst + x, v_pack_u(t0, t0)); } @@ -1317,17 +1312,17 @@ struct VResizeLinearVec_32f16s v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]); - if( (((size_t)S0|(size_t)S1)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) - v_store(dst + x, v_pack(v_round(v_muladd(vx_load_aligned(S0 + x ), b0, vx_load_aligned(S1 + x ) * b1)), - v_round(v_muladd(vx_load_aligned(S0 + x + v_float32::nlanes), b0, vx_load_aligned(S1 + x + v_float32::nlanes) * b1)))); + if( (((size_t)S0|(size_t)S1)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_pack(v_round(v_muladd(vx_load_aligned(S0 + x ), b0, v_mul(vx_load_aligned(S1 + x), b1))), + v_round(v_muladd(vx_load_aligned(S0 + x + VTraits::vlanes()), b0, v_mul(vx_load_aligned(S1 + x + VTraits::vlanes()), b1))))); else - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes) - v_store(dst + x, v_pack(v_round(v_muladd(vx_load(S0 + x ), b0, vx_load(S1 + x ) * b1)), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, vx_load(S1 + x + v_float32::nlanes) * b1)))); - for( ; x < width - v_float32::nlanes; x += v_float32::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_pack(v_round(v_muladd(vx_load(S0 + x ), b0, v_mul(vx_load(S1 + x), b1))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, v_mul(vx_load(S1 + x + VTraits::vlanes()), b1))))); + for( ; x < width - VTraits::vlanes(); x += VTraits::vlanes()) { - v_int32 t0 = v_round(v_muladd(vx_load(S0 + x), b0, vx_load(S1 + x) * b1)); + v_int32 t0 = v_round(v_muladd(vx_load(S0 + x), b0, v_mul(vx_load(S1 + x), b1))); v_store_low(dst + x, v_pack(t0, t0)); } @@ -1344,12 +1339,12 @@ struct VResizeLinearVec_32f v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]); - if( (((size_t)S0|(size_t)S1)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) - v_store(dst + x, v_muladd(vx_load_aligned(S0 + x), b0, vx_load_aligned(S1 + x) * b1)); + if( (((size_t)S0|(size_t)S1)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_muladd(vx_load_aligned(S0 + x), b0, v_mul(vx_load_aligned(S1 + x), b1))); else - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) - v_store(dst + x, v_muladd(vx_load(S0 + x), b0, vx_load(S1 + x) * b1)); + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) + v_store(dst + x, v_muladd(vx_load(S0 + x), b0, v_mul(vx_load(S1 + x), b1))); return x; } @@ -1367,26 +1362,26 @@ struct VResizeCubicVec_32s8u v_float32 b0 = vx_setall_f32(beta[0] * scale), b1 = vx_setall_f32(beta[1] * scale), b2 = vx_setall_f32(beta[2] * scale), b3 = vx_setall_f32(beta[3] * scale); - if( (((size_t)S0|(size_t)S1|(size_t)S2|(size_t)S3)&(CV_SIMD_WIDTH - 1)) == 0 ) - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + if( (((size_t)S0|(size_t)S1|(size_t)S2|(size_t)S3)&(VTraits::vlanes() - 1)) == 0 ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_pack_u_store(dst + x, v_pack(v_round(v_muladd(v_cvt_f32(vx_load_aligned(S0 + x )), b0, v_muladd(v_cvt_f32(vx_load_aligned(S1 + x )), b1, v_muladd(v_cvt_f32(vx_load_aligned(S2 + x )), b2, - v_cvt_f32(vx_load_aligned(S3 + x )) * b3)))), - v_round(v_muladd(v_cvt_f32(vx_load_aligned(S0 + x + v_float32::nlanes)), b0, - v_muladd(v_cvt_f32(vx_load_aligned(S1 + x + v_float32::nlanes)), b1, - v_muladd(v_cvt_f32(vx_load_aligned(S2 + x + v_float32::nlanes)), b2, - v_cvt_f32(vx_load_aligned(S3 + x + v_float32::nlanes)) * b3)))))); + v_mul(v_cvt_f32(vx_load_aligned(S3 + x)), b3))))), + v_round(v_muladd(v_cvt_f32(vx_load_aligned(S0 + x + VTraits::vlanes())), b0, + v_muladd(v_cvt_f32(vx_load_aligned(S1 + x + VTraits::vlanes())), b1, + v_muladd(v_cvt_f32(vx_load_aligned(S2 + x + VTraits::vlanes())), b2, + v_mul(v_cvt_f32(vx_load_aligned(S3 + x + VTraits::vlanes())), b3))))))); else - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_pack_u_store(dst + x, v_pack(v_round(v_muladd(v_cvt_f32(vx_load(S0 + x )), b0, v_muladd(v_cvt_f32(vx_load(S1 + x )), b1, v_muladd(v_cvt_f32(vx_load(S2 + x )), b2, - v_cvt_f32(vx_load(S3 + x )) * b3)))), - v_round(v_muladd(v_cvt_f32(vx_load(S0 + x + v_float32::nlanes)), b0, - v_muladd(v_cvt_f32(vx_load(S1 + x + v_float32::nlanes)), b1, - v_muladd(v_cvt_f32(vx_load(S2 + x + v_float32::nlanes)), b2, - v_cvt_f32(vx_load(S3 + x + v_float32::nlanes)) * b3)))))); + v_mul(v_cvt_f32(vx_load(S3 + x)), b3))))), + v_round(v_muladd(v_cvt_f32(vx_load(S0 + x + VTraits::vlanes())), b0, + v_muladd(v_cvt_f32(vx_load(S1 + x + VTraits::vlanes())), b1, + v_muladd(v_cvt_f32(vx_load(S2 + x + VTraits::vlanes())), b2, + v_mul(v_cvt_f32(vx_load(S3 + x + VTraits::vlanes())), b3))))))); return x; } }; @@ -1400,15 +1395,15 @@ struct VResizeCubicVec_32f16u v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]), b2 = vx_setall_f32(beta[2]), b3 = vx_setall_f32(beta[3]); - for (; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load(S0 + x ), b0, v_muladd(vx_load(S1 + x ), b1, v_muladd(vx_load(S2 + x ), b2, - vx_load(S3 + x ) * b3)))), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, - v_muladd(vx_load(S1 + x + v_float32::nlanes), b1, - v_muladd(vx_load(S2 + x + v_float32::nlanes), b2, - vx_load(S3 + x + v_float32::nlanes) * b3)))))); + v_mul(vx_load(S3 + x), b3))))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, + v_muladd(vx_load(S1 + x + VTraits::vlanes()), b1, + v_muladd(vx_load(S2 + x + VTraits::vlanes()), b2, + v_mul(vx_load(S3 + x + VTraits::vlanes()), b3))))))); return x; } @@ -1423,15 +1418,15 @@ struct VResizeCubicVec_32f16s v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]), b2 = vx_setall_f32(beta[2]), b3 = vx_setall_f32(beta[3]); - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_pack(v_round(v_muladd(vx_load(S0 + x ), b0, v_muladd(vx_load(S1 + x ), b1, v_muladd(vx_load(S2 + x ), b2, - vx_load(S3 + x ) * b3)))), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, - v_muladd(vx_load(S1 + x + v_float32::nlanes), b1, - v_muladd(vx_load(S2 + x + v_float32::nlanes), b2, - vx_load(S3 + x + v_float32::nlanes) * b3)))))); + v_mul(vx_load(S3 + x), b3))))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, + v_muladd(vx_load(S1 + x + VTraits::vlanes()), b1, + v_muladd(vx_load(S2 + x + VTraits::vlanes()), b2, + v_mul(vx_load(S3 + x + VTraits::vlanes()), b3))))))); return x; } @@ -1446,11 +1441,11 @@ struct VResizeCubicVec_32f v_float32 b0 = vx_setall_f32(beta[0]), b1 = vx_setall_f32(beta[1]), b2 = vx_setall_f32(beta[2]), b3 = vx_setall_f32(beta[3]); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_muladd(vx_load(S0 + x), b0, v_muladd(vx_load(S1 + x), b1, v_muladd(vx_load(S2 + x), b2, - vx_load(S3 + x) * b3)))); + v_mul(vx_load(S3 + x), b3))))); return x; } @@ -1484,7 +1479,7 @@ struct VResizeLanczos4Vec_32f16u b4 = vx_setall_f32(beta[4]), b5 = vx_setall_f32(beta[5]), b6 = vx_setall_f32(beta[6]), b7 = vx_setall_f32(beta[7]); - for( ; x <= width - v_uint16::nlanes; x += v_uint16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_pack_u(v_round(v_muladd(vx_load(S0 + x ), b0, v_muladd(vx_load(S1 + x ), b1, v_muladd(vx_load(S2 + x ), b2, @@ -1492,15 +1487,15 @@ struct VResizeLanczos4Vec_32f16u v_muladd(vx_load(S4 + x ), b4, v_muladd(vx_load(S5 + x ), b5, v_muladd(vx_load(S6 + x ), b6, - vx_load(S7 + x ) * b7)))))))), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, - v_muladd(vx_load(S1 + x + v_float32::nlanes), b1, - v_muladd(vx_load(S2 + x + v_float32::nlanes), b2, - v_muladd(vx_load(S3 + x + v_float32::nlanes), b3, - v_muladd(vx_load(S4 + x + v_float32::nlanes), b4, - v_muladd(vx_load(S5 + x + v_float32::nlanes), b5, - v_muladd(vx_load(S6 + x + v_float32::nlanes), b6, - vx_load(S7 + x + v_float32::nlanes) * b7)))))))))); + v_mul(vx_load(S7 + x ), b7))))))))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, + v_muladd(vx_load(S1 + x + VTraits::vlanes()), b1, + v_muladd(vx_load(S2 + x + VTraits::vlanes()), b2, + v_muladd(vx_load(S3 + x + VTraits::vlanes()), b3, + v_muladd(vx_load(S4 + x + VTraits::vlanes()), b4, + v_muladd(vx_load(S5 + x + VTraits::vlanes()), b5, + v_muladd(vx_load(S6 + x + VTraits::vlanes()), b6, + v_mul(vx_load(S7 + x + VTraits::vlanes()), b7))))))))))); return x; } @@ -1520,7 +1515,7 @@ struct VResizeLanczos4Vec_32f16s b4 = vx_setall_f32(beta[4]), b5 = vx_setall_f32(beta[5]), b6 = vx_setall_f32(beta[6]), b7 = vx_setall_f32(beta[7]); - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_pack(v_round(v_muladd(vx_load(S0 + x ), b0, v_muladd(vx_load(S1 + x ), b1, v_muladd(vx_load(S2 + x ), b2, @@ -1528,15 +1523,15 @@ struct VResizeLanczos4Vec_32f16s v_muladd(vx_load(S4 + x ), b4, v_muladd(vx_load(S5 + x ), b5, v_muladd(vx_load(S6 + x ), b6, - vx_load(S7 + x ) * b7)))))))), - v_round(v_muladd(vx_load(S0 + x + v_float32::nlanes), b0, - v_muladd(vx_load(S1 + x + v_float32::nlanes), b1, - v_muladd(vx_load(S2 + x + v_float32::nlanes), b2, - v_muladd(vx_load(S3 + x + v_float32::nlanes), b3, - v_muladd(vx_load(S4 + x + v_float32::nlanes), b4, - v_muladd(vx_load(S5 + x + v_float32::nlanes), b5, - v_muladd(vx_load(S6 + x + v_float32::nlanes), b6, - vx_load(S7 + x + v_float32::nlanes) * b7)))))))))); + v_mul(vx_load(S7 + x), b7))))))))), + v_round(v_muladd(vx_load(S0 + x + VTraits::vlanes()), b0, + v_muladd(vx_load(S1 + x + VTraits::vlanes()), b1, + v_muladd(vx_load(S2 + x + VTraits::vlanes()), b2, + v_muladd(vx_load(S3 + x + VTraits::vlanes()), b3, + v_muladd(vx_load(S4 + x + VTraits::vlanes()), b4, + v_muladd(vx_load(S5 + x + VTraits::vlanes()), b5, + v_muladd(vx_load(S6 + x + VTraits::vlanes()), b6, + v_mul(vx_load(S7 + x + VTraits::vlanes()), b7))))))))))); return x; } @@ -1555,7 +1550,7 @@ struct VResizeLanczos4Vec_32f b4 = vx_setall_f32(beta[4]), b5 = vx_setall_f32(beta[5]), b6 = vx_setall_f32(beta[6]), b7 = vx_setall_f32(beta[7]); - for( ; x <= width - v_float32::nlanes; x += v_float32::nlanes) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) v_store(dst + x, v_muladd(vx_load(S0 + x), b0, v_muladd(vx_load(S1 + x), b1, v_muladd(vx_load(S2 + x), b2, @@ -1563,7 +1558,7 @@ struct VResizeLanczos4Vec_32f v_muladd(vx_load(S4 + x), b4, v_muladd(vx_load(S5 + x), b5, v_muladd(vx_load(S6 + x), b6, - vx_load(S7 + x) * b7)))))))); + v_mul(vx_load(S7 + x), b7))))))))); return x; } @@ -1620,8 +1615,8 @@ struct HResizeLinearVec_X4 DVT s1(S0[sx0+cn], S0[sx1+cn], S0[sx2+cn], S0[sx3+cn]); DVT s0_u(S1[sx0], S1[sx1], S1[sx2], S1[sx3]); DVT s1_u(S1[sx0+cn], S1[sx1+cn], S1[sx2+cn], S1[sx3+cn]); - v_store(&D1[dx], s0_u * a_even + s1_u * a_odd); - v_store(&D0[dx], s0 * a_even + s1 * a_odd); + v_store(&D1[dx], v_add(v_mul(s0_u, a_even), v_mul(s1_u, a_odd))); + v_store(&D0[dx], v_add(v_mul(s0, a_even), v_mul(s1, a_odd))); } } for( ; k < count; k++ ) @@ -1640,7 +1635,7 @@ struct HResizeLinearVec_X4 v_load_deinterleave(&alpha[dx*2], a_even, a_odd); DVT s0(S[sx0], S[sx1], S[sx2], S[sx3]); DVT s1(S[sx0+cn], S[sx1+cn], S[sx2+cn], S[sx3+cn]); - v_store(&D[dx], s0 * a_even + s1 * a_odd); + v_store(&D[dx], v_add(v_mul(s0, a_even), v_mul(s1, a_odd))); } } return dx; @@ -1752,8 +1747,8 @@ struct HResizeLinearVecU8_X4 for( dx = 0; (xofs[dx] + cn) < smax; dx += cn ) { v_int16x8 a = v_load(alpha+dx*2); - v_store(&D0[dx], v_dotprod(v_reinterpret_as_s16(v_load_expand_q(S0+xofs[dx]) | (v_load_expand_q(S0+xofs[dx]+cn)<<16)), a)); - v_store(&D1[dx], v_dotprod(v_reinterpret_as_s16(v_load_expand_q(S1+xofs[dx]) | (v_load_expand_q(S1+xofs[dx]+cn)<<16)), a)); + v_store(&D0[dx], v_dotprod(v_reinterpret_as_s16(v_or(v_load_expand_q(S0 + xofs[dx]), v_shl<16>(v_load_expand_q(S0 + xofs[dx] + cn)))), a)); + v_store(&D1[dx], v_dotprod(v_reinterpret_as_s16(v_or(v_load_expand_q(S1 + xofs[dx]), v_shl<16>(v_load_expand_q(S1 + xofs[dx] + cn)))), a)); } } for( ; k < count; k++ ) @@ -1763,7 +1758,7 @@ struct HResizeLinearVecU8_X4 for( dx = 0; (xofs[dx] + cn) < smax; dx += cn ) { v_int16x8 a = v_load(alpha+dx*2); - v_store(&D[dx], v_dotprod(v_reinterpret_as_s16(v_load_expand_q(S+xofs[dx]) | (v_load_expand_q(S+xofs[dx]+cn)<<16)), a)); + v_store(&D[dx], v_dotprod(v_reinterpret_as_s16(v_or(v_load_expand_q(S + xofs[dx]), v_shl<16>(v_load_expand_q(S + xofs[dx] + cn)))), a)); } } /* Debug check to ensure truthiness that we never vector the final value. */ @@ -2452,27 +2447,27 @@ public: if (cn == 1) { v_uint16 masklow = vx_setall_u16(0x00ff); - for ( ; dx <= w - v_uint16::nlanes; dx += v_uint16::nlanes, S0 += v_uint8::nlanes, S1 += v_uint8::nlanes, D += v_uint16::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += VTraits::vlanes(), S1 += VTraits::vlanes(), D += VTraits::vlanes()) { v_uint16 r0 = v_reinterpret_as_u16(vx_load(S0)); v_uint16 r1 = v_reinterpret_as_u16(vx_load(S1)); - v_rshr_pack_store<2>(D, (r0 >> 8) + (r0 & masklow) + (r1 >> 8) + (r1 & masklow)); + v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_shr<8>(r0), v_and(r0, masklow)), v_shr<8>(r1)), v_and(r1, masklow))); } } else if (cn == 3) { if (CV_SIMD_WIDTH > 64) return 0; - for ( ; dx <= w - 3*v_uint8::nlanes; dx += 3*v_uint8::nlanes, S0 += 6*v_uint8::nlanes, S1 += 6*v_uint8::nlanes, D += 3*v_uint8::nlanes) + for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { v_uint16 t0, t1, t2, t3, t4, t5; v_uint16 s0, s1, s2, s3, s4, s5; - s0 = vx_load_expand(S0 ) + vx_load_expand(S1 ); - s1 = vx_load_expand(S0 + v_uint16::nlanes) + vx_load_expand(S1 + v_uint16::nlanes); - s2 = vx_load_expand(S0 + 2*v_uint16::nlanes) + vx_load_expand(S1 + 2*v_uint16::nlanes); - s3 = vx_load_expand(S0 + 3*v_uint16::nlanes) + vx_load_expand(S1 + 3*v_uint16::nlanes); - s4 = vx_load_expand(S0 + 4*v_uint16::nlanes) + vx_load_expand(S1 + 4*v_uint16::nlanes); - s5 = vx_load_expand(S0 + 5*v_uint16::nlanes) + vx_load_expand(S1 + 5*v_uint16::nlanes); + s0 = v_add(vx_load_expand(S0), vx_load_expand(S1)); + s1 = v_add(vx_load_expand(S0 + VTraits::vlanes()), vx_load_expand(S1 + VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 2 * VTraits::vlanes()), vx_load_expand(S1 + 2 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 4 * VTraits::vlanes()), vx_load_expand(S1 + 4 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 5 * VTraits::vlanes()), vx_load_expand(S1 + 5 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); @@ -2481,18 +2476,18 @@ public: bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; #elif CV_SIMD_WIDTH == 32 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); - bl = s0 + s3; gl = s1 + s4; rl = s2 + s5; + bl = v_add(s0, s3); gl = v_add(s1, s4); rl = v_add(s2, s5); #elif CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; #endif - s0 = vx_load_expand(S0 + 6*v_uint16::nlanes) + vx_load_expand(S1 + 6*v_uint16::nlanes); - s1 = vx_load_expand(S0 + 7*v_uint16::nlanes) + vx_load_expand(S1 + 7*v_uint16::nlanes); - s2 = vx_load_expand(S0 + 8*v_uint16::nlanes) + vx_load_expand(S1 + 8*v_uint16::nlanes); - s3 = vx_load_expand(S0 + 9*v_uint16::nlanes) + vx_load_expand(S1 + 9*v_uint16::nlanes); - s4 = vx_load_expand(S0 +10*v_uint16::nlanes) + vx_load_expand(S1 +10*v_uint16::nlanes); - s5 = vx_load_expand(S0 +11*v_uint16::nlanes) + vx_load_expand(S1 +11*v_uint16::nlanes); + s0 = v_add(vx_load_expand(S0 + 6 * VTraits::vlanes()), vx_load_expand(S1 + 6 * VTraits::vlanes())); + s1 = v_add(vx_load_expand(S0 + 7 * VTraits::vlanes()), vx_load_expand(S1 + 7 * VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 8 * VTraits::vlanes()), vx_load_expand(S1 + 8 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 9 * VTraits::vlanes()), vx_load_expand(S1 + 9 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 10 * VTraits::vlanes()), vx_load_expand(S1 + 10 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 11 * VTraits::vlanes()), vx_load_expand(S1 + 11 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); @@ -2501,7 +2496,7 @@ public: bh = t0 + t3; gh = t1 + t4; rh = t2 + t5; #elif CV_SIMD_WIDTH == 32 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); - bh = s0 + s3; gh = s1 + s4; rh = s2 + s5; + bh = v_add(s0, s3); gh = v_add(s1, s4); rh = v_add(s2, s5); #elif CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); @@ -2513,7 +2508,7 @@ public: else { CV_Assert(cn == 4); - for ( ; dx <= w - v_uint8::nlanes; dx += v_uint8::nlanes, S0 += 2*v_uint8::nlanes, S1 += 2*v_uint8::nlanes, D += v_uint8::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2*VTraits::vlanes(), S1 += 2*VTraits::vlanes(), D += VTraits::vlanes()) { v_uint32 r00, r01, r10, r11; v_load_deinterleave((uint32_t*)S0, r00, r01); @@ -2524,7 +2519,7 @@ public: v_expand(v_reinterpret_as_u8(r01), r01l, r01h); v_expand(v_reinterpret_as_u8(r10), r10l, r10h); v_expand(v_reinterpret_as_u8(r11), r11l, r11h); - v_store(D, v_rshr_pack<2>(r00l + r01l + r10l + r11l, r00h + r01h + r10h + r11h)); + v_store(D, v_rshr_pack<2>(v_add(v_add(v_add(r00l, r01l), r10l), r11l), v_add(v_add(v_add(r00h, r01h), r10h), r11h))); } } @@ -2551,11 +2546,11 @@ public: if (cn == 1) { v_uint32 masklow = vx_setall_u32(0x0000ffff); - for (; dx <= w - v_uint32::nlanes; dx += v_uint32::nlanes, S0 += v_uint16::nlanes, S1 += v_uint16::nlanes, D += v_uint32::nlanes) + for (; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += VTraits::vlanes(), S1 += VTraits::vlanes(), D += VTraits::vlanes()) { v_uint32 r0 = v_reinterpret_as_u32(vx_load(S0)); v_uint32 r1 = v_reinterpret_as_u32(vx_load(S1)); - v_rshr_pack_store<2>(D, (r0 >> 16) + (r0 & masklow) + (r1 >> 16) + (r1 & masklow)); + v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_shr<16>(r0), v_and(r0, masklow)), v_shr<16>(r1)), v_and(r1, masklow))); } } else if (cn == 3) @@ -2574,38 +2569,38 @@ public: v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3)); #endif #elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64 - for ( ; dx <= w - 3*v_uint16::nlanes; dx += 3*v_uint16::nlanes, S0 += 6*v_uint16::nlanes, S1 += 6*v_uint16::nlanes, D += 3*v_uint16::nlanes) + for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { v_uint32 t0, t1, t2, t3, t4, t5; v_uint32 s0, s1, s2, s3, s4, s5; - s0 = vx_load_expand(S0 ) + vx_load_expand(S1 ); - s1 = vx_load_expand(S0 + v_uint32::nlanes) + vx_load_expand(S1 + v_uint32::nlanes); - s2 = vx_load_expand(S0 + 2*v_uint32::nlanes) + vx_load_expand(S1 + 2*v_uint32::nlanes); - s3 = vx_load_expand(S0 + 3*v_uint32::nlanes) + vx_load_expand(S1 + 3*v_uint32::nlanes); - s4 = vx_load_expand(S0 + 4*v_uint32::nlanes) + vx_load_expand(S1 + 4*v_uint32::nlanes); - s5 = vx_load_expand(S0 + 5*v_uint32::nlanes) + vx_load_expand(S1 + 5*v_uint32::nlanes); + s0 = v_add(vx_load_expand(S0), vx_load_expand(S1)); + s1 = v_add(vx_load_expand(S0 + VTraits::vlanes()), vx_load_expand(S1 + VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 2 * VTraits::vlanes()), vx_load_expand(S1 + 2 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 4 * VTraits::vlanes()), vx_load_expand(S1 + 4 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 5 * VTraits::vlanes()), vx_load_expand(S1 + 5 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_uint32 bl, gl, rl; v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); #if CV_SIMD_WIDTH == 32 - bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; + bl = v_add(t0, t3); gl = v_add(t1, t4); rl = v_add(t2, t5); #else //CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bl = s0 + s3; gl = s1 + s4; rl = s2 + s5; #endif - s0 = vx_load_expand(S0 + 6*v_uint32::nlanes) + vx_load_expand(S1 + 6*v_uint32::nlanes); - s1 = vx_load_expand(S0 + 7*v_uint32::nlanes) + vx_load_expand(S1 + 7*v_uint32::nlanes); - s2 = vx_load_expand(S0 + 8*v_uint32::nlanes) + vx_load_expand(S1 + 8*v_uint32::nlanes); - s3 = vx_load_expand(S0 + 9*v_uint32::nlanes) + vx_load_expand(S1 + 9*v_uint32::nlanes); - s4 = vx_load_expand(S0 +10*v_uint32::nlanes) + vx_load_expand(S1 +10*v_uint32::nlanes); - s5 = vx_load_expand(S0 +11*v_uint32::nlanes) + vx_load_expand(S1 +11*v_uint32::nlanes); + s0 = v_add(vx_load_expand(S0 + 6 * VTraits::vlanes()), vx_load_expand(S1 + 6 * VTraits::vlanes())); + s1 = v_add(vx_load_expand(S0 + 7 * VTraits::vlanes()), vx_load_expand(S1 + 7 * VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 8 * VTraits::vlanes()), vx_load_expand(S1 + 8 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 9 * VTraits::vlanes()), vx_load_expand(S1 + 9 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 10 * VTraits::vlanes()), vx_load_expand(S1 + 10 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 11 * VTraits::vlanes()), vx_load_expand(S1 + 11 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_uint32 bh, gh, rh; v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); #if CV_SIMD_WIDTH == 32 - bh = t0 + t3; gh = t1 + t4; rh = t2 + t5; + bh = v_add(t0, t3); gh = v_add(t1, t4); rh = v_add(t2, t5); #else //CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bh = s0 + s3; gh = s1 + s4; rh = s2 + s5; @@ -2649,19 +2644,19 @@ public: v_store(D, v_rshr_pack<2>(r00l + r01l + r10l + r11l, r00h + r01h + r10h + r11h)); } #else - for ( ; dx <= w - v_uint32::nlanes; dx += v_uint32::nlanes, S0 += v_uint16::nlanes, S1 += v_uint16::nlanes, D += v_uint32::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += VTraits::vlanes(), S1 += VTraits::vlanes(), D += VTraits::vlanes()) { v_uint32 r0, r1, r2, r3; v_expand(vx_load(S0), r0, r1); v_expand(vx_load(S1), r2, r3); - r0 += r2; r1 += r3; + r0 = v_add(r0, r2); r1 = v_add(r1, r3); v_uint32 v_d; #if CV_SIMD_WIDTH == 16 v_d = r0 + r1; #elif CV_SIMD_WIDTH == 32 v_uint32 t0, t1; v_recombine(r0, r1, t0, t1); - v_d = t0 + t1; + v_d = v_add(t0, t1); #endif v_rshr_pack_store<2>(D, v_d); } @@ -2691,11 +2686,11 @@ public: if (cn == 1) { v_int32 masklow = vx_setall_s32(0x0000ffff); - for (; dx <= w - v_int32::nlanes; dx += v_int32::nlanes, S0 += v_int16::nlanes, S1 += v_int16::nlanes, D += v_int32::nlanes) + for (; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += VTraits::vlanes(), S1 += VTraits::vlanes(), D += VTraits::vlanes()) { v_int32 r0 = v_reinterpret_as_s32(vx_load(S0)); v_int32 r1 = v_reinterpret_as_s32(vx_load(S1)); - v_rshr_pack_store<2>(D, (r0 >> 16) + (((r0 & masklow)<<16)>>16) + (r1 >> 16) + (((r1 & masklow)<<16)>>16)); + v_rshr_pack_store<2>(D, v_add(v_add(v_add(v_shr<16>(r0), v_shr<16>(v_shl<16>(v_and(r0, masklow)))), v_shr<16>(r1)), v_shr<16>(v_shl<16>(v_and(r1, masklow))))); } } else if (cn == 3) @@ -2704,38 +2699,38 @@ public: for ( ; dx <= w - 4; dx += 3, S0 += 6, S1 += 6, D += 3) v_rshr_pack_store<2>(D, v_load_expand(S0) + v_load_expand(S0 + 3) + v_load_expand(S1) + v_load_expand(S1 + 3)); #elif CV_SIMD_WIDTH == 32 || CV_SIMD_WIDTH == 64 - for ( ; dx <= w - 3*v_int16::nlanes; dx += 3*v_int16::nlanes, S0 += 6*v_int16::nlanes, S1 += 6*v_int16::nlanes, D += 3*v_int16::nlanes) + for ( ; dx <= w - 3*VTraits::vlanes(); dx += 3*VTraits::vlanes(), S0 += 6*VTraits::vlanes(), S1 += 6*VTraits::vlanes(), D += 3*VTraits::vlanes()) { v_int32 t0, t1, t2, t3, t4, t5; v_int32 s0, s1, s2, s3, s4, s5; - s0 = vx_load_expand(S0 ) + vx_load_expand(S1 ); - s1 = vx_load_expand(S0 + v_int32::nlanes) + vx_load_expand(S1 + v_int32::nlanes); - s2 = vx_load_expand(S0 + 2*v_int32::nlanes) + vx_load_expand(S1 + 2*v_int32::nlanes); - s3 = vx_load_expand(S0 + 3*v_int32::nlanes) + vx_load_expand(S1 + 3*v_int32::nlanes); - s4 = vx_load_expand(S0 + 4*v_int32::nlanes) + vx_load_expand(S1 + 4*v_int32::nlanes); - s5 = vx_load_expand(S0 + 5*v_int32::nlanes) + vx_load_expand(S1 + 5*v_int32::nlanes); + s0 = v_add(vx_load_expand(S0), vx_load_expand(S1)); + s1 = v_add(vx_load_expand(S0 + VTraits::vlanes()), vx_load_expand(S1 + VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 2 * VTraits::vlanes()), vx_load_expand(S1 + 2 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 4 * VTraits::vlanes()), vx_load_expand(S1 + 4 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 5 * VTraits::vlanes()), vx_load_expand(S1 + 5 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_int32 bl, gl, rl; v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); #if CV_SIMD_WIDTH == 32 - bl = t0 + t3; gl = t1 + t4; rl = t2 + t5; + bl = v_add(t0, t3); gl = v_add(t1, t4); rl = v_add(t2, t5); #else //CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bl = s0 + s3; gl = s1 + s4; rl = s2 + s5; #endif - s0 = vx_load_expand(S0 + 6*v_int32::nlanes) + vx_load_expand(S1 + 6*v_int32::nlanes); - s1 = vx_load_expand(S0 + 7*v_int32::nlanes) + vx_load_expand(S1 + 7*v_int32::nlanes); - s2 = vx_load_expand(S0 + 8*v_int32::nlanes) + vx_load_expand(S1 + 8*v_int32::nlanes); - s3 = vx_load_expand(S0 + 9*v_int32::nlanes) + vx_load_expand(S1 + 9*v_int32::nlanes); - s4 = vx_load_expand(S0 +10*v_int32::nlanes) + vx_load_expand(S1 +10*v_int32::nlanes); - s5 = vx_load_expand(S0 +11*v_int32::nlanes) + vx_load_expand(S1 +11*v_int32::nlanes); + s0 = v_add(vx_load_expand(S0 + 6 * VTraits::vlanes()), vx_load_expand(S1 + 6 * VTraits::vlanes())); + s1 = v_add(vx_load_expand(S0 + 7 * VTraits::vlanes()), vx_load_expand(S1 + 7 * VTraits::vlanes())); + s2 = v_add(vx_load_expand(S0 + 8 * VTraits::vlanes()), vx_load_expand(S1 + 8 * VTraits::vlanes())); + s3 = v_add(vx_load_expand(S0 + 9 * VTraits::vlanes()), vx_load_expand(S1 + 9 * VTraits::vlanes())); + s4 = v_add(vx_load_expand(S0 + 10 * VTraits::vlanes()), vx_load_expand(S1 + 10 * VTraits::vlanes())); + s5 = v_add(vx_load_expand(S0 + 11 * VTraits::vlanes()), vx_load_expand(S1 + 11 * VTraits::vlanes())); v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); v_int32 bh, gh, rh; v_zip(s0, s3, t0, t1); v_zip(s1, s4, t2, t3); v_zip(s2, s5, t4, t5); #if CV_SIMD_WIDTH == 32 - bh = t0 + t3; gh = t1 + t4; rh = t2 + t5; + bh = v_add(t0, t3); gh = v_add(t1, t4); rh = v_add(t2, t5); #else //CV_SIMD_WIDTH == 64 v_zip(t0, t3, s0, s1); v_zip(t1, t4, s2, s3); v_zip(t2, t5, s4, s5); bh = s0 + s3; gh = s1 + s4; rh = s2 + s5; @@ -2763,7 +2758,7 @@ public: else { CV_Assert(cn == 4); - for (; dx <= w - v_int16::nlanes; dx += v_int16::nlanes, S0 += 2 * v_int16::nlanes, S1 += 2 * v_int16::nlanes, D += v_int16::nlanes) + for (; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2 * VTraits::vlanes(), S1 += 2 * VTraits::vlanes(), D += VTraits::vlanes()) { #if CV_SIMD_WIDTH >= 64 v_int64 r00, r01, r10, r11; @@ -2778,17 +2773,17 @@ public: v_store(D, v_rshr_pack<2>(r00l + r01l + r10l + r11l, r00h + r01h + r10h + r11h)); #else v_int32 r0, r1, r2, r3; - r0 = vx_load_expand(S0 ) + vx_load_expand(S1 ); - r1 = vx_load_expand(S0 + v_int32::nlanes) + vx_load_expand(S1 + v_int32::nlanes); - r2 = vx_load_expand(S0 + 2*v_int32::nlanes) + vx_load_expand(S1 + 2*v_int32::nlanes); - r3 = vx_load_expand(S0 + 3*v_int32::nlanes) + vx_load_expand(S1 + 3*v_int32::nlanes); + r0 = v_add(vx_load_expand(S0), vx_load_expand(S1)); + r1 = v_add(vx_load_expand(S0 + VTraits::vlanes()), vx_load_expand(S1 + VTraits::vlanes())); + r2 = v_add(vx_load_expand(S0 + 2 * VTraits::vlanes()), vx_load_expand(S1 + 2 * VTraits::vlanes())); + r3 = v_add(vx_load_expand(S0 + 3 * VTraits::vlanes()), vx_load_expand(S1 + 3 * VTraits::vlanes())); v_int32 dl, dh; #if CV_SIMD_WIDTH == 16 dl = r0 + r1; dh = r2 + r3; #elif CV_SIMD_WIDTH == 32 v_int32 t0, t1, t2, t3; v_recombine(r0, r1, t0, t1); v_recombine(r2, r3, t2, t3); - dl = t0 + t1; dh = t2 + t3; + dl = v_add(t0, t1); dh = v_add(t2, t3); #endif v_store(D, v_rshr_pack<2>(dl, dh)); #endif @@ -2822,12 +2817,12 @@ struct ResizeAreaFastVec_SIMD_32f if (cn == 1) { v_float32 v_025 = vx_setall_f32(0.25f); - for ( ; dx <= w - v_float32::nlanes; dx += v_float32::nlanes, S0 += 2*v_float32::nlanes, S1 += 2*v_float32::nlanes, D += v_float32::nlanes) + for ( ; dx <= w - VTraits::vlanes(); dx += VTraits::vlanes(), S0 += 2*VTraits::vlanes(), S1 += 2*VTraits::vlanes(), D += VTraits::vlanes()) { v_float32 v_row00, v_row01, v_row10, v_row11; v_load_deinterleave(S0, v_row00, v_row01); v_load_deinterleave(S1, v_row10, v_row11); - v_store(D, ((v_row00 + v_row01) + (v_row10 + v_row11)) * v_025); + v_store(D, v_mul(v_add(v_add(v_row00, v_row01), v_add(v_row10, v_row11)), v_025)); } } else if (cn == 4) @@ -2841,8 +2836,8 @@ struct ResizeAreaFastVec_SIMD_32f for (; dx <= w - v_float32x8::nlanes; dx += v_float32x8::nlanes, S0 += 2*v_float32x8::nlanes, S1 += 2*v_float32x8::nlanes, D += v_float32x8::nlanes) { v_float32x8 dst0, dst1; - v_recombine(v256_load(S0) + v256_load(S1), v256_load(S0 + v_float32x8::nlanes) + v256_load(S1 + v_float32x8::nlanes), dst0, dst1); - v_store(D, (dst0 + dst1) * v_025); + v_recombine(v_add(v256_load(S0), v256_load(S1)), v_add(v256_load(S0 + v_float32x8::nlanes), v256_load(S1 + v_float32x8::nlanes)), dst0, dst1); + v_store(D, v_mul(v_add(dst0, dst1), v_025)); } #endif } diff --git a/modules/imgproc/src/smooth.simd.hpp b/modules/imgproc/src/smooth.simd.hpp index 62ff31ac94..33e58d4e80 100644 --- a/modules/imgproc/src/smooth.simd.hpp +++ b/modules/imgproc/src/smooth.simd.hpp @@ -81,11 +81,11 @@ void hlineSmooth1N(const uint8_t* src, int cn, const ufi { int lencn = len*cn; int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; - v_uint16 v_mul = vx_setall_u16(*((uint16_t*)m)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); + v_uint16 vmul = vx_setall_u16(*((uint16_t*)m)); for (; i <= lencn - VECSZ; i += VECSZ) - v_store((uint16_t*)dst + i, v_mul_wrap(v_mul, vx_load_expand(src + i))); + v_store((uint16_t*)dst + i, v_mul(vmul, vx_load_expand(src + i))); #endif for (; i < lencn; i++) dst[i] = m[0] * src[i]; @@ -101,8 +101,8 @@ void hlineSmooth1N1(const uint8_t* src, int cn, const uf { int lencn = len*cn; int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ; i += VECSZ) v_store((uint16_t*)dst + i, v_shl<8>(vx_load_expand(src + i))); #endif @@ -168,16 +168,14 @@ void hlineSmooth3N(const uint8_t* src, int cn, const ufi src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const uint16_t* _m = (const uint16_t*)m; - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul0 = vx_setall_u16(_m[0]); v_uint16 v_mul1 = vx_setall_u16(_m[1]); v_uint16 v_mul2 = vx_setall_u16(_m[2]); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, v_mul_wrap(vx_load_expand(src - cn), v_mul0) + - v_mul_wrap(vx_load_expand(src), v_mul1) + - v_mul_wrap(vx_load_expand(src + cn), v_mul2)); + v_store((uint16_t*)dst, v_add(v_add(v_mul(vx_load_expand(src - cn), v_mul0), v_mul(vx_load_expand(src), v_mul1)), v_mul(vx_load_expand(src + cn), v_mul2))); #endif for (; i < lencn; i++, src++, dst++) *dst = m[0] * src[-cn] + m[1] * src[0] + m[2] * src[cn]; @@ -220,10 +218,10 @@ void hlineSmooth3N121Impl(const ET* src, int cn, const FT*, int, FT* dst, int le src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; -#if CV_SIMD - const int VECSZ = VFT::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((typename FT::raw_t*)dst, (vx_load_expand(src - cn) + vx_load_expand(src + cn) + (vx_load_expand(src) << 1)) << (FT::fixedShift-2)); + v_store((typename FT::raw_t*)dst, v_shl<(FT::fixedShift-2)>(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn), v_shl<1>((vx_load_expand(src)))))); #endif for (; i < lencn; i++, src++, dst++) *dst = (FT(src[-cn])>>2) + (FT(src[cn])>>2) + (FT(src[0])>>1); @@ -320,14 +318,13 @@ void hlineSmooth3Naba(const uint8_t* src, int cn, const src += cn; dst += cn; int i = cn, lencn = (len - 1)*cn; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const uint16_t* _m = (const uint16_t*)m; - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul0 = vx_setall_u16(_m[0]); v_uint16 v_mul1 = vx_setall_u16(_m[1]); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, v_mul_wrap(vx_load_expand(src - cn) + vx_load_expand(src + cn), v_mul0) + - v_mul_wrap(vx_load_expand(src), v_mul1)); + v_store((uint16_t*)dst, v_add(v_mul(v_add( vx_load_expand(src - cn), vx_load_expand(src + cn)), v_mul0), v_mul(vx_load_expand(src), v_mul1))); #endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = saturate_cast(((uint16_t*)m)[1] * (uint32_t)(src[0]) + ((uint16_t*)m)[0] * ((uint32_t)(src[-cn]) + (uint32_t)(src[cn]))); @@ -514,20 +511,16 @@ void hlineSmooth5N(const uint8_t* src, int cn, const ufi src += 2 * cn; dst += 2 * cn; int i = 2*cn, lencn = (len - 2)*cn; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const uint16_t* _m = (const uint16_t*)m; - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul0 = vx_setall_u16(_m[0]); v_uint16 v_mul1 = vx_setall_u16(_m[1]); v_uint16 v_mul2 = vx_setall_u16(_m[2]); v_uint16 v_mul3 = vx_setall_u16(_m[3]); v_uint16 v_mul4 = vx_setall_u16(_m[4]); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, v_mul_wrap(vx_load_expand(src - 2 * cn), v_mul0) + - v_mul_wrap(vx_load_expand(src - cn), v_mul1) + - v_mul_wrap(vx_load_expand(src), v_mul2) + - v_mul_wrap(vx_load_expand(src + cn), v_mul3) + - v_mul_wrap(vx_load_expand(src + 2 * cn), v_mul4)); + v_store((uint16_t*)dst, v_add(v_add(v_add(v_add(v_mul(vx_load_expand(src - 2 * cn), v_mul0), v_mul(vx_load_expand(src - cn), v_mul1)), v_mul(vx_load_expand(src), v_mul2)), v_mul(vx_load_expand(src + cn), v_mul3)), v_mul(vx_load_expand(src + 2 * cn), v_mul4))); #endif for (; i < lencn; i++, src++, dst++) *dst = m[0] * src[-2*cn] + m[1] * src[-cn] + m[2] * src[0] + m[3] * src[cn] + m[4] * src[2*cn]; @@ -726,11 +719,11 @@ void hlineSmooth5N14641(const uint8_t* src, int cn, cons src += 2 * cn; dst += 2 * cn; int i = 2 * cn, lencn = (len - 2)*cn; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_6 = vx_setall_u16(6); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, (v_mul_wrap(vx_load_expand(src), v_6) + ((vx_load_expand(src - cn) + vx_load_expand(src + cn)) << 2) + vx_load_expand(src - 2 * cn) + vx_load_expand(src + 2 * cn)) << 4); + v_store((uint16_t*)dst, v_shl<4>(v_add(v_add(v_add(v_mul(vx_load_expand(src), v_6), v_shl<2>(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn)))), vx_load_expand(src - 2 * cn)), vx_load_expand(src + 2 * cn)))); #endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = (uint16_t(src[0]) * 6 + ((uint16_t(src[-cn]) + uint16_t(src[cn])) << 2) + uint16_t(src[-2 * cn]) + uint16_t(src[2 * cn])) << 4; @@ -924,16 +917,14 @@ void hlineSmooth5Nabcba(const uint8_t* src, int cn, cons src += 2 * cn; dst += 2 * cn; int i = 2 * cn, lencn = (len - 2)*cn; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) const uint16_t* _m = (const uint16_t*)m; - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul0 = vx_setall_u16(_m[0]); v_uint16 v_mul1 = vx_setall_u16(_m[1]); v_uint16 v_mul2 = vx_setall_u16(_m[2]); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) - v_store((uint16_t*)dst, v_mul_wrap(vx_load_expand(src - 2 * cn) + vx_load_expand(src + 2 * cn), v_mul0) + - v_mul_wrap(vx_load_expand(src - cn) + vx_load_expand(src + cn), v_mul1) + - v_mul_wrap(vx_load_expand(src), v_mul2)); + v_store((uint16_t*)dst, v_add(v_add(v_mul(v_add(vx_load_expand(src - 2 * cn), vx_load_expand(src + 2 * cn)), v_mul0), v_mul(v_add(vx_load_expand(src - cn), vx_load_expand(src + cn)), v_mul1)), v_mul(vx_load_expand(src), v_mul2))); #endif for (; i < lencn; i++, src++, dst++) *((uint16_t*)dst) = saturate_cast(((uint16_t*)m)[0] * ((uint32_t)(src[-2 * cn]) + (uint32_t)(src[2 * cn])) + ((uint16_t*)m)[1] * ((uint32_t)(src[-cn]) + (uint32_t)(src[cn])) + ((uint16_t*)m)[2] * (uint32_t)(src[0])); @@ -1044,13 +1035,13 @@ void hlineSmooth(const uint8_t* src, int cn, const ufixe } i *= cn; int lencn = (len - post_shift + 1)*cn; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ; i+=VECSZ, src+=VECSZ, dst+=VECSZ) { - v_uint16 v_res0 = v_mul_wrap(vx_load_expand(src), vx_setall_u16(*((uint16_t*)m))); + v_uint16 v_res0 = v_mul(vx_load_expand(src), vx_setall_u16(*((uint16_t*)m))); for (int j = 1; j < n; j++) - v_res0 += v_mul_wrap(vx_load_expand(src + j * cn), vx_setall_u16(*((uint16_t*)(m + j)))); + v_res0 = v_add(v_res0, v_mul(vx_load_expand(src + j * cn), vx_setall_u16(*((uint16_t *)(m + j))))); v_store((uint16_t*)dst, v_res0); } #endif @@ -1163,13 +1154,13 @@ void hlineSmoothONa_yzy_a(const uint8_t* src, int cn, co } i *= cn; int lencn = (len - post_shift + 1)*cn; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ; i += VECSZ, src += VECSZ, dst += VECSZ) { - v_uint16 v_res0 = v_mul_wrap(vx_load_expand(src + pre_shift * cn), vx_setall_u16(*((uint16_t*)(m + pre_shift)))); + v_uint16 v_res0 = v_mul(vx_load_expand(src + pre_shift * cn), vx_setall_u16(*((uint16_t*)(m + pre_shift)))); for (int j = 0; j < pre_shift; j ++) - v_res0 += v_mul_wrap(vx_load_expand(src + j * cn) + vx_load_expand(src + (n - 1 - j)*cn), vx_setall_u16(*((uint16_t*)(m + j)))); + v_res0 = v_add(v_res0, v_mul(v_add(vx_load_expand(src + j * cn), vx_load_expand(src + (n - 1 - j) * cn)), vx_setall_u16(*((uint16_t *)(m + j))))); v_store((uint16_t*)dst, v_res0); } #endif @@ -1228,8 +1219,8 @@ void hlineSmoothONa_yzy_a(const uint16_t* src, int cn, } i *= cn; int lencn = (len - post_shift + 1)*cn; -#if CV_SIMD - const int VECSZ = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= lencn - VECSZ * 2; i += VECSZ * 2, src += VECSZ * 2, dst += VECSZ * 2) { v_uint32 v_res0, v_res1; @@ -1239,11 +1230,11 @@ void hlineSmoothONa_yzy_a(const uint16_t* src, int cn, v_uint16 v_weight = vx_setall_u16((uint16_t) *((uint32_t*)(m + j))); v_uint32 v_add0, v_add1; v_mul_expand(vx_load(src + j * cn), v_weight, v_add0, v_add1); - v_res0 += v_add0; - v_res1 += v_add1; + v_res0 = v_add(v_res0, v_add0); + v_res1 = v_add(v_res1, v_add1); v_mul_expand(vx_load(src + (n - 1 - j)*cn), v_weight, v_add0, v_add1); - v_res0 += v_add0; - v_res1 += v_add1; + v_res0 = v_add(v_res0, v_add0); + v_res1 = v_add(v_res1, v_add1); } v_store((uint32_t*)dst, v_res0); v_store((uint32_t*)dst + VECSZ, v_res1); @@ -1285,8 +1276,8 @@ void vlineSmooth1N(const ufixedpoint16* const * src, con { const ufixedpoint16* src0 = src[0]; int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); v_uint16 v_mul = vx_setall_u16(*((uint16_t*)m)<<1); for (; i <= len - VECSZ; i += VECSZ) v_rshr_pack_store<1>(dst + i, v_mul_hi(vx_load((uint16_t*)src0 + i), v_mul)); @@ -1306,8 +1297,8 @@ void vlineSmooth1N1(const ufixedpoint16* const * src, co { const ufixedpoint16* src0 = src[0]; int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= len - VECSZ; i += VECSZ) v_rshr_pack_store<8>(dst + i, vx_load((uint16_t*)(src0 + i))); #endif @@ -1324,10 +1315,10 @@ template <> void vlineSmooth3N(const ufixedpoint16* const * src, const ufixedpoint16* m, int, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); v_int32 v_128_4 = vx_setall_s32(128 << 16); - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); if (len >= VECSZ) { ufixedpoint32 val[] = { (m[0] + m[1] + m[2]) * ufixedpoint16((uint8_t)128) }; @@ -1370,26 +1361,26 @@ void vlineSmooth3N(const ufixedpoint16* const * src, con v_src02 = vx_load(src2 + 2*VECSZ); v_src03 = vx_load(src2 + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul2, v_resj0, v_resj1); - v_res0 += v_resj0; - v_res1 += v_resj1; + v_res0 = v_add(v_res0, v_resj0); + v_res1 = v_add(v_res1, v_resj1); v_mul_expand(v_add_wrap(v_src01, v_128), v_mul2, v_resj0, v_resj1); - v_res2 += v_resj0; - v_res3 += v_resj1; + v_res2 = v_add(v_res2, v_resj0); + v_res3 = v_add(v_res3, v_resj1); v_mul_expand(v_add_wrap(v_src02, v_128), v_mul2, v_resj0, v_resj1); - v_res4 += v_resj0; - v_res5 += v_resj1; + v_res4 = v_add(v_res4, v_resj0); + v_res5 = v_add(v_res5, v_resj1); v_mul_expand(v_add_wrap(v_src03, v_128), v_mul2, v_resj0, v_resj1); - v_res6 += v_resj0; - v_res7 += v_resj1; + v_res6 = v_add(v_res6, v_resj0); + v_res7 = v_add(v_res7, v_resj1); - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 = v_add(v_res0, v_128_4); + v_res1 = v_add(v_res1, v_128_4); + v_res2 = v_add(v_res2, v_128_4); + v_res3 = v_add(v_res3, v_128_4); + v_res4 = v_add(v_res4, v_128_4); + v_res5 = v_add(v_res5, v_128_4); + v_res6 = v_add(v_res6, v_128_4); + v_res7 = v_add(v_res7, v_128_4); v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); @@ -1410,8 +1401,8 @@ template <> void vlineSmooth3N121(const ufixedpoint16* const * src, const ufixedpoint16*, int, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint32 v_src00, v_src01, v_src02, v_src03, v_src10, v_src11, v_src12, v_src13, v_src20, v_src21, v_src22, v_src23; @@ -1421,8 +1412,8 @@ void vlineSmooth3N121(const ufixedpoint16* const * src, v_expand(vx_load((uint16_t*)(src[1]) + i + VECSZ), v_src12, v_src13); v_expand(vx_load((uint16_t*)(src[2]) + i), v_src20, v_src21); v_expand(vx_load((uint16_t*)(src[2]) + i + VECSZ), v_src22, v_src23); - v_store(dst + i, v_pack(v_rshr_pack<10>(v_src00 + v_src20 + (v_src10 + v_src10), v_src01 + v_src21 + (v_src11 + v_src11)), - v_rshr_pack<10>(v_src02 + v_src22 + (v_src12 + v_src12), v_src03 + v_src23 + (v_src13 + v_src13)))); + v_store(dst + i, v_pack(v_rshr_pack<10>(v_add(v_add(v_src00, v_src20), v_add(v_src10, v_src10)), v_add(v_add(v_src01, v_src21), v_add(v_src11, v_src11))), + v_rshr_pack<10>(v_add(v_add(v_src02, v_src22), v_add(v_src12, v_src12)), v_add(v_add(v_src03, v_src23), v_add(v_src13, v_src13))))); } #endif for (; i < len; i++) @@ -1432,8 +1423,8 @@ template <> void vlineSmooth3N121(const ufixedpoint32* const * src, const ufixedpoint32*, int, uint16_t* dst, int len) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint64 v_src00, v_src01, v_src02, v_src03, v_src10, v_src11, v_src12, v_src13, v_src20, v_src21, v_src22, v_src23; @@ -1443,8 +1434,8 @@ void vlineSmooth3N121(const ufixedpoint32* const * src, v_expand(vx_load((uint32_t*)(src[1]) + i + VECSZ), v_src12, v_src13); v_expand(vx_load((uint32_t*)(src[2]) + i), v_src20, v_src21); v_expand(vx_load((uint32_t*)(src[2]) + i + VECSZ), v_src22, v_src23); - v_store(dst + i, v_pack(v_rshr_pack<18>(v_src00 + v_src20 + (v_src10 + v_src10), v_src01 + v_src21 + (v_src11 + v_src11)), - v_rshr_pack<18>(v_src02 + v_src22 + (v_src12 + v_src12), v_src03 + v_src23 + (v_src13 + v_src13)))); + v_store(dst + i, v_pack(v_rshr_pack<18>(v_add(v_add(v_src00, v_src20), v_add(v_src10, v_src10)), v_add(v_add(v_src01, v_src21), v_add(v_src11, v_src11))), + v_rshr_pack<18>(v_add(v_add(v_src02, v_src22), v_add(v_src12, v_src12)), v_add(v_add(v_src03, v_src23), v_add(v_src13, v_src13))))); } #endif for (; i < len; i++) @@ -1460,13 +1451,13 @@ template <> void vlineSmooth5N(const ufixedpoint16* const * src, const ufixedpoint16* m, int, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint16::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); if (len >= 4 * VECSZ) { ufixedpoint32 val[] = { (m[0] + m[1] + m[2] + m[3] + m[4]) * ufixedpoint16((uint8_t)128) }; v_int32 v_128_4 = vx_setall_s32(*((int32_t*)val)); - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); v_int16 v_mul01 = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)m))); v_int16 v_mul23 = v_reinterpret_as_s16(vx_setall_u32(*((uint32_t*)(m + 2)))); v_int16 v_mul4 = v_reinterpret_as_s16(vx_setall_u16(*((uint16_t*)(m + 4)))); @@ -1509,17 +1500,17 @@ void vlineSmooth5N(const ufixedpoint16* const * src, con v_src12 = vx_load(src3 + 2*VECSZ); v_src13 = vx_load(src3 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); - v_res0 += v_dotprod(v_tmp0, v_mul23); - v_res1 += v_dotprod(v_tmp1, v_mul23); + v_res0 = v_add(v_res0, v_dotprod(v_tmp0, v_mul23)); + v_res1 = v_add(v_res1, v_dotprod(v_tmp1, v_mul23)); v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); - v_res2 += v_dotprod(v_tmp0, v_mul23); - v_res3 += v_dotprod(v_tmp1, v_mul23); + v_res2 = v_add(v_res2, v_dotprod(v_tmp0, v_mul23)); + v_res3 = v_add(v_res3, v_dotprod(v_tmp1, v_mul23)); v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); - v_res4 += v_dotprod(v_tmp0, v_mul23); - v_res5 += v_dotprod(v_tmp1, v_mul23); + v_res4 = v_add(v_res4, v_dotprod(v_tmp0, v_mul23)); + v_res5 = v_add(v_res5, v_dotprod(v_tmp1, v_mul23)); v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); - v_res6 += v_dotprod(v_tmp0, v_mul23); - v_res7 += v_dotprod(v_tmp1, v_mul23); + v_res6 = v_add(v_res6, v_dotprod(v_tmp0, v_mul23)); + v_res7 = v_add(v_res7, v_dotprod(v_tmp1, v_mul23)); v_int32 v_resj0, v_resj1; const int16_t* src4 = (const int16_t*)src[4] + i; @@ -1528,26 +1519,26 @@ void vlineSmooth5N(const ufixedpoint16* const * src, con v_src02 = vx_load(src4 + 2*VECSZ); v_src03 = vx_load(src4 + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul4, v_resj0, v_resj1); - v_res0 += v_resj0; - v_res1 += v_resj1; + v_res0 = v_add(v_res0, v_resj0); + v_res1 = v_add(v_res1, v_resj1); v_mul_expand(v_add_wrap(v_src01, v_128), v_mul4, v_resj0, v_resj1); - v_res2 += v_resj0; - v_res3 += v_resj1; + v_res2 = v_add(v_res2, v_resj0); + v_res3 = v_add(v_res3, v_resj1); v_mul_expand(v_add_wrap(v_src02, v_128), v_mul4, v_resj0, v_resj1); - v_res4 += v_resj0; - v_res5 += v_resj1; + v_res4 = v_add(v_res4, v_resj0); + v_res5 = v_add(v_res5, v_resj1); v_mul_expand(v_add_wrap(v_src03, v_128), v_mul4, v_resj0, v_resj1); - v_res6 += v_resj0; - v_res7 += v_resj1; + v_res6 = v_add(v_res6, v_resj0); + v_res7 = v_add(v_res7, v_resj1); - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 = v_add(v_res0, v_128_4); + v_res1 = v_add(v_res1, v_128_4); + v_res2 = v_add(v_res2, v_128_4); + v_res3 = v_add(v_res3, v_128_4); + v_res4 = v_add(v_res4, v_128_4); + v_res5 = v_add(v_res5, v_128_4); + v_res6 = v_add(v_res6, v_128_4); + v_res7 = v_add(v_res7, v_128_4); v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); @@ -1569,9 +1560,9 @@ template <> void vlineSmooth5N14641(const ufixedpoint16* const * src, const ufixedpoint16*, int, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_uint32 v_6 = vx_setall_u32(6); - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint32 v_src00, v_src10, v_src20, v_src30, v_src40; @@ -1588,10 +1579,10 @@ void vlineSmooth5N14641(const ufixedpoint16* const * src v_expand(vx_load((uint16_t*)(src[3]) + i + VECSZ), v_src32, v_src33); v_expand(vx_load((uint16_t*)(src[4]) + i), v_src40, v_src41); v_expand(vx_load((uint16_t*)(src[4]) + i + VECSZ), v_src42, v_src43); - v_store(dst + i, v_pack(v_rshr_pack<12>(v_src20*v_6 + ((v_src10 + v_src30) << 2) + v_src00 + v_src40, - v_src21*v_6 + ((v_src11 + v_src31) << 2) + v_src01 + v_src41), - v_rshr_pack<12>(v_src22*v_6 + ((v_src12 + v_src32) << 2) + v_src02 + v_src42, - v_src23*v_6 + ((v_src13 + v_src33) << 2) + v_src03 + v_src43))); + v_store(dst + i, v_pack(v_rshr_pack<12>(v_add(v_add(v_add(v_mul(v_src20, v_6), v_shl<2>(v_add(v_src10, v_src30))), v_src00), v_src40), + v_add(v_add(v_add(v_mul(v_src21, v_6), v_shl<2>(v_add(v_src11, v_src31))), v_src01), v_src41)), + v_rshr_pack<12>(v_add(v_add(v_add(v_mul(v_src22, v_6), v_shl<2>(v_add(v_src12, v_src32))), v_src02), v_src42), + v_add(v_add(v_add(v_mul(v_src23, v_6), v_shl<2>(v_add(v_src13, v_src33))), v_src03), v_src43)))); } #endif for (; i < len; i++) @@ -1603,8 +1594,8 @@ template <> void vlineSmooth5N14641(const ufixedpoint32* const * src, const ufixedpoint32*, int, uint16_t* dst, int len) { int i = 0; -#if CV_SIMD - const int VECSZ = v_uint32::nlanes; +#if (CV_SIMD || CV_SIMD_SCALABLE) + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint64 v_src00, v_src10, v_src20, v_src30, v_src40; @@ -1621,10 +1612,10 @@ void vlineSmooth5N14641(const ufixedpoint32* const * sr v_expand(vx_load((uint32_t*)(src[3]) + i + VECSZ), v_src32, v_src33); v_expand(vx_load((uint32_t*)(src[4]) + i), v_src40, v_src41); v_expand(vx_load((uint32_t*)(src[4]) + i + VECSZ), v_src42, v_src43); - v_store(dst + i, v_pack(v_rshr_pack<20>((v_src20 << 2) + (v_src20 << 1) + ((v_src10 + v_src30) << 2) + v_src00 + v_src40, - (v_src21 << 2) + (v_src21 << 1) + ((v_src11 + v_src31) << 2) + v_src01 + v_src41), - v_rshr_pack<20>((v_src22 << 2) + (v_src22 << 1) + ((v_src12 + v_src32) << 2) + v_src02 + v_src42, - (v_src23 << 2) + (v_src23 << 1) + ((v_src13 + v_src33) << 2) + v_src03 + v_src43))); + v_store(dst + i, v_pack(v_rshr_pack<20>(v_add(v_add(v_add(v_add(v_shl<2>(v_src20), v_shl<1>(v_src20)), v_shl<2>(v_add(v_src10, v_src30))), v_src00), v_src40), + v_add(v_add(v_add(v_add(v_shl<2>(v_src21), v_shl<1>(v_src21)), v_shl<2>(v_add(v_src11, v_src31))), v_src01), v_src41)), + v_rshr_pack<20>(v_add(v_add(v_add(v_add(v_shl<2>(v_src22), v_shl<1>(v_src22)), v_shl<2>(v_add(v_src12, v_src32))), v_src02), v_src42), + v_add(v_add(v_add(v_add(v_shl<2>(v_src23), v_shl<1>(v_src23)), v_shl<2>(v_add(v_src13, v_src33))), v_src03), v_src43)))); } #endif for (; i < len; i++) @@ -1647,10 +1638,10 @@ template <> void vlineSmooth(const ufixedpoint16* const * src, const ufixedpoint16* m, int n, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); v_int32 v_128_4 = vx_setall_s32(128 << 16); - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); if (len >= VECSZ) { ufixedpoint16 msum = m[0] + m[1]; @@ -1705,17 +1696,17 @@ void vlineSmooth(const ufixedpoint16* const * src, const v_src12 = vx_load(srcj1 + 2*VECSZ); v_src13 = vx_load(srcj1 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src10, v_128), v_tmp0, v_tmp1); - v_res0 += v_dotprod(v_tmp0, v_mul); - v_res1 += v_dotprod(v_tmp1, v_mul); + v_res0 = v_add(v_res0, v_dotprod(v_tmp0, v_mul)); + v_res1 = v_add(v_res1, v_dotprod(v_tmp1, v_mul)); v_zip(v_add_wrap(v_src01, v_128), v_add_wrap(v_src11, v_128), v_tmp0, v_tmp1); - v_res2 += v_dotprod(v_tmp0, v_mul); - v_res3 += v_dotprod(v_tmp1, v_mul); + v_res2 = v_add(v_res2, v_dotprod(v_tmp0, v_mul)); + v_res3 = v_add(v_res3, v_dotprod(v_tmp1, v_mul)); v_zip(v_add_wrap(v_src02, v_128), v_add_wrap(v_src12, v_128), v_tmp0, v_tmp1); - v_res4 += v_dotprod(v_tmp0, v_mul); - v_res5 += v_dotprod(v_tmp1, v_mul); + v_res4 = v_add(v_res4, v_dotprod(v_tmp0, v_mul)); + v_res5 = v_add(v_res5, v_dotprod(v_tmp1, v_mul)); v_zip(v_add_wrap(v_src03, v_128), v_add_wrap(v_src13, v_128), v_tmp0, v_tmp1); - v_res6 += v_dotprod(v_tmp0, v_mul); - v_res7 += v_dotprod(v_tmp1, v_mul); + v_res6 = v_add(v_res6, v_dotprod(v_tmp0, v_mul)); + v_res7 = v_add(v_res7, v_dotprod(v_tmp1, v_mul)); } if(j < n) { @@ -1727,26 +1718,26 @@ void vlineSmooth(const ufixedpoint16* const * src, const v_src02 = vx_load(srcj + 2*VECSZ); v_src03 = vx_load(srcj + 3*VECSZ); v_mul_expand(v_add_wrap(v_src00, v_128), v_mul, v_resj0, v_resj1); - v_res0 += v_resj0; - v_res1 += v_resj1; + v_res0 = v_add(v_res0, v_resj0); + v_res1 = v_add(v_res1, v_resj1); v_mul_expand(v_add_wrap(v_src01, v_128), v_mul, v_resj0, v_resj1); - v_res2 += v_resj0; - v_res3 += v_resj1; + v_res2 = v_add(v_res2, v_resj0); + v_res3 = v_add(v_res3, v_resj1); v_mul_expand(v_add_wrap(v_src02, v_128), v_mul, v_resj0, v_resj1); - v_res4 += v_resj0; - v_res5 += v_resj1; + v_res4 = v_add(v_res4, v_resj0); + v_res5 = v_add(v_res5, v_resj1); v_mul_expand(v_add_wrap(v_src03, v_128), v_mul, v_resj0, v_resj1); - v_res6 += v_resj0; - v_res7 += v_resj1; + v_res6 = v_add(v_res6, v_resj0); + v_res7 = v_add(v_res7, v_resj1); } - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 = v_add(v_res0, v_128_4); + v_res1 = v_add(v_res1, v_128_4); + v_res2 = v_add(v_res2, v_128_4); + v_res3 = v_add(v_res3, v_128_4); + v_res4 = v_add(v_res4, v_128_4); + v_res5 = v_add(v_res5, v_128_4); + v_res6 = v_add(v_res6, v_128_4); + v_res7 = v_add(v_res7, v_128_4); v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); @@ -1780,11 +1771,11 @@ template <> void vlineSmoothONa_yzy_a(const ufixedpoint16* const * src, const ufixedpoint16* m, int n, uint8_t* dst, int len) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int pre_shift = n / 2; - static const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); + const v_int16 v_128 = v_reinterpret_as_s16(vx_setall_u16((uint16_t)1 << 15)); v_int32 v_128_4 = vx_setall_s32(128 << 16); - const int VECSZ = v_uint16::nlanes; + const int VECSZ = VTraits::vlanes(); if (len >= VECSZ) { ufixedpoint16 msum = m[0] + m[pre_shift] + m[n - 1]; @@ -1826,27 +1817,27 @@ void vlineSmoothONa_yzy_a(const ufixedpoint16* const * s v_src21 = vx_load(srcj1 + 2*VECSZ); v_src31 = vx_load(srcj1 + 3*VECSZ); v_zip(v_add_wrap(v_src00, v_128), v_add_wrap(v_src01, v_128), v_tmp0, v_tmp1); - v_res0 += v_dotprod(v_tmp0, v_mul); - v_res1 += v_dotprod(v_tmp1, v_mul); + v_res0 = v_add(v_res0, v_dotprod(v_tmp0, v_mul)); + v_res1 = v_add(v_res1, v_dotprod(v_tmp1, v_mul)); v_zip(v_add_wrap(v_src10, v_128), v_add_wrap(v_src11, v_128), v_tmp2, v_tmp3); - v_res2 += v_dotprod(v_tmp2, v_mul); - v_res3 += v_dotprod(v_tmp3, v_mul); + v_res2 = v_add(v_res2, v_dotprod(v_tmp2, v_mul)); + v_res3 = v_add(v_res3, v_dotprod(v_tmp3, v_mul)); v_zip(v_add_wrap(v_src20, v_128), v_add_wrap(v_src21, v_128), v_tmp4, v_tmp5); - v_res4 += v_dotprod(v_tmp4, v_mul); - v_res5 += v_dotprod(v_tmp5, v_mul); + v_res4 = v_add(v_res4, v_dotprod(v_tmp4, v_mul)); + v_res5 = v_add(v_res5, v_dotprod(v_tmp5, v_mul)); v_zip(v_add_wrap(v_src30, v_128), v_add_wrap(v_src31, v_128), v_tmp6, v_tmp7); - v_res6 += v_dotprod(v_tmp6, v_mul); - v_res7 += v_dotprod(v_tmp7, v_mul); + v_res6 = v_add(v_res6, v_dotprod(v_tmp6, v_mul)); + v_res7 = v_add(v_res7, v_dotprod(v_tmp7, v_mul)); } - v_res0 += v_128_4; - v_res1 += v_128_4; - v_res2 += v_128_4; - v_res3 += v_128_4; - v_res4 += v_128_4; - v_res5 += v_128_4; - v_res6 += v_128_4; - v_res7 += v_128_4; + v_res0 = v_add(v_res0, v_128_4); + v_res1 = v_add(v_res1, v_128_4); + v_res2 = v_add(v_res2, v_128_4); + v_res3 = v_add(v_res3, v_128_4); + v_res4 = v_add(v_res4, v_128_4); + v_res5 = v_add(v_res5, v_128_4); + v_res6 = v_add(v_res6, v_128_4); + v_res7 = v_add(v_res7, v_128_4); v_store(dst + i , v_pack(v_reinterpret_as_u16(v_rshr_pack<16>(v_res0, v_res1)), v_reinterpret_as_u16(v_rshr_pack<16>(v_res2, v_res3)))); @@ -1868,9 +1859,9 @@ template <> void vlineSmoothONa_yzy_a(const ufixedpoint32* const * src, const ufixedpoint32* m, int n, uint16_t* dst, int len) { int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int pre_shift = n / 2; - const int VECSZ = v_uint32::nlanes; + const int VECSZ = VTraits::vlanes(); for (; i <= len - 2*VECSZ; i += 2*VECSZ) { v_uint32 v_src00, v_src10, v_src01, v_src11; @@ -1895,15 +1886,15 @@ void vlineSmoothONa_yzy_a(const ufixedpoint32* const * v_src01 = vx_load(srcj1); v_mul_expand(v_src00, v_mul, v_tmp0, v_tmp1); v_mul_expand(v_src01, v_mul, v_tmp2, v_tmp3); - v_res0 += v_tmp0 + v_tmp2; - v_res1 += v_tmp1 + v_tmp3; + v_res0 = v_add(v_res0, v_add(v_tmp0, v_tmp2)); + v_res1 = v_add(v_res1, v_add(v_tmp1, v_tmp3)); v_src10 = vx_load(srcj0 + VECSZ); v_src11 = vx_load(srcj1 + VECSZ); v_mul_expand(v_src10, v_mul, v_tmp4, v_tmp5); v_mul_expand(v_src11, v_mul, v_tmp6, v_tmp7); - v_res2 += v_tmp4 + v_tmp6; - v_res3 += v_tmp5 + v_tmp7; + v_res2 = v_add(v_res2, v_add(v_tmp4, v_tmp6)); + v_res3 = v_add(v_res3, v_add(v_tmp5, v_tmp7)); } v_store(dst + i, v_pack(v_rshr_pack<32>(v_res0, v_res1), diff --git a/modules/imgproc/src/spatialgradient.cpp b/modules/imgproc/src/spatialgradient.cpp index 1aed1fa031..f422609c40 100644 --- a/modules/imgproc/src/spatialgradient.cpp +++ b/modules/imgproc/src/spatialgradient.cpp @@ -57,6 +57,25 @@ namespace cv * 0 0 0 * 1 2 1 */ +#if (CV_SIMD || CV_SIMD_SCALABLE) +template +static inline void spatialGradientKernel_vec( T& vx, T& vy, + const T& v00, const T& v01, const T& v02, + const T& v10, const T& v12, + const T& v20, const T& v21, const T& v22 ) +{ + // vx = (v22 - v00) + (v02 - v20) + 2 * (v12 - v10) + // vy = (v22 - v00) + (v20 - v02) + 2 * (v21 - v01) + T tmp_add = v_sub(v22, v00), + tmp_sub = v_sub(v02, v20), + tmp_x = v_sub(v12, v10), + tmp_y = v_sub(v21, v01); + + vx = v_add(v_add(v_add(tmp_add, tmp_sub), tmp_x), tmp_x); + vy = v_add(v_add(v_sub(tmp_add, tmp_sub), tmp_y), tmp_y); +} +#endif + template static inline void spatialGradientKernel( T& vx, T& vy, const T& v00, const T& v01, const T& v02, @@ -65,7 +84,6 @@ static inline void spatialGradientKernel( T& vx, T& vy, { // vx = (v22 - v00) + (v02 - v20) + 2 * (v12 - v10) // vy = (v22 - v00) + (v20 - v02) + 2 * (v21 - v01) - T tmp_add = v22 - v00, tmp_sub = v02 - v20, tmp_x = v12 - v10, @@ -125,7 +143,7 @@ void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, int i_start = 0; int j_start = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) // Characters in variable names have the following meanings: // u: unsigned char // s: signed int @@ -148,7 +166,7 @@ void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, short *n_dy = dy.ptr(i+1); // Process rest of columns 16-column chunks at a time - for ( j = 1; j < W - v_uint8::nlanes; j += v_uint8::nlanes) + for ( j = 1; j < W - VTraits::vlanes(); j += VTraits::vlanes()) { // Load top row for 3x3 Sobel filter v_uint8 v_um = vx_load(&p_src[j-1]); @@ -195,22 +213,22 @@ void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, // dx & dy for rows 1, 2, 3 v_int16 v_sdx1, v_sdy1; - spatialGradientKernel( v_sdx1, v_sdy1, + spatialGradientKernel_vec( v_sdx1, v_sdy1, v_s1m1, v_s1n1, v_s1p1, v_s2m1, v_s2p1, v_s3m1, v_s3n1, v_s3p1 ); v_int16 v_sdx2, v_sdy2; - spatialGradientKernel( v_sdx2, v_sdy2, + spatialGradientKernel_vec( v_sdx2, v_sdy2, v_s1m2, v_s1n2, v_s1p2, v_s2m2, v_s2p2, v_s3m2, v_s3n2, v_s3p2 ); // Store v_store(&c_dx[j], v_sdx1); - v_store(&c_dx[j+v_int16::nlanes], v_sdx2); + v_store(&c_dx[j+VTraits::vlanes()], v_sdx2); v_store(&c_dy[j], v_sdy1); - v_store(&c_dy[j+v_int16::nlanes], v_sdy2); + v_store(&c_dy[j+VTraits::vlanes()], v_sdy2); // Load fourth row for 3x3 Sobel filter v_um = vx_load(&m_src[j-1]); @@ -227,21 +245,21 @@ void spatialGradient( InputArray _src, OutputArray _dx, OutputArray _dy, v_int16 v_s4p2 = v_reinterpret_as_s16(v_up2); // dx & dy for rows 2, 3, 4 - spatialGradientKernel( v_sdx1, v_sdy1, + spatialGradientKernel_vec( v_sdx1, v_sdy1, v_s2m1, v_s2n1, v_s2p1, v_s3m1, v_s3p1, v_s4m1, v_s4n1, v_s4p1 ); - spatialGradientKernel( v_sdx2, v_sdy2, + spatialGradientKernel_vec( v_sdx2, v_sdy2, v_s2m2, v_s2n2, v_s2p2, v_s3m2, v_s3p2, v_s4m2, v_s4n2, v_s4p2 ); // Store v_store(&n_dx[j], v_sdx1); - v_store(&n_dx[j+v_int16::nlanes], v_sdx2); + v_store(&n_dx[j+VTraits::vlanes()], v_sdx2); v_store(&n_dy[j], v_sdy1); - v_store(&n_dy[j+v_int16::nlanes], v_sdy2); + v_store(&n_dy[j+VTraits::vlanes()], v_sdy2); } } i_start = i; diff --git a/modules/imgproc/src/stackblur.cpp b/modules/imgproc/src/stackblur.cpp index 5d60a1d365..6becbe5c41 100644 --- a/modules/imgproc/src/stackblur.cpp +++ b/modules/imgproc/src/stackblur.cpp @@ -88,7 +88,7 @@ static unsigned char const stackblurShr[255] = namespace cv{ -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template inline int opRow(const T* , T* , const std::vector& , const float , const int radius, const int CN, const int ) { @@ -107,7 +107,7 @@ inline int opRow(const uchar* srcPtr, uchar* dstPtr, const std::vector::vlanes(); if (kernelSize == 3) { @@ -126,10 +126,10 @@ inline int opRow(const uchar* srcPtr, uchar* dstPtr, const std::vector>shrValTab; - y01 = (y01 * v_mulVal)>>shrValTab; - y10 = (y10 * v_mulVal)>>shrValTab; - y11 = (y11 * v_mulVal)>>shrValTab; + y00 = v_shr(v_mul(y00, v_mulVal), shrValTab); + y01 = v_shr(v_mul(y01, v_mulVal), shrValTab); + y10 = v_shr(v_mul(y10, v_mulVal), shrValTab); + y11 = v_shr(v_mul(y11, v_mulVal), shrValTab); v_store(dstPtr + i, v_pack(v_pack(y00, y01), v_pack(y10, y11))); } @@ -159,12 +159,12 @@ inline int opRow(const uchar* srcPtr, uchar* dstPtr, const std::vector(const uchar* srcPtr, uchar* dstPtr, const std::vector>shrValTab; - s1 = (s1 * v_mulVal)>>shrValTab; - s2 = (s2 * v_mulVal)>>shrValTab; - s3 = (s3 * v_mulVal)>>shrValTab; + s0 = v_shr(v_mul(s0, v_mulVal), shrValTab); + s1 = v_shr(v_mul(s1, v_mulVal), shrValTab); + s2 = v_shr(v_mul(s2, v_mulVal), shrValTab); + s3 = v_shr(v_mul(s3, v_mulVal), shrValTab); v_store(dstPtr + i, v_pack(v_reinterpret_as_u16(v_pack(s0, s1)), v_reinterpret_as_u16(v_pack(s2, s3)))); } @@ -205,7 +205,7 @@ inline int opRow(const ushort* srcPtr, ushort* dstPtr, const std::vector const int mulValTab= stackblurMul[radius]; const int shrValTab= stackblurShr[radius]; - const int VEC_LINE = v_uint16::nlanes; + const int VEC_LINE = VTraits::vlanes(); v_uint32 v_mulVal = vx_setall_u32(mulValTab); if (kernelSize == 3) @@ -220,7 +220,7 @@ inline int opRow(const ushort* srcPtr, ushort* dstPtr, const std::vector x1l = v_add(v_add(x1l, x1l), v_add(x0l, x2l)); x1h = v_add(v_add(x1h, x1h), v_add(x0h, x2h)); - v_store(dstPtr + i, v_pack((x1l * v_mulVal)>>shrValTab, (x1h * v_mulVal)>>shrValTab)); + v_store(dstPtr + i, v_pack(v_shr(v_mul(x1l, v_mulVal), shrValTab), v_shr(v_mul(x1h, v_mulVal), shrValTab))); } } else @@ -243,25 +243,25 @@ inline int opRow(const ushort* srcPtr, ushort* dstPtr, const std::vector v_uint16 k2 = vx_setall_u16(kx[k + 1]); v_uint32 y0, y1; - v_mul_expand(vx_load(srcPtr - j) + vx_load(srcPtr + j), k1, y0, y1); - s0 += y0; - s1 += y1; - v_mul_expand(vx_load(srcPtr - j - CN) + vx_load(srcPtr + j + CN), k2, y0, y1); - s0 += y0; - s1 += y1; + v_mul_expand(v_add(vx_load(srcPtr - j), vx_load(srcPtr + j)), k1, y0, y1); + s0 = v_add(s0, y0); + s1 = v_add(s1, y1); + v_mul_expand(v_add(vx_load(srcPtr - j - CN), vx_load(srcPtr + j + CN)), k2, y0, y1); + s0 = v_add(s0, y0); + s1 = v_add(s1, y1); } if( k < kernelSize / 2 + 1 ) { v_uint16 k1 = vx_setall_u16(kx[k]); v_uint32 y0, y1; - v_mul_expand(vx_load(srcPtr - j) + vx_load(srcPtr + j), k1, y0, y1); - s0 += y0; - s1 += y1; + v_mul_expand(v_add(vx_load(srcPtr - j), vx_load(srcPtr + j)), k1, y0, y1); + s0 = v_add(s0, y0); + s1 = v_add(s1, y1); } - s0 = (s0 * v_mulVal)>>shrValTab; - s1 = (s1 * v_mulVal)>>shrValTab; + s0 = v_shr(v_mul(s0, v_mulVal), shrValTab); + s1 = v_shr(v_mul(s1, v_mulVal), shrValTab); v_store(dstPtr + i, v_pack(s0, s1)); } @@ -282,7 +282,7 @@ inline int opRow(const short* srcPtr, short* dstPtr, const std::vector::vlanes(); v_int32 v_mulVal = vx_setall_s32(mulValTab); if (kernelSize == 3) @@ -297,7 +297,7 @@ inline int opRow(const short* srcPtr, short* dstPtr, const std::vector>shrValTab, (x1h * v_mulVal)>>shrValTab)); + v_store(dstPtr + i, v_pack(v_shr(v_mul(x1l, v_mulVal), shrValTab), v_shr(v_mul(x1h, v_mulVal), shrValTab))); } } else @@ -320,24 +320,24 @@ inline int opRow(const short* srcPtr, short* dstPtr, const std::vector>shrValTab; - s1 = (s1 * v_mulVal)>>shrValTab; + s0 = v_shr(v_mul(s0, v_mulVal), shrValTab); + s1 = v_shr(v_mul(s1, v_mulVal), shrValTab); v_store(dstPtr + i, v_pack(s0, s1)); } @@ -352,7 +352,7 @@ inline int opRow(const float* srcPtr, float* dstPtr, const std::vector::vlanes(); const int VEC_LINE4 = VEC_LINE * 4; if (kernelSize == 3) @@ -364,22 +364,22 @@ inline int opRow(const float* srcPtr, float* dstPtr, const std::vector(const float* srcPtr, float* dstPtr, const std::vector(const float* srcPtr, float* dstPtr, const std::vector inline int opComputeDiff(const uchar*& srcPtr, int*& diff0, const int w, const int CNR1) { int index = 0; - const int VEC_LINE_8 = v_uint8::nlanes; - const int VEC_LINE_32 = v_int32::nlanes; + const int VEC_LINE_8 = VTraits::vlanes(); + const int VEC_LINE_32 = VTraits::vlanes(); for (; index <= w - VEC_LINE_8; index += VEC_LINE_8, diff0+=VEC_LINE_8, srcPtr+=VEC_LINE_8) { v_uint16 x0l, x0h, x1l, x1h; @@ -435,8 +435,8 @@ inline int opComputeDiff(const uchar*& srcPtr, int*& diff0, const in v_expand(vx_load(srcPtr), x1l, x1h); v_int32 y0, y1, y2, y3; - v_expand(v_reinterpret_as_s16(x0l) - v_reinterpret_as_s16(x1l), y0, y1); - v_expand(v_reinterpret_as_s16(x0h) - v_reinterpret_as_s16(x1h), y2, y3); + v_expand(v_sub(v_reinterpret_as_s16(x0l), v_reinterpret_as_s16(x1l)), y0, y1); + v_expand(v_sub(v_reinterpret_as_s16(x0h), v_reinterpret_as_s16(x1h)), y2, y3); v_store(diff0, y0); v_store(diff0 + VEC_LINE_32, y1); @@ -517,7 +517,7 @@ public: // middle int wc = radius * CN; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) wc = opRow(srcPtr, dstPtr, kVec, mulVal, radius, CN, widthCN); #endif for (; wc < widthCN; wc++) @@ -586,7 +586,7 @@ public: // middle auto diff0 = diff + radius * CN; int index = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) index = opComputeDiff(srcPtr, diff0, widthCN, CNR1); #endif @@ -688,7 +688,7 @@ private: float mulVal; }; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template inline int opColumn(const T* , T* , T* , TBuf* , TBuf* , TBuf* , const float , const int , const int , const int , const int , const int ) @@ -703,7 +703,7 @@ inline int opColumn(const float* srcPtr, float* dstPtr, float* sta { int k = 0; v_float32 v_mulVal = vx_setall_f32(mulVal); - const int VEC_LINE = v_float32::nlanes; + const int VEC_LINE = VTraits::vlanes(); const int VEC_LINE4 = 4 * VEC_LINE; auto stackStartPtr = stack + ss * widthLen; @@ -726,20 +726,20 @@ inline int opColumn(const float* srcPtr, float* dstPtr, float* sta v_float32 v_sumIn2 = vx_load(sumIn + VEC_LINE * 2 + k); v_float32 v_sumIn3 = vx_load(sumIn + VEC_LINE * 3+ k); - v_store(dstPtr + k, v_sum0 * v_mulVal); - v_store(dstPtr + VEC_LINE + k, v_sum1 * v_mulVal); - v_store(dstPtr + VEC_LINE * 2 + k, v_sum2 * v_mulVal); - v_store(dstPtr + VEC_LINE * 3 + k, v_sum3 * v_mulVal); + v_store(dstPtr + k, v_mul(v_sum0, v_mulVal)); + v_store(dstPtr + VEC_LINE + k, v_mul(v_sum1, v_mulVal)); + v_store(dstPtr + VEC_LINE * 2 + k, v_mul(v_sum2, v_mulVal)); + v_store(dstPtr + VEC_LINE * 3 + k, v_mul(v_sum3, v_mulVal)); - v_sum0 -= v_sumOut0; - v_sum1 -= v_sumOut1; - v_sum2 -= v_sumOut2; - v_sum3 -= v_sumOut3; + v_sum0 = v_sub(v_sum0, v_sumOut0); + v_sum1 = v_sub(v_sum1, v_sumOut1); + v_sum2 = v_sub(v_sum2, v_sumOut2); + v_sum3 = v_sub(v_sum3, v_sumOut3); - v_sumOut0 -= vx_load(stackStartPtr + k); - v_sumOut1 -= vx_load(stackStartPtr + VEC_LINE + k); - v_sumOut2 -= vx_load(stackStartPtr + VEC_LINE * 2 + k); - v_sumOut3 -= vx_load(stackStartPtr + VEC_LINE * 3 + k); + v_sumOut0 = v_sub(v_sumOut0, vx_load(stackStartPtr + k)); + v_sumOut1 = v_sub(v_sumOut1, vx_load(stackStartPtr + VEC_LINE + k)); + v_sumOut2 = v_sub(v_sumOut2, vx_load(stackStartPtr + VEC_LINE * 2 + k)); + v_sumOut3 = v_sub(v_sumOut3, vx_load(stackStartPtr + VEC_LINE * 3 + k)); v_float32 v_srcPtr0 = vx_load(srcPtr + k); v_float32 v_srcPtr1 = vx_load(srcPtr + VEC_LINE + k); @@ -751,35 +751,35 @@ inline int opColumn(const float* srcPtr, float* dstPtr, float* sta v_store(stackStartPtr + VEC_LINE * 2 + k, v_srcPtr2); v_store(stackStartPtr + VEC_LINE * 3 + k, v_srcPtr3); - v_sumIn0 += v_srcPtr0; - v_sumIn1 += v_srcPtr1; - v_sumIn2 += v_srcPtr2; - v_sumIn3 += v_srcPtr3; + v_sumIn0 = v_add(v_sumIn0, v_srcPtr0); + v_sumIn1 = v_add(v_sumIn1, v_srcPtr1); + v_sumIn2 = v_add(v_sumIn2, v_srcPtr2); + v_sumIn3 = v_add(v_sumIn3, v_srcPtr3); - v_store(sum + k, v_sum0 + v_sumIn0); - v_store(sum + VEC_LINE + k, v_sum1 + v_sumIn1); - v_store(sum + VEC_LINE * 2 + k, v_sum2 + v_sumIn2); - v_store(sum + VEC_LINE * 3 + k, v_sum3 + v_sumIn3); + v_store(sum + k, v_add(v_sum0, v_sumIn0)); + v_store(sum + VEC_LINE + k, v_add(v_sum1, v_sumIn1)); + v_store(sum + VEC_LINE * 2 + k, v_add(v_sum2, v_sumIn2)); + v_store(sum + VEC_LINE * 3 + k, v_add(v_sum3, v_sumIn3)); v_srcPtr0 = vx_load(stackSp1Ptr + k); v_srcPtr1 = vx_load(stackSp1Ptr + VEC_LINE + k); v_srcPtr2 = vx_load(stackSp1Ptr + VEC_LINE * 2 + k); v_srcPtr3 = vx_load(stackSp1Ptr + VEC_LINE * 3 + k); - v_sumOut0 += v_srcPtr0; - v_sumOut1 += v_srcPtr1; - v_sumOut2 += v_srcPtr2; - v_sumOut3 += v_srcPtr3; + v_sumOut0 = v_add(v_sumOut0, v_srcPtr0); + v_sumOut1 = v_add(v_sumOut1, v_srcPtr1); + v_sumOut2 = v_add(v_sumOut2, v_srcPtr2); + v_sumOut3 = v_add(v_sumOut3, v_srcPtr3); v_store(sumOut + k, v_sumOut0); v_store(sumOut + VEC_LINE + k, v_sumOut1); v_store(sumOut + VEC_LINE * 2 + k, v_sumOut2); v_store(sumOut + VEC_LINE * 3 + k, v_sumOut3); - v_sumIn0 -= v_srcPtr0; - v_sumIn1 -= v_srcPtr1; - v_sumIn2 -= v_srcPtr2; - v_sumIn3 -= v_srcPtr3; + v_sumIn0 = v_sub(v_sumIn0, v_srcPtr0); + v_sumIn1 = v_sub(v_sumIn1, v_srcPtr1); + v_sumIn2 = v_sub(v_sumIn2, v_srcPtr2); + v_sumIn3 = v_sub(v_sumIn3, v_srcPtr3); v_store(sumIn + k, v_sumIn0); v_store(sumIn + VEC_LINE + k, v_sumIn1); @@ -793,20 +793,20 @@ inline int opColumn(const float* srcPtr, float* dstPtr, float* sta v_float32 v_sumOut = vx_load(sumOut + k); v_float32 v_sumIn = vx_load(sumIn + k); - v_store(dstPtr + k, v_sum * v_mulVal); - v_sum -= v_sumOut; - v_sumOut -= vx_load(stackStartPtr + k); + v_store(dstPtr + k, v_mul(v_sum, v_mulVal)); + v_sum = v_sub(v_sum, v_sumOut); + v_sumOut = v_sub(v_sumOut, vx_load(stackStartPtr + k)); v_float32 v_srcPtr = vx_load(srcPtr + k); v_store(stackStartPtr + k, v_srcPtr); - v_sumIn += v_srcPtr; - v_store(sum + k, v_sum + v_sumIn); + v_sumIn = v_add(v_sumIn, v_srcPtr); + v_store(sum + k, v_add(v_sum, v_sumIn)); v_srcPtr = vx_load(stackSp1Ptr + k); - v_sumOut += v_srcPtr; + v_sumOut = v_add(v_sumOut, v_srcPtr); v_store(sumOut + k, v_sumOut); - v_sumIn -= v_srcPtr; + v_sumIn = v_sub(v_sumIn, v_srcPtr); v_store(sumIn + k, v_sumIn); } return k; @@ -820,8 +820,8 @@ inline int opColumn(const uchar* srcPtr, uchar* dstPtr, uchar* stack int k = 0; if (mulValTab != 0 && shrValTab != 0) { - const int VEC_LINE_8 = v_uint8::nlanes; - const int VEC_LINE_32 = v_int32::nlanes; + const int VEC_LINE_8 = VTraits::vlanes(); + const int VEC_LINE_32 = VTraits::vlanes(); v_int32 v_mulVal = vx_setall_s32(mulValTab); auto stackStartPtr = stack + ss * widthLen; @@ -850,13 +850,13 @@ inline int opColumn(const uchar* srcPtr, uchar* dstPtr, uchar* stack v_store(dstPtr + k, v_pack( - v_reinterpret_as_u16(v_pack((v_sum0 * v_mulVal)>>shrValTab, (v_sum1 * v_mulVal)>>shrValTab)), - v_reinterpret_as_u16(v_pack((v_sum2 * v_mulVal)>>shrValTab, (v_sum3 * v_mulVal)>>shrValTab)))); + v_reinterpret_as_u16(v_pack(v_shr(v_mul(v_sum0, v_mulVal), shrValTab), v_shr(v_mul(v_sum1, v_mulVal), shrValTab))), + v_reinterpret_as_u16(v_pack(v_shr(v_mul(v_sum2, v_mulVal), shrValTab), v_shr(v_mul(v_sum3, v_mulVal), shrValTab))))); - v_sum0 -= v_sumOut0; - v_sum1 -= v_sumOut1; - v_sum2 -= v_sumOut2; - v_sum3 -= v_sumOut3; + v_sum0 = v_sub(v_sum0, v_sumOut0); + v_sum1 = v_sub(v_sum1, v_sumOut1); + v_sum2 = v_sub(v_sum2, v_sumOut2); + v_sum3 = v_sub(v_sum3, v_sumOut3); v_uint16 x0l, x0h; v_int32 v_ss0, v_ss1, v_ss2, v_ss3; @@ -865,10 +865,10 @@ inline int opColumn(const uchar* srcPtr, uchar* dstPtr, uchar* stack v_expand(v_reinterpret_as_s16(x0l), v_ss0, v_ss1); v_expand(v_reinterpret_as_s16(x0h), v_ss2, v_ss3); - v_sumOut0 -= v_ss0; - v_sumOut1 -= v_ss1; - v_sumOut2 -= v_ss2; - v_sumOut3 -= v_ss3; + v_sumOut0 = v_sub(v_sumOut0, v_ss0); + v_sumOut1 = v_sub(v_sumOut1, v_ss1); + v_sumOut2 = v_sub(v_sumOut2, v_ss2); + v_sumOut3 = v_sub(v_sumOut3, v_ss3); v_expand(vx_load(srcPtr + k), x0l, x0h); v_expand(v_reinterpret_as_s16(x0l), v_ss0, v_ss1); @@ -876,34 +876,34 @@ inline int opColumn(const uchar* srcPtr, uchar* dstPtr, uchar* stack memcpy(stackStartPtr + k,srcPtr + k, VEC_LINE_8 * sizeof (uchar)); - v_sumIn0 += v_ss0; - v_sumIn1 += v_ss1; - v_sumIn2 += v_ss2; - v_sumIn3 += v_ss3; + v_sumIn0 = v_add(v_sumIn0, v_ss0); + v_sumIn1 = v_add(v_sumIn1, v_ss1); + v_sumIn2 = v_add(v_sumIn2, v_ss2); + v_sumIn3 = v_add(v_sumIn3, v_ss3); - v_store(sum + k, v_sum0 + v_sumIn0); - v_store(sum + VEC_LINE_32 + k, v_sum1 + v_sumIn1); - v_store(sum + VEC_LINE_32 * 2 + k, v_sum2 + v_sumIn2); - v_store(sum + VEC_LINE_32 * 3 + k, v_sum3 + v_sumIn3); + v_store(sum + k, v_add(v_sum0, v_sumIn0)); + v_store(sum + VEC_LINE_32 + k, v_add(v_sum1, v_sumIn1)); + v_store(sum + VEC_LINE_32 * 2 + k, v_add(v_sum2, v_sumIn2)); + v_store(sum + VEC_LINE_32 * 3 + k, v_add(v_sum3, v_sumIn3)); v_expand(vx_load(stackSp1Ptr + k), x0l, x0h); v_expand(v_reinterpret_as_s16(x0l), v_ss0, v_ss1); v_expand(v_reinterpret_as_s16(x0h), v_ss2, v_ss3); - v_sumOut0 += v_ss0; - v_sumOut1 += v_ss1; - v_sumOut2 += v_ss2; - v_sumOut3 += v_ss3; + v_sumOut0 = v_add(v_sumOut0, v_ss0); + v_sumOut1 = v_add(v_sumOut1, v_ss1); + v_sumOut2 = v_add(v_sumOut2, v_ss2); + v_sumOut3 = v_add(v_sumOut3, v_ss3); v_store(sumOut + k, v_sumOut0); v_store(sumOut + VEC_LINE_32 + k, v_sumOut1); v_store(sumOut + VEC_LINE_32 * 2 + k, v_sumOut2); v_store(sumOut + VEC_LINE_32 * 3 + k, v_sumOut3); - v_sumIn0 -= v_ss0; - v_sumIn1 -= v_ss1; - v_sumIn2 -= v_ss2; - v_sumIn3 -= v_ss3; + v_sumIn0 = v_sub(v_sumIn0, v_ss0); + v_sumIn1 = v_sub(v_sumIn1, v_ss1); + v_sumIn2 = v_sub(v_sumIn2, v_ss2); + v_sumIn3 = v_sub(v_sumIn3, v_ss3); v_store(sumIn + k, v_sumIn0); v_store(sumIn + VEC_LINE_32 + k, v_sumIn1); @@ -922,8 +922,8 @@ inline int opColumn(const short* srcPtr, short* dstPtr, short* stack int k = 0; if (mulValTab != 0 && shrValTab != 0) { - const int VEC_LINE_16 = v_int16::nlanes; - const int VEC_LINE_32 = v_int32::nlanes; + const int VEC_LINE_16 = VTraits::vlanes(); + const int VEC_LINE_32 = VTraits::vlanes(); v_int32 v_mulVal = vx_setall_s32(mulValTab); auto stackStartPtr = stack + ss * widthLen; @@ -943,39 +943,39 @@ inline int opColumn(const short* srcPtr, short* dstPtr, short* stack v_sumOut0 = vx_load(sumOut + k); v_sumOut1 = vx_load(sumOut + k + VEC_LINE_32); - v_store(dstPtr + k,v_pack((v_sum0 * v_mulVal)>>shrValTab, (v_sum1 * v_mulVal)>>shrValTab)); + v_store(dstPtr + k,v_pack(v_shr(v_mul(v_sum0, v_mulVal), shrValTab), v_shr(v_mul(v_sum1, v_mulVal), shrValTab))); - v_sum0 -= v_sumOut0; - v_sum1 -= v_sumOut1; + v_sum0 = v_sub(v_sum0, v_sumOut0); + v_sum1 = v_sub(v_sum1, v_sumOut1); v_int32 v_ss0, v_ss1; v_expand(vx_load(stackStartPtr + k), v_ss0, v_ss1); - v_sumOut0 -= v_ss0; - v_sumOut1 -= v_ss1; + v_sumOut0 = v_sub(v_sumOut0, v_ss0); + v_sumOut1 = v_sub(v_sumOut1, v_ss1); v_expand(vx_load(srcPtr + k), v_ss0, v_ss1); memcpy(stackStartPtr + k,srcPtr + k, VEC_LINE_16 * sizeof (short)); - v_sumIn0 += v_ss0; - v_sumIn1 += v_ss1; + v_sumIn0 = v_add(v_sumIn0, v_ss0); + v_sumIn1 = v_add(v_sumIn1, v_ss1); - v_sum0 += v_sumIn0; - v_sum1 += v_sumIn1; + v_sum0 = v_add(v_sum0, v_sumIn0); + v_sum1 = v_add(v_sum1, v_sumIn1); v_store(sum + k, v_sum0); v_store(sum + VEC_LINE_32 + k, v_sum1); v_expand(vx_load(stackSp1Ptr + k), v_ss0, v_ss1); - v_sumOut0 += v_ss0; - v_sumOut1 += v_ss1; + v_sumOut0 = v_add(v_sumOut0, v_ss0); + v_sumOut1 = v_add(v_sumOut1, v_ss1); v_store(sumOut + k, v_sumOut0); v_store(sumOut + VEC_LINE_32 + k, v_sumOut1); - v_sumIn0 -= v_ss0; - v_sumIn1 -= v_ss1; + v_sumIn0 = v_sub(v_sumIn0, v_ss0); + v_sumIn1 = v_sub(v_sumIn1, v_ss1); v_store(sumIn + k, v_sumIn0); v_store(sumIn + VEC_LINE_32 + k, v_sumIn1); @@ -992,8 +992,8 @@ inline int opColumn(const ushort* srcPtr, ushort* dstPtr, ushort* s int k = 0; if (mulValTab != 0 && shrValTab != 0) { - const int VEC_LINE_16 = v_uint16::nlanes; - const int VEC_LINE_32 = v_int32::nlanes; + const int VEC_LINE_16 = VTraits::vlanes(); + const int VEC_LINE_32 = VTraits::vlanes(); v_uint32 v_mulVal = vx_setall_u32((uint32_t)mulValTab); auto stackStartPtr = stack + ss * widthLen; @@ -1013,40 +1013,40 @@ inline int opColumn(const ushort* srcPtr, ushort* dstPtr, ushort* s v_sumOut0 = vx_load(sumOut + k); v_sumOut1 = vx_load(sumOut + k + VEC_LINE_32); - v_store(dstPtr + k, v_pack((v_reinterpret_as_u32(v_sum0) * v_mulVal)>>shrValTab, (v_reinterpret_as_u32(v_sum1) * v_mulVal)>>shrValTab)); + v_store(dstPtr + k, v_pack(v_shr(v_mul(v_reinterpret_as_u32(v_sum0), v_mulVal), shrValTab), v_shr(v_mul(v_reinterpret_as_u32(v_sum1), v_mulVal), shrValTab))); - v_sum0 -= v_sumOut0; - v_sum1 -= v_sumOut1; + v_sum0 = v_sub(v_sum0, v_sumOut0); + v_sum1 = v_sub(v_sum1, v_sumOut1); v_uint32 v_ss0, v_ss1; v_expand(vx_load(stackStartPtr + k), v_ss0, v_ss1); - v_sumOut0 -= v_reinterpret_as_s32(v_ss0); - v_sumOut1 -= v_reinterpret_as_s32(v_ss1); + v_sumOut0 = v_sub(v_sumOut0, v_reinterpret_as_s32(v_ss0)); + v_sumOut1 = v_sub(v_sumOut1, v_reinterpret_as_s32(v_ss1)); v_expand(vx_load(srcPtr + k), v_ss0, v_ss1); memcpy(stackStartPtr + k,srcPtr + k, VEC_LINE_16 * sizeof (ushort)); - v_sumIn0 += v_reinterpret_as_s32(v_ss0); - v_sumIn1 += v_reinterpret_as_s32(v_ss1); + v_sumIn0 = v_add(v_sumIn0, v_reinterpret_as_s32(v_ss0)); + v_sumIn1 = v_add(v_sumIn1, v_reinterpret_as_s32(v_ss1)); - v_sum0 += v_sumIn0; - v_sum1 += v_sumIn1; + v_sum0 = v_add(v_sum0, v_sumIn0); + v_sum1 = v_add(v_sum1, v_sumIn1); v_store(sum + k, v_sum0); v_store(sum + VEC_LINE_32 + k, v_sum1); v_expand(vx_load(stackSp1Ptr + k), v_ss0, v_ss1); - v_sumOut0 += v_reinterpret_as_s32(v_ss0); - v_sumOut1 += v_reinterpret_as_s32(v_ss1); + v_sumOut0 = v_add(v_sumOut0, v_reinterpret_as_s32(v_ss0)); + v_sumOut1 = v_add(v_sumOut1, v_reinterpret_as_s32(v_ss1)); v_store(sumOut + k, v_sumOut0); v_store(sumOut + VEC_LINE_32 + k, v_sumOut1); - v_sumIn0 -= v_reinterpret_as_s32(v_ss0); - v_sumIn1 -= v_reinterpret_as_s32(v_ss1); + v_sumIn0 = v_sub(v_sumIn0, v_reinterpret_as_s32(v_ss0)); + v_sumIn1 = v_sub(v_sumIn1, v_reinterpret_as_s32(v_ss1)); v_store(sumIn + k, v_sumIn0); v_store(sumIn + VEC_LINE_32 + k, v_sumIn1); @@ -1152,7 +1152,7 @@ public: } int k = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) k = opColumn(srcPtr, dstPtr, stack, sum, sumIn, sumOut, mulVal, mulValTab, shrValTab, widthLen, stackStart, sp1); #endif diff --git a/modules/imgproc/src/thresh.cpp b/modules/imgproc/src/thresh.cpp index f411d257a9..b2a4661772 100644 --- a/modules/imgproc/src/thresh.cpp +++ b/modules/imgproc/src/thresh.cpp @@ -190,7 +190,7 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) int j = 0; const uchar* src = _src.ptr(); uchar* dst = _dst.ptr(); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_uint8 thresh_u = vx_setall_u8( thresh ); v_uint8 maxval16 = vx_setall_u8( maxval ); @@ -199,12 +199,12 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_BINARY: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = thresh_u < v0; - v0 = v0 & maxval16; + v0 = v_lt(thresh_u, v0); + v0 = v_and(v0, maxval16); v_store( dst + j, v0 ); } } @@ -213,12 +213,12 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_BINARY_INV: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = v0 <= thresh_u; - v0 = v0 & maxval16; + v0 = v_le(v0, thresh_u); + v0 = v_and(v0, maxval16); v_store( dst + j, v0 ); } } @@ -227,11 +227,11 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_TRUNC: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = v0 - ( v0 - thresh_u ); + v0 = v_sub(v0, v_sub(v0, thresh_u)); v_store( dst + j, v0 ); } } @@ -240,11 +240,11 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_TOZERO: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = ( thresh_u < v0 ) & v0; + v0 = v_and(v_lt(thresh_u, v0), v0); v_store( dst + j, v0 ); } } @@ -253,11 +253,11 @@ thresh_8u( const Mat& _src, Mat& _dst, uchar thresh, uchar maxval, int type ) case THRESH_TOZERO_INV: for( int i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { - for( j = 0; j <= roi.width - v_uint8::nlanes; j += v_uint8::nlanes) + for( j = 0; j <= roi.width - VTraits::vlanes(); j += VTraits::vlanes()) { v_uint8 v0; v0 = vx_load( src + j ); - v0 = ( v0 <= thresh_u ) & v0; + v0 = v_and(v_le(v0, thresh_u), v0); v_store( dst + j, v0 ); } } @@ -351,7 +351,7 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) const ushort* src = _src.ptr(); ushort* dst = _dst.ptr(); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int i, j; v_uint16 thresh_u = vx_setall_u16(thresh); v_uint16 maxval16 = vx_setall_u16(maxval); @@ -361,25 +361,25 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) case THRESH_BINARY: for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { - for (j = 0; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (j = 0; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); - v0 = thresh_u < v0; - v1 = thresh_u < v1; - v0 = v0 & maxval16; - v1 = v1 & maxval16; + v1 = vx_load(src + j + VTraits::vlanes()); + v0 = v_lt(thresh_u, v0); + v1 = v_lt(thresh_u, v1); + v0 = v_and(v0, maxval16); + v1 = v_and(v1, maxval16); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); - v0 = thresh_u < v0; - v0 = v0 & maxval16; + v0 = v_lt(thresh_u, v0); + v0 = v_and(v0, maxval16); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -391,25 +391,25 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; - for (; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); - v0 = v0 <= thresh_u; - v1 = v1 <= thresh_u; - v0 = v0 & maxval16; - v1 = v1 & maxval16; + v1 = vx_load(src + j + VTraits::vlanes()); + v0 = v_le(v0, thresh_u); + v1 = v_le(v1, thresh_u); + v0 = v_and(v0, maxval16); + v1 = v_and(v1, maxval16); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); - v0 = v0 <= thresh_u; - v0 = v0 & maxval16; + v0 = v_le(v0, thresh_u); + v0 = v_and(v0, maxval16); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -421,22 +421,22 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; - for (; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); + v1 = vx_load(src + j + VTraits::vlanes()); v0 = v_min(v0, thresh_u); v1 = v_min(v1, thresh_u); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); v0 = v_min(v0, thresh_u); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -448,22 +448,22 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; - for (; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); - v0 = (thresh_u < v0) & v0; - v1 = (thresh_u < v1) & v1; + v1 = vx_load(src + j + VTraits::vlanes()); + v0 = v_and(v_lt(thresh_u, v0), v0); + v1 = v_and(v_lt(thresh_u, v1), v1); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); - v0 = (thresh_u < v0) & v0; + v0 = v_and(v_lt(thresh_u, v0), v0); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -475,22 +475,22 @@ thresh_16u(const Mat& _src, Mat& _dst, ushort thresh, ushort maxval, int type) for (i = 0; i < roi.height; i++, src += src_step, dst += dst_step) { j = 0; - for (; j <= roi.width - 2*v_uint16::nlanes; j += 2*v_uint16::nlanes) + for (; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes()) { v_uint16 v0, v1; v0 = vx_load(src + j); - v1 = vx_load(src + j + v_uint16::nlanes); - v0 = (v0 <= thresh_u) & v0; - v1 = (v1 <= thresh_u) & v1; + v1 = vx_load(src + j + VTraits::vlanes()); + v0 = v_and(v_le(v0, thresh_u), v0); + v1 = v_and(v_le(v1, thresh_u), v1); v_store(dst + j, v0); - v_store(dst + j + v_uint16::nlanes, v1); + v_store(dst + j + VTraits::vlanes(), v1); } - if (j <= roi.width - v_uint16::nlanes) + if (j <= roi.width - VTraits::vlanes()) { v_uint16 v0 = vx_load(src + j); - v0 = (v0 <= thresh_u) & v0; + v0 = v_and(v_le(v0, thresh_u), v0); v_store(dst + j, v0); - j += v_uint16::nlanes; + j += VTraits::vlanes(); } for (; j < roi.width; j++) @@ -571,7 +571,7 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) } #endif -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int i, j; v_int16 thresh8 = vx_setall_s16( thresh ); v_int16 maxval8 = vx_setall_s16( maxval ); @@ -582,25 +582,25 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); - v0 = thresh8 < v0; - v1 = thresh8 < v1; - v0 = v0 & maxval8; - v1 = v1 & maxval8; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_lt(thresh8, v0); + v1 = v_lt(thresh8, v1); + v0 = v_and(v0, maxval8); + v1 = v_and(v1, maxval8); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); - v0 = thresh8 < v0; - v0 = v0 & maxval8; + v0 = v_lt(thresh8, v0); + v0 = v_and(v0, maxval8); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -612,25 +612,25 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); - v0 = v0 <= thresh8; - v1 = v1 <= thresh8; - v0 = v0 & maxval8; - v1 = v1 & maxval8; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_le(v0, thresh8); + v1 = v_le(v1, thresh8); + v0 = v_and(v0, maxval8); + v1 = v_and(v1, maxval8); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); - v0 = v0 <= thresh8; - v0 = v0 & maxval8; + v0 = v_le(v0, thresh8); + v0 = v_and(v0, maxval8); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -642,22 +642,22 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); + v1 = vx_load( src + j + VTraits::vlanes() ); v0 = v_min( v0, thresh8 ); v1 = v_min( v1, thresh8 ); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); v0 = v_min( v0, thresh8 ); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -669,22 +669,22 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); - v0 = ( thresh8 < v0 ) & v0; - v1 = ( thresh8 < v1 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_lt(thresh8, v0), v0); + v1 = v_and(v_lt(thresh8, v1), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); - v0 = ( thresh8 < v0 ) & v0; + v0 = v_and(v_lt(thresh8, v0), v0); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -696,22 +696,22 @@ thresh_16s( const Mat& _src, Mat& _dst, short thresh, short maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_int16::nlanes; j += 2*v_int16::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_int16 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_int16::nlanes ); - v0 = ( v0 <= thresh8 ) & v0; - v1 = ( v1 <= thresh8 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_le(v0, thresh8), v0); + v1 = v_and(v_le(v1, thresh8), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_int16::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_int16::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_int16 v0 = vx_load( src + j ); - v0 = ( v0 <= thresh8 ) & v0; + v0 = v_and(v_le(v0, thresh8), v0); v_store( dst + j, v0 ); - j += v_int16::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -777,7 +777,7 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) } #endif -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) int i, j; v_float32 thresh4 = vx_setall_f32( thresh ); v_float32 maxval4 = vx_setall_f32( maxval ); @@ -788,25 +788,25 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); - v0 = thresh4 < v0; - v1 = thresh4 < v1; - v0 = v0 & maxval4; - v1 = v1 & maxval4; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_lt(thresh4, v0); + v1 = v_lt(thresh4, v1); + v0 = v_and(v0, maxval4); + v1 = v_and(v1, maxval4); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); - v0 = thresh4 < v0; - v0 = v0 & maxval4; + v0 = v_lt(thresh4, v0); + v0 = v_and(v0, maxval4); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -818,25 +818,25 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); - v0 = v0 <= thresh4; - v1 = v1 <= thresh4; - v0 = v0 & maxval4; - v1 = v1 & maxval4; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_le(v0, thresh4); + v1 = v_le(v1, thresh4); + v0 = v_and(v0, maxval4); + v1 = v_and(v1, maxval4); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); - v0 = v0 <= thresh4; - v0 = v0 & maxval4; + v0 = v_le(v0, thresh4); + v0 = v_and(v0, maxval4); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -848,22 +848,22 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); + v1 = vx_load( src + j + VTraits::vlanes() ); v0 = v_min( v0, thresh4 ); v1 = v_min( v1, thresh4 ); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); v0 = v_min( v0, thresh4 ); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -875,22 +875,22 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); - v0 = ( thresh4 < v0 ) & v0; - v1 = ( thresh4 < v1 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_lt(thresh4, v0), v0); + v1 = v_and(v_lt(thresh4, v1), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); - v0 = ( thresh4 < v0 ) & v0; + v0 = v_and(v_lt(thresh4, v0), v0); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -902,22 +902,22 @@ thresh_32f( const Mat& _src, Mat& _dst, float thresh, float maxval, int type ) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float32::nlanes; j += 2*v_float32::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float32 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float32::nlanes ); - v0 = ( v0 <= thresh4 ) & v0; - v1 = ( v1 <= thresh4 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_le(v0, thresh4), v0); + v1 = v_and(v_le(v1, thresh4), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_float32::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float32::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float32 v0 = vx_load( src + j ); - v0 = ( v0 <= thresh4 ) & v0; + v0 = v_and(v_le(v0, thresh4), v0); v_store( dst + j, v0 ); - j += v_float32::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -948,7 +948,7 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) roi.height = 1; } -#if CV_SIMD_64F +#if (CV_SIMD_64F || CV_SIMD_SCALABLE_64F) int i, j; v_float64 thresh2 = vx_setall_f64( thresh ); v_float64 maxval2 = vx_setall_f64( maxval ); @@ -959,25 +959,25 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); - v0 = thresh2 < v0; - v1 = thresh2 < v1; - v0 = v0 & maxval2; - v1 = v1 & maxval2; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_lt(thresh2, v0); + v1 = v_lt(thresh2, v1); + v0 = v_and(v0, maxval2); + v1 = v_and(v1, maxval2); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); - v0 = thresh2 < v0; - v0 = v0 & maxval2; + v0 = v_lt(thresh2, v0); + v0 = v_and(v0, maxval2); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -989,25 +989,25 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); - v0 = v0 <= thresh2; - v1 = v1 <= thresh2; - v0 = v0 & maxval2; - v1 = v1 & maxval2; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_le(v0, thresh2); + v1 = v_le(v1, thresh2); + v0 = v_and(v0, maxval2); + v1 = v_and(v1, maxval2); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); - v0 = v0 <= thresh2; - v0 = v0 & maxval2; + v0 = v_le(v0, thresh2); + v0 = v_and(v0, maxval2); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -1019,22 +1019,22 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); + v1 = vx_load( src + j + VTraits::vlanes() ); v0 = v_min( v0, thresh2 ); v1 = v_min( v1, thresh2 ); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); v0 = v_min( v0, thresh2 ); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -1046,22 +1046,22 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); - v0 = ( thresh2 < v0 ) & v0; - v1 = ( thresh2 < v1 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_lt(thresh2, v0), v0); + v1 = v_and(v_lt(thresh2, v1), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); - v0 = ( thresh2 < v0 ) & v0; + v0 = v_and(v_lt(thresh2, v0), v0); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) @@ -1073,22 +1073,22 @@ thresh_64f(const Mat& _src, Mat& _dst, double thresh, double maxval, int type) for( i = 0; i < roi.height; i++, src += src_step, dst += dst_step ) { j = 0; - for( ; j <= roi.width - 2*v_float64::nlanes; j += 2*v_float64::nlanes ) + for( ; j <= roi.width - 2*VTraits::vlanes(); j += 2*VTraits::vlanes() ) { v_float64 v0, v1; v0 = vx_load( src + j ); - v1 = vx_load( src + j + v_float64::nlanes ); - v0 = ( v0 <= thresh2 ) & v0; - v1 = ( v1 <= thresh2 ) & v1; + v1 = vx_load( src + j + VTraits::vlanes() ); + v0 = v_and(v_le(v0, thresh2), v0); + v1 = v_and(v_le(v1, thresh2), v1); v_store( dst + j, v0 ); - v_store( dst + j + v_float64::nlanes, v1 ); + v_store( dst + j + VTraits::vlanes(), v1 ); } - if( j <= roi.width - v_float64::nlanes ) + if( j <= roi.width - VTraits::vlanes() ) { v_float64 v0 = vx_load( src + j ); - v0 = ( v0 <= thresh2 ) & v0; + v0 = v_and(v_le(v0, thresh2), v0); v_store( dst + j, v0 ); - j += v_float64::nlanes; + j += VTraits::vlanes(); } for( ; j < roi.width; j++ ) diff --git a/modules/js/src/core_bindings.cpp b/modules/js/src/core_bindings.cpp index 60fe496ce3..addee2de20 100644 --- a/modules/js/src/core_bindings.cpp +++ b/modules/js/src/core_bindings.cpp @@ -89,9 +89,11 @@ using namespace cv; using namespace cv::segmentation; // FIXIT +#ifdef HAVE_OPENCV_OBJDETECT using namespace cv::aruco; typedef aruco::DetectorParameters aruco_DetectorParameters; typedef QRCodeDetectorAruco::Params QRCodeDetectorAruco_Params; +#endif #ifdef HAVE_OPENCV_DNN using namespace cv::dnn; diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_board.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_board.hpp index 1f41474405..e8300c82bf 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_board.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_board.hpp @@ -166,11 +166,11 @@ public: */ CV_WRAP std::vector getChessboardCorners() const; - /** @brief get CharucoBoard::nearestMarkerIdx + /** @brief get CharucoBoard::nearestMarkerIdx, for each charuco corner, nearest marker index in ids array */ CV_PROP std::vector > getNearestMarkerIdx() const; - /** @brief get CharucoBoard::nearestMarkerCorners + /** @brief get CharucoBoard::nearestMarkerCorners, for each charuco corner, nearest marker corner id of each marker */ CV_PROP std::vector > getNearestMarkerCorners() const; diff --git a/modules/objdetect/misc/python/test/test_objdetect_aruco.py b/modules/objdetect/misc/python/test/test_objdetect_aruco.py index 92bad17073..00d8095b69 100644 --- a/modules/objdetect/misc/python/test/test_objdetect_aruco.py +++ b/modules/objdetect/misc/python/test/test_objdetect_aruco.py @@ -394,5 +394,69 @@ class aruco_objdetect_test(NewOpenCVTests): self.assertEqual(2, img_points.shape[1]) np.testing.assert_array_equal(chessboard_corners, obj_points[:, :2].reshape(-1, 2)) + def test_draw_detected_markers(self): + detected_points = [[[10, 10], [50, 10], [50, 50], [10, 50]]] + img = np.zeros((60, 60), dtype=np.uint8) + + # add extra dimension in Python to create Nx4 Mat with 2 channels + points1 = np.array(detected_points).reshape(-1, 4, 1, 2) + img = cv.aruco.drawDetectedMarkers(img, points1, borderColor=255) + + # check that the marker borders are painted + contours, _ = cv.findContours(img, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) + self.assertEqual(len(contours), 1) + self.assertEqual(img[10, 10], 255) + self.assertEqual(img[50, 10], 255) + self.assertEqual(img[50, 50], 255) + self.assertEqual(img[10, 50], 255) + + # must throw Exception without extra dimension + points2 = np.array(detected_points) + with self.assertRaises(Exception): + img = cv.aruco.drawDetectedMarkers(img, points2, borderColor=255) + + def test_draw_detected_charuco(self): + detected_points = [[[10, 10], [50, 10], [50, 50], [10, 50]]] + img = np.zeros((60, 60), dtype=np.uint8) + + # add extra dimension in Python to create Nx1 Mat with 2 channels + points = np.array(detected_points).reshape(-1, 1, 2) + img = cv.aruco.drawDetectedCornersCharuco(img, points, cornerColor=255) + + # check that the 4 charuco corners are painted + contours, _ = cv.findContours(img, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) + self.assertEqual(len(contours), 4) + for contour in contours: + center_x = round(np.average(contour[:, 0, 0])) + center_y = round(np.average(contour[:, 0, 1])) + center = [center_x, center_y] + self.assertTrue(center in detected_points[0]) + + # must throw Exception without extra dimension + points2 = np.array(detected_points) + with self.assertRaises(Exception): + img = cv.aruco.drawDetectedCornersCharuco(img, points2, borderColor=255) + + def test_draw_detected_diamonds(self): + detected_points = [[[10, 10], [50, 10], [50, 50], [10, 50]]] + img = np.zeros((60, 60), dtype=np.uint8) + + # add extra dimension in Python to create Nx4 Mat with 2 channels + points = np.array(detected_points).reshape(-1, 4, 1, 2) + img = cv.aruco.drawDetectedDiamonds(img, points, borderColor=255) + + # check that the diamonds borders are painted + contours, _ = cv.findContours(img, cv.RETR_EXTERNAL, cv.CHAIN_APPROX_SIMPLE) + self.assertEqual(len(contours), 1) + self.assertEqual(img[10, 10], 255) + self.assertEqual(img[50, 10], 255) + self.assertEqual(img[50, 50], 255) + self.assertEqual(img[10, 50], 255) + + # must throw Exception without extra dimension + points2 = np.array(detected_points) + with self.assertRaises(Exception): + img = cv.aruco.drawDetectedDiamonds(img, points2, borderColor=255) + if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/objdetect/src/aruco/apriltag/apriltag_quad_thresh.cpp b/modules/objdetect/src/aruco/apriltag/apriltag_quad_thresh.cpp index 38d1b2ffb8..96b7fe517e 100644 --- a/modules/objdetect/src/aruco/apriltag/apriltag_quad_thresh.cpp +++ b/modules/objdetect/src/aruco/apriltag/apriltag_quad_thresh.cpp @@ -349,7 +349,7 @@ int quad_segment_maxima(const DetectorParameters &td, int sz, struct line_fit_pt } y[iy] = acc; } - copy(y.begin(), y.end(), errs.begin()); + std::copy(y.begin(), y.end(), errs.begin()); } std::vector maxima(sz); diff --git a/modules/objdetect/src/aruco/aruco_board.cpp b/modules/objdetect/src/aruco/aruco_board.cpp index 2ad6c5757b..cf45a96450 100644 --- a/modules/objdetect/src/aruco/aruco_board.cpp +++ b/modules/objdetect/src/aruco/aruco_board.cpp @@ -319,8 +319,9 @@ struct CharucoBoardImpl : Board::Impl { // vector of chessboard 3D corners precalculated std::vector chessboardCorners; - // for each charuco corner, nearest marker id and nearest marker corner id of each marker + // for each charuco corner, nearest marker index in ids array std::vector > nearestMarkerIdx; + // for each charuco corner, nearest marker corner id of each marker std::vector > nearestMarkerCorners; void createCharucoBoard(); diff --git a/modules/objdetect/src/aruco/aruco_detector.cpp b/modules/objdetect/src/aruco/aruco_detector.cpp index a8df4e6d44..73643177ee 100644 --- a/modules/objdetect/src/aruco/aruco_detector.cpp +++ b/modules/objdetect/src/aruco/aruco_detector.cpp @@ -1321,24 +1321,26 @@ void drawDetectedMarkers(InputOutputArray _image, InputArrayOfArrays _corners, int nMarkers = (int)_corners.total(); for(int i = 0; i < nMarkers; i++) { Mat currentMarker = _corners.getMat(i); - CV_Assert(currentMarker.total() == 4 && currentMarker.type() == CV_32FC2); + CV_Assert(currentMarker.total() == 4 && currentMarker.channels() == 2); + if (currentMarker.type() != CV_32SC2) + currentMarker.convertTo(currentMarker, CV_32SC2); // draw marker sides for(int j = 0; j < 4; j++) { - Point2f p0, p1; - p0 = currentMarker.ptr(0)[j]; - p1 = currentMarker.ptr(0)[(j + 1) % 4]; + Point p0, p1; + p0 = currentMarker.ptr(0)[j]; + p1 = currentMarker.ptr(0)[(j + 1) % 4]; line(_image, p0, p1, borderColor, 1); } // draw first corner mark - rectangle(_image, currentMarker.ptr(0)[0] - Point2f(3, 3), - currentMarker.ptr(0)[0] + Point2f(3, 3), cornerColor, 1, LINE_AA); + rectangle(_image, currentMarker.ptr(0)[0] - Point(3, 3), + currentMarker.ptr(0)[0] + Point(3, 3), cornerColor, 1, LINE_AA); // draw ID if(_ids.total() != 0) { - Point2f cent(0, 0); + Point cent(0, 0); for(int p = 0; p < 4; p++) - cent += currentMarker.ptr(0)[p]; + cent += currentMarker.ptr(0)[p]; cent = cent / 4.; stringstream s; s << "id=" << _ids.getMat().ptr(0)[i]; diff --git a/modules/objdetect/src/aruco/charuco_detector.cpp b/modules/objdetect/src/aruco/charuco_detector.cpp index 3838a6f5c5..a7b17c4798 100644 --- a/modules/objdetect/src/aruco/charuco_detector.cpp +++ b/modules/objdetect/src/aruco/charuco_detector.cpp @@ -5,6 +5,7 @@ #include "../precomp.hpp" #include +#include #include "opencv2/objdetect/charuco_detector.hpp" #include "aruco_utils.hpp" @@ -26,12 +27,13 @@ struct CharucoDetector::CharucoDetectorImpl { bool checkBoard(InputArrayOfArrays markerCorners, InputArray markerIds, InputArray charucoCorners, InputArray charucoIds) { vector mCorners; markerCorners.getMatVector(mCorners); - Mat mIds = markerIds.getMat(); + const Mat& mIds = markerIds.getMat(); - Mat chCorners = charucoCorners.getMat(); - Mat chIds = charucoIds.getMat(); + const Mat& chCorners = charucoCorners.getMat(); + const Mat& chIds = charucoIds.getMat(); + const vector& boardIds = board.getIds(); - vector > nearestMarkerIdx = board.getNearestMarkerIdx(); + const vector >& nearestMarkerIdx = board.getNearestMarkerIdx(); vector distance(board.getNearestMarkerIdx().size(), Point2f(0.f, std::numeric_limits::max())); // distance[i].x: max distance from the i-th charuco corner to charuco corner-forming markers. // The two charuco corner-forming markers of i-th charuco corner are defined in getNearestMarkerIdx()[i] @@ -41,13 +43,19 @@ struct CharucoDetector::CharucoDetectorImpl { Point2f charucoCorner(chCorners.ptr(0)[i]); for (size_t j = 0ull; j < mIds.total(); j++) { int idMaker = mIds.ptr(0)[j]; + // skip the check if the marker is not in the current board. + if (find(boardIds.begin(), boardIds.end(), idMaker) == boardIds.end()) + continue; Point2f centerMarker((mCorners[j].ptr(0)[0] + mCorners[j].ptr(0)[1] + mCorners[j].ptr(0)[2] + mCorners[j].ptr(0)[3]) / 4.f); float dist = sqrt(normL2Sqr(centerMarker - charucoCorner)); - // check distance from the charuco corner to charuco corner-forming markers - if (nearestMarkerIdx[chId][0] == idMaker || nearestMarkerIdx[chId][1] == idMaker) { - int nearestCornerId = nearestMarkerIdx[chId][0] == idMaker ? board.getNearestMarkerCorners()[chId][0] : board.getNearestMarkerCorners()[chId][1]; + // nearestMarkerIdx contains for each charuco corner, nearest marker index in ids array + const int nearestMarkerId1 = boardIds[nearestMarkerIdx[chId][0]]; + const int nearestMarkerId2 = boardIds[nearestMarkerIdx[chId][1]]; + if (nearestMarkerId1 == idMaker || nearestMarkerId2 == idMaker) { + int nearestCornerId = nearestMarkerId1 == idMaker ? board.getNearestMarkerCorners()[chId][0] : board.getNearestMarkerCorners()[chId][1]; Point2f nearestCorner = mCorners[j].ptr(0)[nearestCornerId]; + // distToNearest: distance from the charuco corner to charuco corner-forming markers float distToNearest = sqrt(normL2Sqr(nearestCorner - charucoCorner)); distance[chId].x = max(distance[chId].x, distToNearest); // check that nearestCorner is nearest point @@ -363,6 +371,7 @@ void CharucoDetector::detectBoard(InputArray image, OutputArray charucoCorners, InputOutputArrayOfArrays markerCorners, InputOutputArray markerIds) const { charucoDetectorImpl->detectBoard(image, charucoCorners, charucoIds, markerCorners, markerIds); if (charucoDetectorImpl->checkBoard(markerCorners, markerIds, charucoCorners, charucoIds) == false) { + CV_LOG_DEBUG(NULL, "ChArUco board is built incorrectly"); charucoCorners.release(); charucoIds.release(); } @@ -511,20 +520,27 @@ void drawDetectedCornersCharuco(InputOutputArray _image, InputArray _charucoCorn InputArray _charucoIds, Scalar cornerColor) { CV_Assert(!_image.getMat().empty() && (_image.getMat().channels() == 1 || _image.getMat().channels() == 3)); - CV_Assert((_charucoCorners.getMat().total() == _charucoIds.getMat().total()) || - _charucoIds.getMat().total() == 0); + CV_Assert((_charucoCorners.total() == _charucoIds.total()) || + _charucoIds.total() == 0); + CV_Assert(_charucoCorners.channels() == 2); - size_t nCorners = _charucoCorners.getMat().total(); + Mat charucoCorners = _charucoCorners.getMat(); + if (charucoCorners.type() != CV_32SC2) + charucoCorners.convertTo(charucoCorners, CV_32SC2); + Mat charucoIds; + if (!_charucoIds.empty()) + charucoIds = _charucoIds.getMat(); + size_t nCorners = charucoCorners.total(); for(size_t i = 0; i < nCorners; i++) { - Point2f corner = _charucoCorners.getMat().at((int)i); + Point corner = charucoCorners.at((int)i); // draw first corner mark - rectangle(_image, corner - Point2f(3, 3), corner + Point2f(3, 3), cornerColor, 1, LINE_AA); + rectangle(_image, corner - Point(3, 3), corner + Point(3, 3), cornerColor, 1, LINE_AA); // draw ID if(!_charucoIds.empty()) { - int id = _charucoIds.getMat().at((int)i); + int id = charucoIds.at((int)i); stringstream s; s << "id=" << id; - putText(_image, s.str(), corner + Point2f(5, -5), FONT_HERSHEY_SIMPLEX, 0.5, + putText(_image, s.str(), corner + Point(5, -5), FONT_HERSHEY_SIMPLEX, 0.5, cornerColor, 2); } } @@ -544,25 +560,27 @@ void drawDetectedDiamonds(InputOutputArray _image, InputArrayOfArrays _corners, int nMarkers = (int)_corners.total(); for(int i = 0; i < nMarkers; i++) { Mat currentMarker = _corners.getMat(i); - CV_Assert(currentMarker.total() == 4 && currentMarker.type() == CV_32FC2); + CV_Assert(currentMarker.total() == 4 && currentMarker.channels() == 2); + if (currentMarker.type() != CV_32SC2) + currentMarker.convertTo(currentMarker, CV_32SC2); // draw marker sides for(int j = 0; j < 4; j++) { - Point2f p0, p1; - p0 = currentMarker.at< Point2f >(j); - p1 = currentMarker.at< Point2f >((j + 1) % 4); + Point p0, p1; + p0 = currentMarker.at(j); + p1 = currentMarker.at((j + 1) % 4); line(_image, p0, p1, borderColor, 1); } // draw first corner mark - rectangle(_image, currentMarker.at< Point2f >(0) - Point2f(3, 3), - currentMarker.at< Point2f >(0) + Point2f(3, 3), cornerColor, 1, LINE_AA); + rectangle(_image, currentMarker.at(0) - Point(3, 3), + currentMarker.at(0) + Point(3, 3), cornerColor, 1, LINE_AA); // draw id composed by four numbers if(_ids.total() != 0) { - Point2f cent(0, 0); + Point cent(0, 0); for(int p = 0; p < 4; p++) - cent += currentMarker.at< Point2f >(p); + cent += currentMarker.at(p); cent = cent / 4.; stringstream s; s << "id=" << _ids.getMat().at< Vec4i >(i); diff --git a/modules/objdetect/test/test_charucodetection.cpp b/modules/objdetect/test/test_charucodetection.cpp index 5f392a0f62..ff6f4fbe5f 100644 --- a/modules/objdetect/test/test_charucodetection.cpp +++ b/modules/objdetect/test/test_charucodetection.cpp @@ -691,6 +691,58 @@ TEST(Charuco, testmatchImagePoints) } } +typedef testing::TestWithParam CharucoDraw; +INSTANTIATE_TEST_CASE_P(/**/, CharucoDraw, testing::Values(CV_8UC2, CV_8SC2, CV_16UC2, CV_16SC2, CV_32SC2, CV_32FC2, CV_64FC2)); +TEST_P(CharucoDraw, testDrawDetected) { + vector> detected_golds = {{Point(20, 20), Point(80, 20), Point(80, 80), Point2f(20, 80)}}; + Point center_gold = (detected_golds[0][0] + detected_golds[0][1] + detected_golds[0][2] + detected_golds[0][3]) / 4; + int type = GetParam(); + vector detected(detected_golds[0].size(), Mat(4, 1, type)); + // copy detected_golds to detected with any 2 channels type + for (size_t i = 0ull; i < detected_golds[0].size(); i++) { + detected[0].row((int)i) = Scalar(detected_golds[0][i].x, detected_golds[0][i].y); + } + vector> contours; + Point detectedCenter; + Moments m; + Mat img; + + // check drawDetectedMarkers + img = Mat::zeros(100, 100, CV_8UC1); + ASSERT_NO_THROW(aruco::drawDetectedMarkers(img, detected, noArray(), Scalar(255, 255, 255))); + // check that the marker borders are painted + findContours(img, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); + ASSERT_EQ(contours.size(), 1ull); + m = moments(contours[0]); + detectedCenter = Point(cvRound(m.m10/m.m00), cvRound(m.m01/m.m00)); + ASSERT_EQ(detectedCenter, center_gold); + + + // check drawDetectedCornersCharuco + img = Mat::zeros(100, 100, CV_8UC1); + ASSERT_NO_THROW(aruco::drawDetectedCornersCharuco(img, detected[0], noArray(), Scalar(255, 255, 255))); + // check that the 4 charuco corners are painted + findContours(img, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); + ASSERT_EQ(contours.size(), 4ull); + for (size_t i = 0ull; i < 4ull; i++) { + m = moments(contours[i]); + detectedCenter = Point(cvRound(m.m10/m.m00), cvRound(m.m01/m.m00)); + // detectedCenter must be in detected_golds + ASSERT_TRUE(find(detected_golds[0].begin(), detected_golds[0].end(), detectedCenter) != detected_golds[0].end()); + } + + + // check drawDetectedDiamonds + img = Mat::zeros(100, 100, CV_8UC1); + ASSERT_NO_THROW(aruco::drawDetectedDiamonds(img, detected, noArray(), Scalar(255, 255, 255))); + // check that the diamonds borders are painted + findContours(img, contours, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE); + ASSERT_EQ(contours.size(), 1ull); + m = moments(contours[0]); + detectedCenter = Point(cvRound(m.m10/m.m00), cvRound(m.m01/m.m00)); + ASSERT_EQ(detectedCenter, center_gold); +} + typedef testing::TestWithParam CharucoBoard; INSTANTIATE_TEST_CASE_P(/**/, CharucoBoard, testing::Values(Size(3, 2), Size(3, 2), Size(6, 2), Size(2, 6), Size(3, 4), Size(4, 3), Size(7, 3), Size(3, 7))); @@ -719,4 +771,60 @@ TEST_P(CharucoBoard, testWrongSizeDetection) ASSERT_TRUE(detectedCharucoIds.empty()); } +// Temporary disabled in https://github.com/opencv/opencv/pull/24338 +// 5.x version produces conrnes with different shape than 4.x (32F_C2 instead of 2x 32FC1) +TEST(Charuco, DISABLED_testSeveralBoardsWithCustomIds) +{ + Size res{500, 500}; + Mat K = (Mat_(3,3) << + 0.5*res.width, 0, 0.5*res.width, + 0, 0.5*res.height, 0.5*res.height, + 0, 0, 1); + + Mat expected_corners = (Mat_(9,2) << + 200, 200, + 250, 200, + 300, 200, + 200, 250, + 250, 250, + 300, 250, + 200, 300, + 250, 300, + 300, 300 + ); + + + aruco::Dictionary dict = cv::aruco::getPredefinedDictionary(aruco::DICT_4X4_50); + vector ids1 = {0, 1, 33, 3, 4, 5, 6, 8}, ids2 = {7, 9, 44, 11, 12, 13, 14, 15}; + aruco::CharucoBoard board1(Size(4, 4), 1.f, .8f, dict, ids1), board2(Size(4, 4), 1.f, .8f, dict, ids2); + + // generate ChArUco board + Mat gray; + { + Mat gray1, gray2; + board1.generateImage(Size(res.width, res.height), gray1, 150); + board2.generateImage(Size(res.width, res.height), gray2, 150); + hconcat(gray1, gray2, gray); + } + + aruco::CharucoParameters charucoParameters; + charucoParameters.cameraMatrix = K; + aruco::CharucoDetector detector1(board1, charucoParameters), detector2(board2, charucoParameters); + + vector ids; + vector corners; + Mat c_ids1, c_ids2, c_corners1, c_corners2; + + detector1.detectBoard(gray, c_corners1, c_ids1, corners, ids); + detector2.detectBoard(gray, c_corners2, c_ids2, corners, ids); + + ASSERT_EQ(ids.size(), size_t(16)); + ASSERT_EQ(c_corners1.rows, expected_corners.rows); + EXPECT_NEAR(0, cvtest::norm(expected_corners, c_corners1.reshape(1), NORM_INF), 3e-1); + + ASSERT_EQ(c_corners2.rows, expected_corners.rows); + expected_corners.col(0) += 500; + EXPECT_NEAR(0, cvtest::norm(expected_corners, c_corners2.reshape(1), NORM_INF), 3e-1); +} + }} // namespace diff --git a/modules/python/src2/cv2.hpp b/modules/python/src2/cv2.hpp index 9293a593f2..b7992582ad 100644 --- a/modules/python/src2/cv2.hpp +++ b/modules/python/src2/cv2.hpp @@ -39,12 +39,20 @@ class ArgInfo { +private: + static const uint32_t arg_outputarg_flag = 0x1; + static const uint32_t arg_arithm_op_src_flag = 0x2; + public: const char* name; bool outputarg; + bool arithm_op_src; // more fields may be added if necessary - ArgInfo(const char* name_, bool outputarg_) : name(name_), outputarg(outputarg_) {} + ArgInfo(const char* name_, uint32_t arg_) : + name(name_), + outputarg((arg_ & arg_outputarg_flag) != 0), + arithm_op_src((arg_ & arg_arithm_op_src_flag) != 0) {} private: ArgInfo(const ArgInfo&) = delete; diff --git a/modules/python/src2/cv2_convert.cpp b/modules/python/src2/cv2_convert.cpp index e9e1fed4fd..40e1608fae 100644 --- a/modules/python/src2/cv2_convert.cpp +++ b/modules/python/src2/cv2_convert.cpp @@ -63,20 +63,39 @@ bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) if( PyInt_Check(o) ) { double v[] = {static_cast(PyInt_AsLong((PyObject*)o)), 0., 0., 0.}; + if ( info.arithm_op_src ) + { + // Normally cv.XXX(x) means cv.XXX( (x, 0., 0., 0.) ); + // However cv.add(mat,x) means cv::add(mat, (x,x,x,x) ). + v[1] = v[0]; + v[2] = v[0]; + v[3] = v[0]; + } m = Mat(4, 1, CV_64F, v).clone(); return true; } if( PyFloat_Check(o) ) { double v[] = {PyFloat_AsDouble((PyObject*)o), 0., 0., 0.}; + + if ( info.arithm_op_src ) + { + // Normally cv.XXX(x) means cv.XXX( (x, 0., 0., 0.) ); + // However cv.add(mat,x) means cv::add(mat, (x,x,x,x) ). + v[1] = v[0]; + v[2] = v[0]; + v[3] = v[0]; + } m = Mat(4, 1, CV_64F, v).clone(); return true; } if( PyTuple_Check(o) ) { - int i, sz = (int)PyTuple_Size((PyObject*)o); - m = Mat(sz, 1, CV_64F); - for( i = 0; i < sz; i++ ) + // see https://github.com/opencv/opencv/issues/24057 + const int sz = (int)PyTuple_Size((PyObject*)o); + const int sz2 = info.arithm_op_src ? std::max(4, sz) : sz; // Scalar has 4 elements. + m = Mat::zeros(sz2, 1, CV_64F); + for( int i = 0; i < sz; i++ ) { PyObject* oi = PyTuple_GetItem(o, i); if( PyInt_Check(oi) ) @@ -241,6 +260,31 @@ bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) } } + // see https://github.com/opencv/opencv/issues/24057 + if ( ( info.arithm_op_src ) && ( ndims == 1 ) && ( size[0] <= 4 ) ) + { + const int sz = size[0]; // Real Data Length(1, 2, 3 or 4) + const int sz2 = 4; // Scalar has 4 elements. + m = Mat::zeros(sz2, 1, CV_64F); + + const char *base_ptr = PyArray_BYTES(oarr); + for(int i = 0; i < sz; i++ ) + { + PyObject* oi = PyArray_GETITEM(oarr, base_ptr + step[0] * i); + if( PyInt_Check(oi) ) + m.at(i) = (double)PyInt_AsLong(oi); + else if( PyFloat_Check(oi) ) + m.at(i) = (double)PyFloat_AsDouble(oi); + else + { + failmsg("%s has some non-numerical elements", info.name); + m.release(); + return false; + } + } + return true; + } + // handle degenerate case // FIXIT: Don't force 1D for Scalars if( ndims == 0) { @@ -807,7 +851,7 @@ bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) } { const String centerItemName = format("'%s' center point", info.name); - const ArgInfo centerItemInfo(centerItemName.c_str(), false); + const ArgInfo centerItemInfo(centerItemName.c_str(), 0); SafeSeqItem centerItem(obj, 0); if (!pyopencv_to(centerItem.item, dst.center, centerItemInfo)) { @@ -816,7 +860,7 @@ bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) } { const String sizeItemName = format("'%s' size", info.name); - const ArgInfo sizeItemInfo(sizeItemName.c_str(), false); + const ArgInfo sizeItemInfo(sizeItemName.c_str(), 0); SafeSeqItem sizeItem(obj, 1); if (!pyopencv_to(sizeItem.item, dst.size, sizeItemInfo)) { @@ -825,7 +869,7 @@ bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) } { const String angleItemName = format("'%s' angle", info.name); - const ArgInfo angleItemInfo(angleItemName.c_str(), false); + const ArgInfo angleItemInfo(angleItemName.c_str(), 0); SafeSeqItem angleItem(obj, 2); if (!pyopencv_to(angleItem.item, dst.angle, angleItemInfo)) { @@ -1075,7 +1119,7 @@ bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) } { const String typeItemName = format("'%s' criteria type", info.name); - const ArgInfo typeItemInfo(typeItemName.c_str(), false); + const ArgInfo typeItemInfo(typeItemName.c_str(), 0); SafeSeqItem typeItem(obj, 0); if (!pyopencv_to(typeItem.item, dst.type, typeItemInfo)) { @@ -1084,7 +1128,7 @@ bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) } { const String maxCountItemName = format("'%s' max count", info.name); - const ArgInfo maxCountItemInfo(maxCountItemName.c_str(), false); + const ArgInfo maxCountItemInfo(maxCountItemName.c_str(), 0); SafeSeqItem maxCountItem(obj, 1); if (!pyopencv_to(maxCountItem.item, dst.maxCount, maxCountItemInfo)) { @@ -1093,7 +1137,7 @@ bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) } { const String epsilonItemName = format("'%s' epsilon", info.name); - const ArgInfo epsilonItemInfo(epsilonItemName.c_str(), false); + const ArgInfo epsilonItemInfo(epsilonItemName.c_str(), 0); SafeSeqItem epsilonItem(obj, 2); if (!pyopencv_to(epsilonItem.item, dst.epsilon, epsilonItemInfo)) { diff --git a/modules/python/src2/cv2_convert.hpp b/modules/python/src2/cv2_convert.hpp index 43ef7b2302..96a30e521f 100644 --- a/modules/python/src2/cv2_convert.hpp +++ b/modules/python/src2/cv2_convert.hpp @@ -286,13 +286,13 @@ bool pyopencv_to(PyObject *obj, std::map &map, const ArgInfo& info) while(PyDict_Next(obj, &pos, &py_key, &py_value)) { K cpp_key; - if (!pyopencv_to(py_key, cpp_key, ArgInfo("key", false))) { + if (!pyopencv_to(py_key, cpp_key, ArgInfo("key", 0))) { failmsg("Can't parse dict key. Key on position %lu has a wrong type", pos); return false; } V cpp_value; - if (!pyopencv_to(py_value, cpp_value, ArgInfo("value", false))) { + if (!pyopencv_to(py_value, cpp_value, ArgInfo("value", 0))) { failmsg("Can't parse dict value. Value on position %lu has a wrong type", pos); return false; } diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index 13b6d7ed1e..3235966b23 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -109,7 +109,7 @@ gen_template_set_prop_from_map = Template(""" if( PyMapping_HasKeyString(src, (char*)"$propname") ) { tmp = PyMapping_GetItemString(src, (char*)"$propname"); - ok = tmp && pyopencv_to_safe(tmp, dst.$propname, ArgInfo("$propname", false)); + ok = tmp && pyopencv_to_safe(tmp, dst.$propname, ArgInfo("$propname", 0)); Py_DECREF(tmp); if(!ok) return false; }""") @@ -163,7 +163,7 @@ static int pyopencv_${name}_set_${member}(pyopencv_${name}_t* p, PyObject *value PyErr_SetString(PyExc_TypeError, "Cannot delete the ${member} attribute"); return -1; } - return pyopencv_to_safe(value, p->v${access}${member}, ArgInfo("value", false)) ? 0 : -1; + return pyopencv_to_safe(value, p->v${access}${member}, ArgInfo("value", 0)) ? 0 : -1; } """) @@ -181,7 +181,7 @@ static int pyopencv_${name}_set_${member}(pyopencv_${name}_t* p, PyObject *value failmsgp("Incorrect type of object (must be '${name}' or its derivative)"); return -1; } - return pyopencv_to_safe(value, _self_${access}${member}, ArgInfo("value", false)) ? 0 : -1; + return pyopencv_to_safe(value, _self_${access}${member}, ArgInfo("value", 0)) ? 0 : -1; } """) @@ -492,6 +492,10 @@ class ArgInfo(object): def inputarg(self): return '/O' not in self._modifiers + @property + def arithm_op_src_arg(self): + return '/AOS' in self._modifiers + @property def outputarg(self): return '/O' in self._modifiers or '/IO' in self._modifiers @@ -517,7 +521,9 @@ class ArgInfo(object): "UMat", "vector_UMat"] # or self.tp.startswith("vector") def crepr(self): - return "ArgInfo(\"%s\", %d)" % (self.name, self.outputarg) + arg = 0x01 if self.outputarg else 0x0 + arg += 0x02 if self.arithm_op_src_arg else 0x0 + return "ArgInfo(\"%s\", %d)" % (self.name, arg) def find_argument_class_info(argument_type, function_namespace, diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index 0e883b013b..c9139c516b 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -537,6 +537,13 @@ class CppHeaderParser(object): funcname = self.get_dotted_name(funcname) + # see https://github.com/opencv/opencv/issues/24057 + is_arithm_op_func = funcname in {"cv.add", + "cv.subtract", + "cv.absdiff", + "cv.multiply", + "cv.divide"} + if not self.wrap_mode: decl = self.parse_func_decl_no_wrap(decl_str, static_method, docstring) decl[0] = funcname @@ -597,6 +604,8 @@ class CppHeaderParser(object): if arg_type == "InputArray": arg_type = mat + if is_arithm_op_func: + modlist.append("/AOS") # Arithm Ope Source elif arg_type == "InputOutputArray": arg_type = mat modlist.append("/IO") diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 9f7406587c..51ece30dc6 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -42,6 +42,93 @@ def get_conversion_error_msg(value, expected, actual): def get_no_exception_msg(value): return 'Exception is not risen for {} of type {}'.format(value, type(value).__name__) + +def rpad(src, dst_size, pad_value=0): + """Extend `src` up to `dst_size` with given value. + + Args: + src (np.ndarray | tuple | list): 1d array like object to pad. + dst_size (_type_): Desired `src` size after padding. + pad_value (int, optional): Padding value. Defaults to 0. + + Returns: + np.ndarray: 1d array with len == `dst_size`. + """ + src = np.asarray(src) + if len(src.shape) != 1: + raise ValueError("Only 1d arrays are supported") + + # Considering the meaning, it is desirable to use np.pad(). + # However, the old numpy doesn't include the following fixes and cannot work as expected. + # So an alternative fix that combines np.append() and np.fill() is used. + # https://docs.scipy.org/doc/numpy-1.13.0/release.html#support-for-returning-arrays-of-arbitrary-dimensions-in-apply-along-axis + + return np.append(src, np.full( dst_size - len(src), pad_value, dtype=src.dtype) ) + +def get_ocv_arithm_op_table(apply_saturation=False): + def saturate(func): + def wrapped_func(x, y): + dst_dtype = x.dtype + if apply_saturation: + if np.issubdtype(x.dtype, np.integer): + x = x.astype(np.int64) + # Apply padding or truncation for array-like `y` inputs + if not isinstance(y, (float, int)): + if len(y) > x.shape[-1]: + y = y[:x.shape[-1]] + else: + y = rpad(y, x.shape[-1], pad_value=0) + + dst = func(x, y) + if apply_saturation: + min_val, max_val = get_limits(dst_dtype) + dst = np.clip(dst, min_val, max_val) + return dst.astype(dst_dtype) + return wrapped_func + + @saturate + def subtract(x, y): + return x - y + + @saturate + def add(x, y): + return x + y + + @saturate + def divide(x, y): + if not isinstance(y, (int, float)): + dst_dtype = np.result_type(x, y) + y = np.array(y).astype(dst_dtype) + _, max_value = get_limits(dst_dtype) + y[y == 0] = max_value + + # to compatible between python2 and python3, it calicurates with float. + # python2: int / int = int + # python3: int / int = float + dst = 1.0 * x / y + + if np.issubdtype(x.dtype, np.integer): + dst = np.rint(dst) + return dst + + @saturate + def multiply(x, y): + return x * y + + @saturate + def absdiff(x, y): + res = np.abs(x - y) + return res + + return { + cv.subtract: subtract, + cv.add: add, + cv.multiply: multiply, + cv.divide: divide, + cv.absdiff: absdiff + } + + class Bindings(NewOpenCVTests): def test_inheritance(self): @@ -816,6 +903,32 @@ class Arguments(NewOpenCVTests): np.testing.assert_equal(dst, src_copy) self.assertEqual(arguments_dump, 'lambda=25, sigma=5.5') + def test_arithm_op_without_saturation(self): + np.random.seed(4231568) + src = np.random.randint(20, 40, 8 * 4 * 3).astype(np.uint8).reshape(8, 4, 3) + operations = get_ocv_arithm_op_table(apply_saturation=False) + for ocv_op, numpy_op in operations.items(): + for val in (2, 4, (5, ), (6, 4), (2., 4., 1.), + np.uint8([1, 2, 2]), np.float64([5, 2, 6, 3]),): + dst = ocv_op(src, val) + expected = numpy_op(src, val) + # Temporarily allows a difference of 1 for arm64 workaround. + self.assertLess(np.max(np.abs(dst - expected)), 2, + msg="Operation '{}' is failed for {}".format(ocv_op.__name__, val ) ) + + def test_arithm_op_with_saturation(self): + np.random.seed(4231568) + src = np.random.randint(20, 40, 4 * 8 * 4).astype(np.uint8).reshape(4, 8, 4) + operations = get_ocv_arithm_op_table(apply_saturation=True) + + for ocv_op, numpy_op in operations.items(): + for val in (10, 4, (40, ), (15, 12), (25., 41., 15.), + np.uint8([1, 2, 20]), np.float64([50, 21, 64, 30]),): + dst = ocv_op(src, val) + expected = numpy_op(src, val) + # Temporarily allows a difference of 1 for arm64 workaround. + self.assertLess(np.max(np.abs(dst - expected)), 2, + msg="Saturated Operation '{}' is failed for {}".format(ocv_op.__name__, val ) ) class CanUsePurePythonModuleFunction(NewOpenCVTests): def test_can_get_ocv_version(self): diff --git a/modules/python/test/tests_common.py b/modules/python/test/tests_common.py index 2c8d26bd38..00f80175ce 100644 --- a/modules/python/test/tests_common.py +++ b/modules/python/test/tests_common.py @@ -36,6 +36,8 @@ class NewOpenCVTests(unittest.TestCase): return candidate if required: self.fail('File ' + filename + ' not found') + else: + self.skipTest('File ' + filename + ' not found') return None diff --git a/modules/stereo/src/stereobm.cpp b/modules/stereo/src/stereobm.cpp index 7571fdc004..2b3aeab927 100644 --- a/modules/stereo/src/stereobm.cpp +++ b/modules/stereo/src/stereobm.cpp @@ -230,13 +230,13 @@ prefilterXSobel( const Mat& src, Mat& dst, int ftzero ) dptr0[0] = dptr0[size.width-1] = dptr1[0] = dptr1[size.width-1] = val0; x = 1; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { v_int16 ftz = vx_setall_s16((short) ftzero); v_int16 ftz2 = vx_setall_s16((short)(ftzero*2)); v_int16 z = vx_setzero_s16(); - for(; x <= (size.width - 1) - v_int16::nlanes; x += v_int16::nlanes) + for(; x <= (size.width - 1) - VTraits::vlanes(); x += VTraits::vlanes()) { v_int16 s00 = v_reinterpret_as_s16(vx_load_expand(srow0 + x + 1)); v_int16 s01 = v_reinterpret_as_s16(vx_load_expand(srow0 + x - 1)); @@ -247,13 +247,13 @@ prefilterXSobel( const Mat& src, Mat& dst, int ftzero ) v_int16 s30 = v_reinterpret_as_s16(vx_load_expand(srow3 + x + 1)); v_int16 s31 = v_reinterpret_as_s16(vx_load_expand(srow3 + x - 1)); - v_int16 d0 = s00 - s01; - v_int16 d1 = s10 - s11; - v_int16 d2 = s20 - s21; - v_int16 d3 = s30 - s31; + v_int16 d0 = v_sub(s00, s01); + v_int16 d1 = v_sub(s10, s11); + v_int16 d2 = v_sub(s20, s21); + v_int16 d3 = v_sub(s30, s31); - v_uint16 v0 = v_reinterpret_as_u16(v_max(v_min(d0 + d1 + d1 + d2 + ftz, ftz2), z)); - v_uint16 v1 = v_reinterpret_as_u16(v_max(v_min(d1 + d2 + d2 + d3 + ftz, ftz2), z)); + v_uint16 v0 = v_reinterpret_as_u16(v_max(v_min(v_add(v_add(v_add(v_add(d0, d1), d1), d2), ftz), ftz2), z)); + v_uint16 v1 = v_reinterpret_as_u16(v_max(v_min(v_add(v_add(v_add(v_add(d1, d2), d2), d3), ftz), ftz2), z)); v_pack_store(dptr0 + x, v0); v_pack_store(dptr1 + x, v1); @@ -276,10 +276,10 @@ prefilterXSobel( const Mat& src, Mat& dst, int ftzero ) { uchar* dptr = dst.ptr(y); x = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { v_uint8 val0_16 = vx_setall_u8(val0); - for(; x <= size.width-v_uint8::nlanes; x+=v_uint8::nlanes) + for(; x <= size.width-VTraits::vlanes(); x+=VTraits::vlanes()) v_store(dptr + x, val0_16); } #endif @@ -355,7 +355,7 @@ public: for (size_t i = 0; i < nstripes; ++i) { // 1D: [1][ ndisp ][1] -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (params.useShorts()) area.allocate(sad_short[i], ndisp + 2); else @@ -363,7 +363,7 @@ public: area.allocate(sad[i], ndisp + 2); // 2D: [ wsz/2 + 1 ][ height ][ wsz/2 + 1 ] * [ ndisp ] -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (params.useShorts()) area.allocate(hsad_short[i], (height + wsz + 2) * ndisp); else @@ -389,7 +389,7 @@ public: } }; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) template static void findStereoCorrespondenceBM_SIMD( const Mat& left, const Mat& right, Mat& disp, Mat& cost, const StereoBMParams& state, @@ -421,8 +421,8 @@ static void findStereoCorrespondenceBM_SIMD( const Mat& left, const Mat& right, short costbuf = 0; int coststep = cost.data ? (int)(cost.step/sizeof(costbuf)) : 0; const uchar * tab = bufX.tab; - short v_seq[v_int16::nlanes]; - for (short i = 0; i < v_int16::nlanes; ++i) + short v_seq[VTraits::max_nlanes]; + for (short i = 0; i < VTraits::vlanes(); ++i) v_seq[i] = i; ushort *sad = bufX.sad_short[bufNum] + 1; @@ -445,19 +445,19 @@ static void findStereoCorrespondenceBM_SIMD( const Mat& left, const Mat& right, { int lval = lptr[0]; v_uint8 lv = vx_setall_u8((uchar)lval); - for( d = 0; d <= ndisp - v_uint8::nlanes; d += v_uint8::nlanes ) + for( d = 0; d <= ndisp - VTraits::vlanes(); d += VTraits::vlanes() ) { v_uint8 diff = v_absdiff(lv, vx_load(rptr + d)); v_store(cbuf + d, diff); - v_store(hsad + d, vx_load(hsad + d) + v_expand_low(diff)); - v_store(hsad + d + v_uint16::nlanes, vx_load(hsad + d + v_uint16::nlanes) + v_expand_high(diff)); + v_store(hsad + d, v_add(vx_load(hsad + d), v_expand_low(diff))); + v_store(hsad + d + VTraits::vlanes(), v_add(vx_load(hsad + d + VTraits::vlanes()), v_expand_high(diff))); } - if( d <= ndisp - v_uint16::nlanes ) + if( d <= ndisp - VTraits::vlanes() ) { v_uint8 diff = v_absdiff(lv, vx_load_low(rptr + d)); v_store_low(cbuf + d, diff); - v_store(hsad + d, vx_load(hsad + d) + v_expand_low(diff)); - d += v_uint16::nlanes; + v_store(hsad + d, v_add(vx_load(hsad + d), v_expand_low(diff))); + d += VTraits::vlanes(); } for( ; d < ndisp; d++ ) { @@ -495,20 +495,20 @@ static void findStereoCorrespondenceBM_SIMD( const Mat& left, const Mat& right, { int lval = lptr[0]; v_uint8 lv = vx_setall_u8((uchar)lval); - for( d = 0; d <= ndisp - v_uint8::nlanes; d += v_uint8::nlanes ) + for( d = 0; d <= ndisp - VTraits::vlanes(); d += VTraits::vlanes() ) { v_uint8 diff = v_absdiff(lv, vx_load(rptr + d)); v_int8 cbs = v_reinterpret_as_s8(vx_load(cbuf_sub + d)); v_store(cbuf + d, diff); - v_store(hsad + d, v_reinterpret_as_u16(v_reinterpret_as_s16(vx_load(hsad + d) + v_expand_low(diff)) - v_expand_low(cbs))); - v_store(hsad + d + v_uint16::nlanes, v_reinterpret_as_u16(v_reinterpret_as_s16(vx_load(hsad + d + v_uint16::nlanes) + v_expand_high(diff)) - v_expand_high(cbs))); + v_store(hsad + d, v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(v_add(vx_load(hsad + d), v_expand_low(diff))), v_expand_low(cbs)))); + v_store(hsad + d + VTraits::vlanes(), v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(v_add(vx_load(hsad + d + VTraits::vlanes()), v_expand_high(diff))), v_expand_high(cbs)))); } - if( d <= ndisp - v_uint16::nlanes) + if( d <= ndisp - VTraits::vlanes()) { v_uint8 diff = v_absdiff(lv, vx_load_low(rptr + d)); v_store_low(cbuf + d, diff); - v_store(hsad + d, v_reinterpret_as_u16(v_reinterpret_as_s16(vx_load(hsad + d) + v_expand_low(diff)) - vx_load_expand((schar*)cbuf_sub + d))); - d += v_uint16::nlanes; + v_store(hsad + d, v_reinterpret_as_u16(v_sub(v_reinterpret_as_s16(v_add(vx_load(hsad + d), v_expand_low(diff))), vx_load_expand((schar *)cbuf_sub + d)))); + d += VTraits::vlanes(); } for( ; d < ndisp; d++ ) { @@ -532,20 +532,20 @@ static void findStereoCorrespondenceBM_SIMD( const Mat& left, const Mat& right, hsad = hsad0 + (1 - dy0)*ndisp; for( y = 1 - dy0; y < wsz2; y++, hsad += ndisp ) { - for( d = 0; d <= ndisp-2*v_uint16::nlanes; d += 2*v_uint16::nlanes ) + for( d = 0; d <= ndisp-2*VTraits::vlanes(); d += 2*VTraits::vlanes() ) { - v_store(sad + d, vx_load(sad + d) + vx_load(hsad + d)); - v_store(sad + d + v_uint16::nlanes, vx_load(sad + d + v_uint16::nlanes) + vx_load(hsad + d + v_uint16::nlanes)); + v_store(sad + d, v_add(vx_load(sad + d), vx_load(hsad + d))); + v_store(sad + d + VTraits::vlanes(), v_add(vx_load(sad + d + VTraits::vlanes()), vx_load(hsad + d + VTraits::vlanes()))); } - if( d <= ndisp-v_uint16::nlanes ) + if( d <= ndisp-VTraits::vlanes() ) { - v_store(sad + d, vx_load(sad + d) + vx_load(hsad + d)); - d += v_uint16::nlanes; + v_store(sad + d, v_add(vx_load(sad + d), vx_load(hsad + d))); + d += VTraits::vlanes(); } - if( d <= ndisp-v_uint16::nlanes/2 ) + if( d <= ndisp-VTraits::vlanes()/2 ) { - v_store_low(sad + d, vx_load_low(sad + d) + vx_load_low(hsad + d)); - d += v_uint16::nlanes/2; + v_store_low(sad + d, v_add(vx_load_low(sad + d), vx_load_low(hsad + d))); + d += VTraits::vlanes()/2; } for( ; d < ndisp; d++ ) sad[d] = sad[d] + hsad[d]; @@ -563,29 +563,29 @@ static void findStereoCorrespondenceBM_SIMD( const Mat& left, const Mat& right, v_int16 minsad8 = vx_setall_s16(SHRT_MAX); v_int16 mind8 = vx_setall_s16(0); - for( d = 0; d <= ndisp - 2*v_int16::nlanes; d += 2*v_int16::nlanes ) + for( d = 0; d <= ndisp - 2*VTraits::vlanes(); d += 2*VTraits::vlanes() ) { - v_int16 sad8 = v_reinterpret_as_s16(vx_load(hsad + d)) - v_reinterpret_as_s16(vx_load(hsad_sub + d)) + v_reinterpret_as_s16(vx_load(sad + d)); + v_int16 sad8 = v_add(v_sub(v_reinterpret_as_s16(vx_load(hsad + d)), v_reinterpret_as_s16(vx_load(hsad_sub + d))), v_reinterpret_as_s16(vx_load(sad + d))); v_store(sad + d, v_reinterpret_as_u16(sad8)); - mind8 = v_max(mind8, (minsad8 > sad8) & vx_setall_s16((short)d)); + mind8 = v_max(mind8, v_and(v_gt(minsad8, sad8), vx_setall_s16((short)d))); minsad8 = v_min(minsad8, sad8); - sad8 = v_reinterpret_as_s16(vx_load(hsad + d + v_int16::nlanes)) - v_reinterpret_as_s16(vx_load(hsad_sub + d + v_int16::nlanes)) + v_reinterpret_as_s16(vx_load(sad + d + v_int16::nlanes)); - v_store(sad + d + v_int16::nlanes, v_reinterpret_as_u16(sad8)); - mind8 = v_max(mind8, (minsad8 > sad8) & vx_setall_s16((short)d+v_int16::nlanes)); + sad8 = v_add(v_sub(v_reinterpret_as_s16(vx_load(hsad + d + VTraits::vlanes())), v_reinterpret_as_s16(vx_load(hsad_sub + d + VTraits::vlanes()))), v_reinterpret_as_s16(vx_load(sad + d + VTraits::vlanes()))); + v_store(sad + d + VTraits::vlanes(), v_reinterpret_as_u16(sad8)); + mind8 = v_max(mind8, v_and(v_gt(minsad8, sad8), vx_setall_s16((short)(d + VTraits::vlanes())))); minsad8 = v_min(minsad8, sad8); } - if( d <= ndisp - v_int16::nlanes ) + if( d <= ndisp - VTraits::vlanes() ) { - v_int16 sad8 = v_reinterpret_as_s16(vx_load(hsad + d)) - v_reinterpret_as_s16(vx_load(hsad_sub + d)) + v_reinterpret_as_s16(vx_load(sad + d)); + v_int16 sad8 = v_add(v_sub(v_reinterpret_as_s16(vx_load(hsad + d)), v_reinterpret_as_s16(vx_load(hsad_sub + d))), v_reinterpret_as_s16(vx_load(sad + d))); v_store(sad + d, v_reinterpret_as_u16(sad8)); - mind8 = v_max(mind8, (minsad8 > sad8) & vx_setall_s16((short)d)); + mind8 = v_max(mind8, v_and(v_gt(minsad8, sad8), vx_setall_s16((short)d))); minsad8 = v_min(minsad8, sad8); - d += v_int16::nlanes; + d += VTraits::vlanes(); } minsad = v_reduce_min(minsad8); - v_int16 v_mask = (vx_setall_s16((short)minsad) == minsad8); - mind = v_reduce_min(((mind8+vx_load(v_seq)) & v_mask) | (vx_setall_s16(SHRT_MAX) & ~v_mask)); + v_int16 v_mask = (v_eq(vx_setall_s16((short)minsad), minsad8)); + mind = v_reduce_min(v_or(v_and(v_add(mind8, vx_load(v_seq)), v_mask), v_and(vx_setall_s16(32767), v_not(v_mask)))); for( ; d < ndisp; d++ ) { int sad8 = (int)(hsad[d]) - hsad_sub[d] + sad[d]; @@ -609,34 +609,34 @@ static void findStereoCorrespondenceBM_SIMD( const Mat& left, const Mat& right, int thresh = minsad + (minsad * uniquenessRatio/100); v_int32 thresh4 = vx_setall_s32(thresh + 1); v_int32 d1 = vx_setall_s32(mind-1), d2 = vx_setall_s32(mind+1); - v_int32 dd_4 = vx_setall_s32(v_int32::nlanes); + v_int32 dd_4 = vx_setall_s32(VTraits::vlanes()); v_int32 d4 = vx_load_expand(v_seq); - for( d = 0; d <= ndisp - v_int16::nlanes; d += v_int16::nlanes ) + for( d = 0; d <= ndisp - VTraits::vlanes(); d += VTraits::vlanes() ) { v_int32 sad4_l, sad4_h; v_expand(v_reinterpret_as_s16(vx_load(sad + d)), sad4_l, sad4_h); - if( v_check_any((thresh4 > sad4_l) & ((d1 > d4) | (d4 > d2))) ) + if( v_check_any(v_and(v_gt(thresh4, sad4_l), v_or(v_gt(d1, d4), v_gt(d4, d2)))) ) break; - d4 += dd_4; - if( v_check_any((thresh4 > sad4_h) & ((d1 > d4) | (d4 > d2))) ) + d4 = v_add(d4, dd_4); + if( v_check_any(v_and(v_gt(thresh4, sad4_h), v_or(v_gt(d1, d4), v_gt(d4, d2)))) ) break; - d4 += dd_4; + d4 = v_add(d4, dd_4); } - if( d <= ndisp - v_int16::nlanes ) + if( d <= ndisp - VTraits::vlanes() ) { dptr[y*dstep] = FILTERED; continue; } - if( d <= ndisp - v_int32::nlanes ) + if( d <= ndisp - VTraits::vlanes() ) { v_int32 sad4_l = vx_load_expand((short*)sad + d); - if (v_check_any((thresh4 > sad4_l) & ((d1 > d4) | (d4 > d2)))) + if (v_check_any(v_and(v_gt(thresh4, sad4_l), v_or(v_gt(d1, d4), v_gt(d4, d2))))) { dptr[y*dstep] = FILTERED; continue; } - d += v_int16::nlanes; + d += VTraits::vlanes(); } for( ; d < ndisp; d++ ) { @@ -698,11 +698,11 @@ findStereoCorrespondenceBM( const Mat& left, const Mat& right, int coststep = cost.data ? (int)(cost.step/sizeof(costbuf)) : 0; const uchar * tab = bufX.tab; -#if CV_SIMD - int v_seq[v_int32::nlanes]; - for (int i = 0; i < v_int32::nlanes; ++i) +#if (CV_SIMD || CV_SIMD_SCALABLE) + int v_seq[VTraits::max_nlanes]; + for (int i = 0; i < VTraits::vlanes(); ++i) v_seq[i] = i; - v_int32 d0_4 = vx_load(v_seq), dd_4 = vx_setall_s32(v_int32::nlanes); + v_int32 d0_4 = vx_load(v_seq), dd_4 = vx_setall_s32(VTraits::vlanes()); #endif int *sad = bufX.sad[bufNum] + 1; @@ -724,17 +724,17 @@ findStereoCorrespondenceBM( const Mat& left, const Mat& right, { int lval = lptr[0]; d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { v_uint8 lv = vx_setall_u8((uchar)lval); - for( ; d <= ndisp - v_uint8::nlanes; d += v_uint8::nlanes ) + for( ; d <= ndisp - VTraits::vlanes(); d += VTraits::vlanes() ) { v_uint8 rv = vx_load(rptr + d); v_int32 hsad_0 = vx_load(hsad + d); - v_int32 hsad_1 = vx_load(hsad + d + v_int32::nlanes); - v_int32 hsad_2 = vx_load(hsad + d + 2*v_int32::nlanes); - v_int32 hsad_3 = vx_load(hsad + d + 3*v_int32::nlanes); + v_int32 hsad_1 = vx_load(hsad + d + VTraits::vlanes()); + v_int32 hsad_2 = vx_load(hsad + d + 2*VTraits::vlanes()); + v_int32 hsad_3 = vx_load(hsad + d + 3*VTraits::vlanes()); v_uint8 diff = v_absdiff(lv, rv); v_store(cbuf + d, diff); @@ -744,15 +744,15 @@ findStereoCorrespondenceBM( const Mat& left, const Mat& right, v_expand(diff0, diff00, diff01); v_expand(diff1, diff10, diff11); - hsad_0 += v_reinterpret_as_s32(diff00); - hsad_1 += v_reinterpret_as_s32(diff01); - hsad_2 += v_reinterpret_as_s32(diff10); - hsad_3 += v_reinterpret_as_s32(diff11); + hsad_0 = v_add(hsad_0, v_reinterpret_as_s32(diff00)); + hsad_1 = v_add(hsad_1, v_reinterpret_as_s32(diff01)); + hsad_2 = v_add(hsad_2, v_reinterpret_as_s32(diff10)); + hsad_3 = v_add(hsad_3, v_reinterpret_as_s32(diff11)); v_store(hsad + d, hsad_0); - v_store(hsad + d + v_int32::nlanes, hsad_1); - v_store(hsad + d + 2*v_int32::nlanes, hsad_2); - v_store(hsad + d + 3*v_int32::nlanes, hsad_3); + v_store(hsad + d + VTraits::vlanes(), hsad_1); + v_store(hsad + d + 2*VTraits::vlanes(), hsad_2); + v_store(hsad + d + 3*VTraits::vlanes(), hsad_3); } } #endif @@ -792,16 +792,16 @@ findStereoCorrespondenceBM( const Mat& left, const Mat& right, { int lval = lptr[0]; d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { v_uint8 lv = vx_setall_u8((uchar)lval); - for( ; d <= ndisp - v_uint8::nlanes; d += v_uint8::nlanes ) + for( ; d <= ndisp - VTraits::vlanes(); d += VTraits::vlanes() ) { v_uint8 rv = vx_load(rptr + d); v_int32 hsad_0 = vx_load(hsad + d); - v_int32 hsad_1 = vx_load(hsad + d + v_int32::nlanes); - v_int32 hsad_2 = vx_load(hsad + d + 2*v_int32::nlanes); - v_int32 hsad_3 = vx_load(hsad + d + 3*v_int32::nlanes); + v_int32 hsad_1 = vx_load(hsad + d + VTraits::vlanes()); + v_int32 hsad_2 = vx_load(hsad + d + 2*VTraits::vlanes()); + v_int32 hsad_3 = vx_load(hsad + d + 3*VTraits::vlanes()); v_uint8 cbs = vx_load(cbuf_sub + d); v_uint8 diff = v_absdiff(lv, rv); v_store(cbuf + d, diff); @@ -815,19 +815,19 @@ findStereoCorrespondenceBM( const Mat& left, const Mat& right, v_expand(v_reinterpret_as_s16(cbs0), cbs00, cbs01); v_expand(v_reinterpret_as_s16(cbs1), cbs10, cbs11); - v_int32 diff_0 = diff00 - cbs00; - v_int32 diff_1 = diff01 - cbs01; - v_int32 diff_2 = diff10 - cbs10; - v_int32 diff_3 = diff11 - cbs11; - hsad_0 += diff_0; - hsad_1 += diff_1; - hsad_2 += diff_2; - hsad_3 += diff_3; + v_int32 diff_0 = v_sub(diff00, cbs00); + v_int32 diff_1 = v_sub(diff01, cbs01); + v_int32 diff_2 = v_sub(diff10, cbs10); + v_int32 diff_3 = v_sub(diff11, cbs11); + hsad_0 = v_add(hsad_0, diff_0); + hsad_1 = v_add(hsad_1, diff_1); + hsad_2 = v_add(hsad_2, diff_2); + hsad_3 = v_add(hsad_3, diff_3); v_store(hsad + d, hsad_0); - v_store(hsad + d + v_int32::nlanes, hsad_1); - v_store(hsad + d + 2*v_int32::nlanes, hsad_2); - v_store(hsad + d + 3*v_int32::nlanes, hsad_3); + v_store(hsad + d + VTraits::vlanes(), hsad_1); + v_store(hsad + d + 2*VTraits::vlanes(), hsad_2); + v_store(hsad + d + 3*VTraits::vlanes(), hsad_3); } } #endif @@ -854,18 +854,18 @@ findStereoCorrespondenceBM( const Mat& left, const Mat& right, for( y = 1 - dy0; y < wsz2; y++, hsad += ndisp ) { d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { - for( d = 0; d <= ndisp-2*v_int32::nlanes; d += 2*v_int32::nlanes ) + for( d = 0; d <= ndisp-2*VTraits::vlanes(); d += 2*VTraits::vlanes() ) { v_int32 s0 = vx_load(sad + d); - v_int32 s1 = vx_load(sad + d + v_int32::nlanes); + v_int32 s1 = vx_load(sad + d + VTraits::vlanes()); v_int32 t0 = vx_load(hsad + d); - v_int32 t1 = vx_load(hsad + d + v_int32::nlanes); - s0 += t0; - s1 += t1; + v_int32 t1 = vx_load(hsad + d + VTraits::vlanes()); + s0 = v_add(s0, t0); + s1 = v_add(s1, t1); v_store(sad + d, s0); - v_store(sad + d + v_int32::nlanes, s1); + v_store(sad + d + VTraits::vlanes(), s1); } } #endif @@ -883,30 +883,30 @@ findStereoCorrespondenceBM( const Mat& left, const Mat& right, hsad = hsad0 + MIN(y + wsz2, height+dy1-1)*ndisp; hsad_sub = hsad0 + MAX(y - wsz2 - 1, -dy0)*ndisp; d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) { v_int32 minsad4 = vx_setall_s32(INT_MAX); v_int32 mind4 = vx_setall_s32(0), d4 = d0_4; - for( ; d <= ndisp - 2*v_int32::nlanes; d += 2*v_int32::nlanes ) + for( ; d <= ndisp - 2*VTraits::vlanes(); d += 2*VTraits::vlanes() ) { - v_int32 sad4 = vx_load(sad + d) + vx_load(hsad + d) - vx_load(hsad_sub + d); + v_int32 sad4 = v_sub(v_add(vx_load(sad + d), vx_load(hsad + d)), vx_load(hsad_sub + d)); v_store(sad + d, sad4); - mind4 = v_select(minsad4 > sad4, d4, mind4); + mind4 = v_select(v_gt(minsad4, sad4), d4, mind4); minsad4 = v_min(minsad4, sad4); - d4 += dd_4; + d4 = v_add(d4, dd_4); - sad4 = vx_load(sad + d + v_int32::nlanes) + vx_load(hsad + d + v_int32::nlanes) - vx_load(hsad_sub + d + v_int32::nlanes); - v_store(sad + d + v_int32::nlanes, sad4); - mind4 = v_select(minsad4 > sad4, d4, mind4); + sad4 = v_sub(v_add(vx_load(sad + d + VTraits::vlanes()), vx_load(hsad + d + VTraits::vlanes())), vx_load(hsad_sub + d + VTraits::vlanes())); + v_store(sad + d + VTraits::vlanes(), sad4); + mind4 = v_select(v_gt(minsad4, sad4), d4, mind4); minsad4 = v_min(minsad4, sad4); - d4 += dd_4; + d4 = v_add(d4, dd_4); } - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) minsad_buf[v_int32::nlanes], mind_buf[v_int32::nlanes]; + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) minsad_buf[VTraits::max_nlanes], mind_buf[VTraits::max_nlanes]; v_store(minsad_buf, minsad4); v_store(mind_buf, mind4); - for (int i = 0; i < v_int32::nlanes; ++i) + for (int i = 0; i < VTraits::vlanes(); ++i) if(minsad_buf[i] < minsad || (minsad == minsad_buf[i] && mind_buf[i] < mind)) { minsad = minsad_buf[i]; mind = mind_buf[i]; } } #endif @@ -1101,7 +1101,7 @@ struct FindStereoCorrespInvoker : public ParallelLoopBody Mat disp_i = disp->rowRange(row0, row1); Mat cost_i = state.disp12MaxDiff >= 0 ? cost->rowRange(row0, row1) : Mat(); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) if (state.useShorts()) { if( disp_i.type() == CV_16S) diff --git a/modules/stereo/src/stereosgbm.cpp b/modules/stereo/src/stereosgbm.cpp index affa2c6749..5423efb640 100644 --- a/modules/stereo/src/stereosgbm.cpp +++ b/modules/stereo/src/stereosgbm.cpp @@ -122,7 +122,7 @@ struct StereoSGBMParams int mode; }; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) #if CV_SIMD_WIDTH == 16 static inline v_int16 vx_setseq_s16() { return v_int16(0, 1, 2, 3, 4, 5, 6, 7); } @@ -135,10 +135,10 @@ static inline v_int16 vx_setseq_s16() #else struct vseq_s16 { - short data[v_int16::nlanes]; + short data[VTraits::max_nlanes]; vseq_s16() { - for (int i = 0; i < v_int16::nlanes; i++) + for (int i = 0; i < VTraits::vlanes(); i++) data[i] = i; } }; @@ -152,8 +152,8 @@ static inline v_int16 vx_setseq_s16() static inline void min_pos(const v_int16& val, const v_int16& pos, short &min_val, short &min_pos) { min_val = v_reduce_min(val); - v_int16 v_mask = (vx_setall_s16(min_val) == val); - min_pos = v_reduce_min(((pos+vx_setseq_s16()) & v_mask) | (vx_setall_s16(SHRT_MAX) & ~v_mask)); + v_int16 v_mask = (v_eq(vx_setall_s16(min_val), val)); + min_pos = v_reduce_min(v_or(v_and(v_add(pos, vx_setseq_s16()), v_mask), v_and(vx_setall_s16(SHRT_MAX), v_not(v_mask)))); } #endif @@ -269,26 +269,26 @@ static void calcPixelCostBT( const Mat& img1, const Mat& img2, int y, int u1 = std::max(ul, ur); u1 = std::max(u1, u); int d = minD; - #if CV_SIMD + #if (CV_SIMD || CV_SIMD_SCALABLE) v_uint8 _u = vx_setall_u8((uchar)u), _u0 = vx_setall_u8((uchar)u0); v_uint8 _u1 = vx_setall_u8((uchar)u1); - for( ; d <= maxD - 2*v_int16::nlanes; d += 2*v_int16::nlanes ) + for( ; d <= maxD - 2*VTraits::vlanes(); d += 2*VTraits::vlanes() ) { v_uint8 _v = vx_load(prow2 + width-x-1 + d); v_uint8 _v0 = vx_load(buffer + width-x-1 + d); v_uint8 _v1 = vx_load(buffer + width-x-1 + d + width2); - v_uint8 c0 = v_max(_u - _v1, _v0 - _u); - v_uint8 c1 = v_max(_v - _u1, _u0 - _v); + v_uint8 c0 = v_max(v_sub(_u, _v1), v_sub(_v0, _u)); + v_uint8 c1 = v_max(v_sub(_v, _u1), v_sub(_u0, _v)); v_uint8 diff = v_min(c0, c1); v_int16 _c0 = vx_load_aligned(cost + x*D + d); - v_int16 _c1 = vx_load_aligned(cost + x*D + d + v_int16::nlanes); + v_int16 _c1 = vx_load_aligned(cost + x*D + d + VTraits::vlanes()); v_uint16 diff1,diff2; v_expand(diff,diff1,diff2); - v_store_aligned(cost + x*D + d, _c0 + v_reinterpret_as_s16(diff1 >> diff_scale)); - v_store_aligned(cost + x*D + d + v_int16::nlanes, _c1 + v_reinterpret_as_s16(diff2 >> diff_scale)); + v_store_aligned(cost + x*D + d, v_add(_c0, v_reinterpret_as_s16(v_shr(diff1, diff_scale)))); + v_store_aligned(cost + x*D + d + VTraits::vlanes(), v_add(_c1, v_reinterpret_as_s16(v_shr(diff2, diff_scale)))); } #endif for( ; d < maxD; d++ ) @@ -554,13 +554,13 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, calcPixelCostBT( img1, img2, k, minD, maxD, mem.pixDiff, mem.tempBuf, mem.getClipTab() ); memset(hsumAdd, 0, Da*sizeof(CostType)); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 h_scale = vx_setall_s16((short)SW2 + 1); - for( d = 0; d < Da; d += v_int16::nlanes ) + for( d = 0; d < Da; d += VTraits::vlanes() ) { - v_int16 v_hsumAdd = vx_load_aligned(mem.pixDiff + d) * h_scale; + v_int16 v_hsumAdd = v_mul(vx_load_aligned(mem.pixDiff + d), h_scale); for( x = Da; x <= SW2*Da; x += Da ) - v_hsumAdd += vx_load_aligned(mem.pixDiff + x + d); + v_hsumAdd = v_add(v_hsumAdd, vx_load_aligned(mem.pixDiff + x + d)); v_store_aligned(hsumAdd + d, v_hsumAdd); } #else @@ -577,9 +577,9 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, const CostType* hsumSub = mem.getHSumBuf(std::max(y - SH2 - 1, 0)); const CostType* Cprev = mem.getCBuf(y - 1); -#if CV_SIMD - for (d = 0; d < Da; d += v_int16::nlanes) - v_store_aligned(C + d, vx_load_aligned(Cprev + d) + vx_load_aligned(hsumAdd + d) - vx_load_aligned(hsumSub + d)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (d = 0; d < Da; d += VTraits::vlanes()) + v_store_aligned(C + d, v_sub(v_add(vx_load_aligned(Cprev + d), vx_load_aligned(hsumAdd + d)), vx_load_aligned(hsumSub + d))); #else for (d = 0; d < D; d++) C[d] = (CostType)(Cprev[d] + hsumAdd[d] - hsumSub[d]); @@ -589,12 +589,12 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, { const CostType* pixAdd = mem.pixDiff + std::min(x + SW2*Da, (width1-1)*Da); const CostType* pixSub = mem.pixDiff + std::max(x - (SW2+1)*Da, 0); -#if CV_SIMD - for( d = 0; d < Da; d += v_int16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( d = 0; d < Da; d += VTraits::vlanes() ) { - v_int16 hv = vx_load_aligned(hsumAdd + x - Da + d) - vx_load_aligned(pixSub + d) + vx_load_aligned(pixAdd + d); + v_int16 hv = v_add(v_sub(vx_load_aligned(hsumAdd + x - Da + d), vx_load_aligned(pixSub + d)), vx_load_aligned(pixAdd + d)); v_store_aligned(hsumAdd + x + d, hv); - v_store_aligned(C + x + d, vx_load_aligned(Cprev + x + d) - vx_load_aligned(hsumSub + x + d) + hv); + v_store_aligned(C + x + d, v_add(v_sub(vx_load_aligned(Cprev + x + d), vx_load_aligned(hsumSub + x + d)), hv)); } #else for( d = 0; d < D; d++ ) @@ -607,10 +607,10 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, } else { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 v_scale = vx_setall_s16(k == 0 ? (short)SH2 + 1 : 1); - for (d = 0; d < Da; d += v_int16::nlanes) - v_store_aligned(C + d, vx_load_aligned(C + d) + vx_load_aligned(hsumAdd + d) * v_scale); + for (d = 0; d < Da; d += VTraits::vlanes()) + v_store_aligned(C + d, v_add(vx_load_aligned(C + d), v_mul(vx_load_aligned(hsumAdd + d), v_scale))); #else int scale = k == 0 ? SH2 + 1 : 1; for (d = 0; d < D; d++) @@ -621,12 +621,12 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, const CostType* pixAdd = mem.pixDiff + std::min(x + SW2*Da, (width1-1)*Da); const CostType* pixSub = mem.pixDiff + std::max(x - (SW2+1)*Da, 0); -#if CV_SIMD - for (d = 0; d < Da; d += v_int16::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (d = 0; d < Da; d += VTraits::vlanes()) { - v_int16 hv = vx_load_aligned(hsumAdd + x - Da + d) + vx_load_aligned(pixAdd + d) - vx_load_aligned(pixSub + d); + v_int16 hv = v_sub(v_add(vx_load_aligned(hsumAdd + x - Da + d), vx_load_aligned(pixAdd + d)), vx_load_aligned(pixSub + d)); v_store_aligned(hsumAdd + x + d, hv); - v_store_aligned(C + x + d, vx_load_aligned(C + x + d) + hv * v_scale); + v_store_aligned(C + x + d, v_add(vx_load_aligned(C + x + d), v_mul(hv, v_scale))); } #else for( d = 0; d < D; d++ ) @@ -645,9 +645,9 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, { const CostType* hsumSub = mem.getHSumBuf(std::max(y - SH2 - 1, 0)); const CostType* Cprev = mem.getCBuf(y - 1); -#if CV_SIMD - for (x = 0; x < width1*Da; x += v_int16::nlanes) - v_store_aligned(C + x, vx_load_aligned(Cprev + x) - vx_load_aligned(hsumSub + x) + vx_load_aligned(hsumAdd + x)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (x = 0; x < width1*Da; x += VTraits::vlanes()) + v_store_aligned(C + x, v_add(v_sub(vx_load_aligned(Cprev + x), vx_load_aligned(hsumSub + x)), vx_load_aligned(hsumAdd + x))); #else for (x = 0; x < width1*Da; x++) C[x] = (CostType)(Cprev[x] + hsumAdd[x] - hsumSub[x]); @@ -655,9 +655,9 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, } else { -#if CV_SIMD - for (x = 0; x < width1*Da; x += v_int16::nlanes) - v_store_aligned(C + x, vx_load_aligned(C + x) + vx_load_aligned(hsumAdd + x)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (x = 0; x < width1*Da; x += VTraits::vlanes()) + v_store_aligned(C + x, v_add(vx_load_aligned(C + x), vx_load_aligned(hsumAdd + x))); #else for (x = 0; x < width1*Da; x++) C[x] = (CostType)(C[x] + hsumAdd[x]); @@ -713,7 +713,7 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, CostType* minL = mem.getMinLr(lrID, x); d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 _P1 = vx_setall_s16((short)P1); v_int16 _delta0 = vx_setall_s16((short)delta0); @@ -725,31 +725,31 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, v_int16 _minL2 = vx_setall_s16((short)MAX_COST); v_int16 _minL3 = vx_setall_s16((short)MAX_COST); - for( ; d <= D - v_int16::nlanes; d += v_int16::nlanes ) + for( ; d <= D - VTraits::vlanes(); d += VTraits::vlanes() ) { v_int16 Cpd = vx_load_aligned(Cp + d); v_int16 Spd = vx_load_aligned(Sp + d); v_int16 L; - L = v_min(v_min(v_min(vx_load_aligned(Lr_p0 + d), vx_load(Lr_p0 + d - 1) + _P1), vx_load(Lr_p0 + d + 1) + _P1), _delta0) - _delta0 + Cpd; + L = v_add(v_sub(v_min(v_min(v_min(vx_load_aligned(Lr_p0 + d), v_add(vx_load(Lr_p0 + d - 1), _P1)), v_add(vx_load(Lr_p0 + d + 1), _P1)), _delta0), _delta0), Cpd); v_store_aligned(Lr_p + d, L); _minL0 = v_min(_minL0, L); - Spd += L; + Spd = v_add(Spd, L); - L = v_min(v_min(v_min(vx_load_aligned(Lr_p1 + d), vx_load(Lr_p1 + d - 1) + _P1), vx_load(Lr_p1 + d + 1) + _P1), _delta1) - _delta1 + Cpd; + L = v_add(v_sub(v_min(v_min(v_min(vx_load_aligned(Lr_p1 + d), v_add(vx_load(Lr_p1 + d - 1), _P1)), v_add(vx_load(Lr_p1 + d + 1), _P1)), _delta1), _delta1), Cpd); v_store_aligned(Lr_p + d + Dlra, L); _minL1 = v_min(_minL1, L); - Spd += L; + Spd = v_add(Spd, L); - L = v_min(v_min(v_min(vx_load_aligned(Lr_p2 + d), vx_load(Lr_p2 + d - 1) + _P1), vx_load(Lr_p2 + d + 1) + _P1), _delta2) - _delta2 + Cpd; + L = v_add(v_sub(v_min(v_min(v_min(vx_load_aligned(Lr_p2 + d), v_add(vx_load(Lr_p2 + d - 1), _P1)), v_add(vx_load(Lr_p2 + d + 1), _P1)), _delta2), _delta2), Cpd); v_store_aligned(Lr_p + d + Dlra*2, L); _minL2 = v_min(_minL2, L); - Spd += L; + Spd = v_add(Spd, L); - L = v_min(v_min(v_min(vx_load_aligned(Lr_p3 + d), vx_load(Lr_p3 + d - 1) + _P1), vx_load(Lr_p3 + d + 1) + _P1), _delta3) - _delta3 + Cpd; + L = v_add(v_sub(v_min(v_min(v_min(vx_load_aligned(Lr_p3 + d), v_add(vx_load(Lr_p3 + d - 1), _P1)), v_add(vx_load(Lr_p3 + d + 1), _P1)), _delta3), _delta3), Cpd); v_store_aligned(Lr_p + d + Dlra*3, L); _minL3 = v_min(_minL3, L); - Spd += L; + Spd = v_add(Spd, L); v_store_aligned(Sp + d, Spd); } @@ -768,7 +768,7 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, t0 = v_min(t0, t1); t0 = v_min(t0, v_rotate_right<4>(t0)); #if CV_SIMD_WIDTH == 32 - CostType buf[v_int16::nlanes]; + CostType buf[VTraits::max_nlanes]; v_store_low(buf, v_min(t0, v_rotate_right<8>(t0))); minL[0] = buf[0]; minL[1] = buf[1]; @@ -816,10 +816,10 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, if( pass == npasses ) { x = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 v_inv_dist = vx_setall_s16((DispType)INVALID_DISP_SCALED); v_int16 v_max_cost = vx_setall_s16(MAX_COST); - for( ; x <= width - v_int16::nlanes; x += v_int16::nlanes ) + for( ; x <= width - VTraits::vlanes(); x += VTraits::vlanes() ) { v_store(disp1ptr + x, v_inv_dist); v_store(mem.disp2ptr + x, v_inv_dist); @@ -849,23 +849,23 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, d = 0; int delta0 = P2 + *mem.getMinLr(lrID, x + 1); int minL0 = MAX_COST; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 _P1 = vx_setall_s16((short)P1); v_int16 _delta0 = vx_setall_s16((short)delta0); v_int16 _minL0 = vx_setall_s16((short)MAX_COST); v_int16 _minS = vx_setall_s16(MAX_COST), _bestDisp = vx_setall_s16(-1); - for( ; d <= D - v_int16::nlanes; d += v_int16::nlanes ) + for( ; d <= D - VTraits::vlanes(); d += VTraits::vlanes() ) { v_int16 Cpd = vx_load_aligned(Cp + d); - v_int16 L0 = v_min(v_min(v_min(vx_load_aligned(Lr_p0 + d), vx_load(Lr_p0 + d - 1) + _P1), vx_load(Lr_p0 + d + 1) + _P1), _delta0) - _delta0 + Cpd; + v_int16 L0 = v_add(v_sub(v_min(v_min(v_min(vx_load_aligned(Lr_p0 + d), v_add(vx_load(Lr_p0 + d - 1), _P1)), v_add(vx_load(Lr_p0 + d + 1), _P1)), _delta0), _delta0), Cpd); v_store_aligned(Lr_p + d, L0); _minL0 = v_min(_minL0, L0); - L0 += vx_load_aligned(Sp + d); + L0 = v_add(L0, vx_load_aligned(Sp + d)); v_store_aligned(Sp + d, L0); - _bestDisp = v_select(_minS > L0, vx_setall_s16((short)d), _bestDisp); + _bestDisp = v_select(v_gt(_minS, L0), vx_setall_s16((short)d), _bestDisp); _minS = v_min(_minS, L0); } minL0 = (CostType)v_reduce_min(_minL0); @@ -890,12 +890,12 @@ static void computeDisparitySGBM( const Mat& img1, const Mat& img2, else { d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 _minS = vx_setall_s16(MAX_COST), _bestDisp = vx_setall_s16(-1); - for( ; d <= D - v_int16::nlanes; d+= v_int16::nlanes ) + for( ; d <= D - VTraits::vlanes(); d+= VTraits::vlanes() ) { v_int16 L0 = vx_load_aligned(Sp + d); - _bestDisp = v_select(_minS > L0, vx_setall_s16((short)d), _bestDisp); + _bestDisp = v_select(v_gt(_minS, L0), vx_setall_s16((short)d), _bestDisp); _minS = v_min( L0, _minS ); } min_pos(_minS, _bestDisp, minS, bestDisp); @@ -1038,9 +1038,9 @@ struct CalcVerticalSums: public ParallelLoopBody for( x = (x1 - SW2)*Da; x <= (x1 + SW2)*Da; x += Da ) { int xbord = x <= 0 ? 0 : (x > (width1 - 1)*Da ? (width1 - 1)*Da : x); -#if CV_SIMD - for( d = 0; d < Da; d += v_int16::nlanes ) - v_store_aligned(hsumAdd + x1*Da + d, vx_load_aligned(hsumAdd + x1*Da + d) + vx_load_aligned(pixDiff + xbord + d)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( d = 0; d < Da; d += VTraits::vlanes() ) + v_store_aligned(hsumAdd + x1*Da + d, v_add(vx_load_aligned(hsumAdd + x1 * this->Da + d), vx_load_aligned(pixDiff + xbord + d))); #else for( d = 0; d < D; d++ ) hsumAdd[x1*Da + d] = (CostType)(hsumAdd[x1*Da + d] + pixDiff[xbord + d]); @@ -1051,9 +1051,9 @@ struct CalcVerticalSums: public ParallelLoopBody { const CostType* hsumSub = mem.getHSumBuf(std::max(y - SH2 - 1, 0)); const CostType* Cprev = mem.getCBuf(y - 1); -#if CV_SIMD - for( d = 0; d < Da; d += v_int16::nlanes ) - v_store_aligned(C + x1*Da + d, vx_load_aligned(Cprev + x1*Da + d) + vx_load_aligned(hsumAdd + x1*Da + d) - vx_load_aligned(hsumSub + x1*Da + d)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( d = 0; d < Da; d += VTraits::vlanes() ) + v_store_aligned(C + x1*Da + d, v_sub(v_add(vx_load_aligned(Cprev + x1 * this->Da + d), vx_load_aligned(hsumAdd + x1 * this->Da + d)), vx_load_aligned(hsumSub + x1 * this->Da + d))); #else for( d = 0; d < D; d++ ) C[x1*Da + d] = (CostType)(Cprev[x1*Da + d] + hsumAdd[x1*Da + d] - hsumSub[x1*Da + d]); @@ -1063,12 +1063,12 @@ struct CalcVerticalSums: public ParallelLoopBody const CostType* pixAdd = pixDiff + std::min(x + SW2*Da, (width1-1)*Da); const CostType* pixSub = pixDiff + std::max(x - (SW2+1)*Da, 0); -#if CV_SIMD - for( d = 0; d < Da; d += v_int16::nlanes ) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( d = 0; d < Da; d += VTraits::vlanes() ) { - v_int16 hv = vx_load_aligned(hsumAdd + x - Da + d) - vx_load_aligned(pixSub + d) + vx_load_aligned(pixAdd + d); + v_int16 hv = v_add(v_sub(vx_load_aligned(hsumAdd + x - this->Da + d), vx_load_aligned(pixSub + d)), vx_load_aligned(pixAdd + d)); v_store_aligned(hsumAdd + x + d, hv); - v_store_aligned(C + x + d, vx_load_aligned(Cprev + x + d) - vx_load_aligned(hsumSub + x + d) + hv); + v_store_aligned(C + x + d, v_add(v_sub(vx_load_aligned(Cprev + x + d), vx_load_aligned(hsumSub + x + d)), hv)); } #else for( d = 0; d < D; d++ ) @@ -1081,10 +1081,10 @@ struct CalcVerticalSums: public ParallelLoopBody } else { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 v_scale = vx_setall_s16(k == 0 ? (short)SH2 + 1 : 1); - for (d = 0; d < Da; d += v_int16::nlanes) - v_store_aligned(C + x1*Da + d, vx_load_aligned(C + x1*Da + d) + vx_load_aligned(hsumAdd + x1*Da + d) * v_scale); + for (d = 0; d < Da; d += VTraits::vlanes()) + v_store_aligned(C + x1*Da + d, v_add(vx_load_aligned(C + x1 * this->Da + d), v_mul(vx_load_aligned(hsumAdd + x1 * this->Da + d), v_scale))); #else int scale = k == 0 ? SH2 + 1 : 1; for (d = 0; d < D; d++) @@ -1094,12 +1094,12 @@ struct CalcVerticalSums: public ParallelLoopBody { const CostType* pixAdd = pixDiff + std::min(x + SW2*Da, (width1-1)*Da); const CostType* pixSub = pixDiff + std::max(x - (SW2+1)*Da, 0); -#if CV_SIMD - for (d = 0; d < Da; d += v_int16::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (d = 0; d < Da; d += VTraits::vlanes()) { - v_int16 hv = vx_load_aligned(hsumAdd + x - Da + d) + vx_load_aligned(pixAdd + d) - vx_load_aligned(pixSub + d); + v_int16 hv = v_sub(v_add(vx_load_aligned(hsumAdd + x - this->Da + d), vx_load_aligned(pixAdd + d)), vx_load_aligned(pixSub + d)); v_store_aligned(hsumAdd + x + d, hv); - v_store_aligned(C + x + d, vx_load_aligned(C + x + d) + hv * v_scale); + v_store_aligned(C + x + d, v_add(vx_load_aligned(C + x + d), v_mul(hv, v_scale))); } #else for( d = 0; d < D; d++ ) @@ -1119,9 +1119,9 @@ struct CalcVerticalSums: public ParallelLoopBody const CostType* hsumSub = mem.getHSumBuf(std::max(y - SH2 - 1, 0)); const CostType* Cprev = mem.getCBuf(y - 1); -#if CV_SIMD - for( x = x1*Da; x < x2*Da; x += v_int16::nlanes ) - v_store_aligned(C + x, vx_load_aligned(Cprev + x) - vx_load_aligned(hsumSub + x) + vx_load_aligned(hsumAdd + x)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( x = x1*Da; x < x2*Da; x += VTraits::vlanes() ) + v_store_aligned(C + x, v_add(v_sub(vx_load_aligned(Cprev + x), vx_load_aligned(hsumSub + x)), vx_load_aligned(hsumAdd + x))); #else for( x = x1*Da; x < x2*Da; x++ ) C[x] = (CostType)(Cprev[x] + hsumAdd[x] - hsumSub[x]); @@ -1130,9 +1130,9 @@ struct CalcVerticalSums: public ParallelLoopBody else*/ if(y == 0) { -#if CV_SIMD - for( x = x1*Da; x < x2*Da; x += v_int16::nlanes ) - v_store_aligned(C + x, vx_load_aligned(C + x) + vx_load_aligned(hsumAdd + x)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( x = x1*Da; x < x2*Da; x += VTraits::vlanes() ) + v_store_aligned(C + x, v_add(vx_load_aligned(C + x), vx_load_aligned(hsumAdd + x))); #else for( x = x1*Da; x < x2*Da; x++ ) C[x] = (CostType)(C[x] + hsumAdd[x]); @@ -1166,19 +1166,19 @@ struct CalcVerticalSums: public ParallelLoopBody CostType& minL = *(mem.getMinLr(lrID, x)); d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 _P1 = vx_setall_s16((short)P1); v_int16 _delta = vx_setall_s16((short)delta); v_int16 _minL = vx_setall_s16((short)MAX_COST); - for( ; d <= D - v_int16::nlanes; d += v_int16::nlanes ) + for( ; d <= D - VTraits::vlanes(); d += VTraits::vlanes() ) { v_int16 Cpd = vx_load_aligned(Cp + d); - v_int16 L = v_min(v_min(v_min(vx_load_aligned(Lr_ppr + d), vx_load(Lr_ppr + d - 1) + _P1), vx_load(Lr_ppr + d + 1) + _P1), _delta) - _delta + Cpd; + v_int16 L = v_add(v_sub(v_min(v_min(v_min(vx_load_aligned(Lr_ppr + d), v_add(vx_load(Lr_ppr + d - 1), _P1)), v_add(vx_load(Lr_ppr + d + 1), _P1)), _delta), _delta), Cpd); v_store_aligned(Lr_p + d, L); _minL = v_min(_minL, L); - v_store_aligned(Sp + d, vx_load_aligned(Sp + d) + L); + v_store_aligned(Sp + d, v_add(vx_load_aligned(Sp + d), L)); } minL = v_reduce_min(_minL); #else @@ -1263,10 +1263,10 @@ struct CalcHorizontalSums: public ParallelLoopBody CostType* S = mem.getSBuf(y); x = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 v_inv_dist = vx_setall_s16((DispType)INVALID_DISP_SCALED); v_int16 v_max_cost = vx_setall_s16(MAX_COST); - for (; x <= width - v_int16::nlanes; x += v_int16::nlanes) + for (; x <= width - VTraits::vlanes(); x += VTraits::vlanes()) { v_store(disp1ptr + x, v_inv_dist); v_store(disp2ptr + x, v_inv_dist); @@ -1303,19 +1303,19 @@ struct CalcHorizontalSums: public ParallelLoopBody CostType* Sp = S + x*Da; d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 _P1 = vx_setall_s16((short)P1); v_int16 _delta = vx_setall_s16((short)delta); v_int16 _minL = vx_setall_s16((short)MAX_COST); - for( ; d <= D - v_int16::nlanes; d += v_int16::nlanes) + for( ; d <= D - VTraits::vlanes(); d += VTraits::vlanes()) { v_int16 Cpd = vx_load_aligned(Cp + d); - v_int16 L = v_min(v_min(v_min(vx_load(Lr_ppr + d), vx_load(Lr_ppr + d - 1) + _P1), vx_load(Lr_ppr + d + 1) + _P1), _delta) - _delta + Cpd; + v_int16 L = v_add(v_sub(v_min(v_min(v_min(vx_load(Lr_ppr + d), v_add(vx_load(Lr_ppr + d - 1), _P1)), v_add(vx_load(Lr_ppr + d + 1), _P1)), _delta), _delta), Cpd); v_store(Lr_p + d, L); _minL = v_min(_minL, L); - v_store_aligned(Sp + d, vx_load_aligned(Sp + d) + L); + v_store_aligned(Sp + d, v_add(vx_load_aligned(Sp + d), L)); } minLr = v_reduce_min(_minL); #else @@ -1348,22 +1348,22 @@ struct CalcHorizontalSums: public ParallelLoopBody minLr = MAX_COST; d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 _P1 = vx_setall_s16((short)P1); v_int16 _delta = vx_setall_s16((short)delta); v_int16 _minL = vx_setall_s16((short)MAX_COST); v_int16 _minS = vx_setall_s16(MAX_COST), _bestDisp = vx_setall_s16(-1); - for( ; d <= D - v_int16::nlanes; d += v_int16::nlanes ) + for( ; d <= D - VTraits::vlanes(); d += VTraits::vlanes() ) { v_int16 Cpd = vx_load_aligned(Cp + d); - v_int16 L = v_min(v_min(v_min(vx_load(Lr_ppr + d), vx_load(Lr_ppr + d - 1) + _P1), vx_load(Lr_ppr + d + 1) + _P1), _delta) - _delta + Cpd; + v_int16 L = v_add(v_sub(v_min(v_min(v_min(vx_load(Lr_ppr + d), v_add(vx_load(Lr_ppr + d - 1), _P1)), v_add(vx_load(Lr_ppr + d + 1), _P1)), _delta), _delta), Cpd); v_store(Lr_p + d, L); _minL = v_min(_minL, L); - L += vx_load_aligned(Sp + d); + L = v_add(L, vx_load_aligned(Sp + d)); v_store_aligned(Sp + d, L); - _bestDisp = v_select(_minS > L, vx_setall_s16((short)d), _bestDisp); + _bestDisp = v_select(v_gt(_minS, L), vx_setall_s16((short)d), _bestDisp); _minS = v_min( L, _minS ); } minLr = v_reduce_min(_minL); @@ -1580,8 +1580,8 @@ struct SGBM3WayMainLoop : public ParallelLoopBody utils::BufferArea aux_area; PixType* clipTab; -#if CV_SIMD - short idx_row[v_int16::nlanes]; +#if (CV_SIMD || CV_SIMD_SCALABLE) + short idx_row[VTraits::max_nlanes]; #endif SGBM3WayMainLoop(const Mat& _img1, const Mat& _img2, Mat* _dst_disp, const StereoSGBMParams& params, int stripe_size, int _stripe_overlap); void operator () (const Range& range) const CV_OVERRIDE; @@ -1636,8 +1636,8 @@ SGBM3WayMainLoop::SGBM3WayMainLoop(const Mat& _img1, uniquenessRatio = params.uniquenessRatio >= 0 ? params.uniquenessRatio : 10; disp12MaxDiff = params.disp12MaxDiff > 0 ? params.disp12MaxDiff : 1; -#if CV_SIMD - for(short i = 0; i < v_int16::nlanes; ++i) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for(short i = 0; i < VTraits::vlanes(); ++i) idx_row[i] = i; #endif } @@ -1658,13 +1658,13 @@ void SGBM3WayMainLoop::getRawMatchingCost(const BufferSGBM3Way &mem, int y, int { calcPixelCostBT( *img1, *img2, k, minD, maxD, pixDiff, tmpBuf, clipTab + TAB_OFS ); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 sw2_1 = vx_setall_s16((short)SW2 + 1); - for (d = 0; d < Da; d += v_int16::nlanes) + for (d = 0; d < Da; d += VTraits::vlanes()) { - v_int16 hsA = vx_load_aligned(pixDiff + d) * sw2_1; + v_int16 hsA = v_mul(vx_load_aligned(pixDiff + d), sw2_1); for (x = Da; x <= SW2 * Da; x += Da) - hsA += vx_load_aligned(pixDiff + x + d); + hsA = v_add(hsA, vx_load_aligned(pixDiff + x + d)); v_store_aligned(hsumAdd + d, hsA); } #else @@ -1680,9 +1680,9 @@ void SGBM3WayMainLoop::getRawMatchingCost(const BufferSGBM3Way &mem, int y, int { const CostType* hsumSub = mem.getHSumBuf(std::max(y - SH2 - 1, src_start_idx)); -#if CV_SIMD - for (d = 0; d < Da; d += v_int16::nlanes) - v_store_aligned(C + d, vx_load_aligned(C + d) + vx_load_aligned(hsumAdd + d) - vx_load_aligned(hsumSub + d)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (d = 0; d < Da; d += VTraits::vlanes()) + v_store_aligned(C + d, v_sub(v_add(vx_load_aligned(C + d), vx_load_aligned(hsumAdd + d)), vx_load_aligned(hsumSub + d))); #else for (d = 0; d < D; d++) C[d] = (CostType)(C[d] + hsumAdd[d] - hsumSub[d]); @@ -1692,13 +1692,13 @@ void SGBM3WayMainLoop::getRawMatchingCost(const BufferSGBM3Way &mem, int y, int { const CostType* pixAdd = pixDiff + std::min(x + SW2*Da, (width1-1)*Da); const CostType* pixSub = pixDiff + std::max(x - (SW2+1)*Da, 0); -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 hv_reg; - for( d = 0; d < Da; d+=v_int16::nlanes ) + for( d = 0; d < Da; d+=VTraits::vlanes() ) { - hv_reg = vx_load_aligned(hsumAdd+x-Da+d) + vx_load_aligned(pixAdd+d) - vx_load_aligned(pixSub+d); + hv_reg = v_sub(v_add(vx_load_aligned(hsumAdd + x - this->Da + d), vx_load_aligned(pixAdd + d)), vx_load_aligned(pixSub + d)); v_store_aligned(hsumAdd+x+d,hv_reg); - v_store_aligned(C+x+d,vx_load_aligned(C+x+d)+hv_reg-vx_load_aligned(hsumSub+x+d)); + v_store_aligned(C+x+d,v_sub(v_add(vx_load_aligned(C + x + d), hv_reg), vx_load_aligned(hsumSub + x + d))); } #else for( d = 0; d < D; d++ ) @@ -1711,10 +1711,10 @@ void SGBM3WayMainLoop::getRawMatchingCost(const BufferSGBM3Way &mem, int y, int } else { -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 v_scale = vx_setall_s16(k == src_start_idx ? (short)SH2 + 1 : 1); - for (d = 0; d < Da; d += v_int16::nlanes) - v_store_aligned(C + d, vx_load_aligned(C + d) + vx_load_aligned(hsumAdd + d) * v_scale); + for (d = 0; d < Da; d += VTraits::vlanes()) + v_store_aligned(C + d, v_add(vx_load_aligned(C + d), v_mul(vx_load_aligned(hsumAdd + d), v_scale))); #else int scale = k == src_start_idx ? SH2 + 1 : 1; for (d = 0; d < D; d++) @@ -1724,12 +1724,12 @@ void SGBM3WayMainLoop::getRawMatchingCost(const BufferSGBM3Way &mem, int y, int { const CostType* pixAdd = pixDiff + std::min(x + SW2*Da, (width1-1)*Da); const CostType* pixSub = pixDiff + std::max(x - (SW2+1)*Da, 0); -#if CV_SIMD - for (d = 0; d < Da; d += v_int16::nlanes) +#if (CV_SIMD || CV_SIMD_SCALABLE) + for (d = 0; d < Da; d += VTraits::vlanes()) { - v_int16 hv = vx_load_aligned(hsumAdd + x - Da + d) + vx_load_aligned(pixAdd + d) - vx_load_aligned(pixSub + d); + v_int16 hv = v_sub(v_add(vx_load_aligned(hsumAdd + x - this->Da + d), vx_load_aligned(pixAdd + d)), vx_load_aligned(pixSub + d)); v_store_aligned(hsumAdd + x + d, hv); - v_store_aligned(C + x + d, vx_load_aligned(C + x + d) + hv * v_scale); + v_store_aligned(C + x + d, v_add(vx_load_aligned(C + x + d), v_mul(hv, v_scale))); } #else for (d = 0; d < D; d++) @@ -1747,9 +1747,9 @@ void SGBM3WayMainLoop::getRawMatchingCost(const BufferSGBM3Way &mem, int y, int if( y > src_start_idx ) { const CostType* hsumSub = mem.getHSumBuf(std::max(y - SH2 - 1, src_start_idx)); -#if CV_SIMD - for( x = 0; x < width1*Da; x += v_int16::nlanes) - v_store_aligned(C + x, vx_load_aligned(C + x) + vx_load_aligned(hsumAdd + x) - vx_load_aligned(hsumSub + x)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( x = 0; x < width1*Da; x += VTraits::vlanes()) + v_store_aligned(C + x, v_sub(v_add(vx_load_aligned(C + x), vx_load_aligned(hsumAdd + x)), vx_load_aligned(hsumSub + x))); #else for( x = 0; x < width1*Da; x++ ) C[x] = (CostType)(C[x] + hsumAdd[x] - hsumSub[x]); @@ -1757,9 +1757,9 @@ void SGBM3WayMainLoop::getRawMatchingCost(const BufferSGBM3Way &mem, int y, int } else { -#if CV_SIMD - for( x = 0; x < width1*Da; x += v_int16::nlanes) - v_store_aligned(C + x, vx_load_aligned(C + x) + vx_load_aligned(hsumAdd + x)); +#if (CV_SIMD || CV_SIMD_SCALABLE) + for( x = 0; x < width1*Da; x += VTraits::vlanes()) + v_store_aligned(C + x, v_add(vx_load_aligned(C + x), vx_load_aligned(hsumAdd + x))); #else for( x = 0; x < width1*Da; x++ ) C[x] = (CostType)(C[x] + hsumAdd[x]); @@ -1780,7 +1780,7 @@ void SGBM3WayMainLoop::accumulateCostsLeftTop(const BufferSGBM3Way &mem, int x, CostType *costs = mem.curCostVolumeLine - Da + x; CostType& topMinCost = mem.vertPassMin[x/Da]; int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 P1_reg = vx_setall_s16(cv::saturate_cast(P1)); v_int16 leftMinCostP2_reg = vx_setall_s16(cv::saturate_cast(leftMinCost+P2)); @@ -1797,18 +1797,18 @@ void SGBM3WayMainLoop::accumulateCostsLeftTop(const BufferSGBM3Way &mem, int x, v_int16 src_shifted_left,src_shifted_right; v_int16 res; - for(;i::vlanes();i+= VTraits::vlanes()) { //process leftBuf: //lookahead load: - src2 = vx_load_aligned(leftBuf_prev+i+v_int16::nlanes); + src2 = vx_load_aligned(leftBuf_prev+i+VTraits::vlanes()); //get shifted versions of the current block and add P1: src_shifted_left = v_rotate_left<1> (src1_leftBuf,src0_leftBuf); src_shifted_right = v_rotate_right<1> (src1_leftBuf,src2 ); // process and save current block: - res = vx_load_aligned(costs+i) + (v_min(v_min(src_shifted_left,src_shifted_right) + P1_reg,v_min(src1_leftBuf,leftMinCostP2_reg))-leftMinCostP2_reg); + res = v_add(vx_load_aligned(costs + i), v_sub(v_min(v_add(v_min(src_shifted_left, src_shifted_right), P1_reg), v_min(src1_leftBuf, leftMinCostP2_reg)), leftMinCostP2_reg)); leftMinCost_new_reg = v_min(leftMinCost_new_reg,res); v_store_aligned(leftBuf+i, res); @@ -1818,14 +1818,14 @@ void SGBM3WayMainLoop::accumulateCostsLeftTop(const BufferSGBM3Way &mem, int x, //process topBuf: //lookahead load: - src2 = vx_load_aligned(topBuf+i+v_int16::nlanes); + src2 = vx_load_aligned(topBuf+i+VTraits::vlanes()); //get shifted versions of the current block and add P1: src_shifted_left = v_rotate_left<1> (src1_topBuf,src0_topBuf); src_shifted_right = v_rotate_right<1> (src1_topBuf,src2 ); // process and save current block: - res = vx_load_aligned(costs+i) + (v_min(v_min(src_shifted_left,src_shifted_right) + P1_reg,v_min(src1_topBuf,topMinCostP2_reg))-topMinCostP2_reg); + res = v_add(vx_load_aligned(costs + i), v_sub(v_min(v_add(v_min(src_shifted_left, src_shifted_right), P1_reg), v_min(src1_topBuf, topMinCostP2_reg)), topMinCostP2_reg)); topMinCost_new_reg = v_min(topMinCost_new_reg,res); v_store_aligned(topBuf+i, res); @@ -1842,17 +1842,17 @@ void SGBM3WayMainLoop::accumulateCostsLeftTop(const BufferSGBM3Way &mem, int x, src_shifted_left = v_rotate_left<1> (src1_leftBuf,src0_leftBuf); src_shifted_right = v_rotate_right<1> (src1_leftBuf,src2 ); - res = vx_load_aligned(costs+Da-v_int16::nlanes) + (v_min(v_min(src_shifted_left,src_shifted_right) + P1_reg,v_min(src1_leftBuf,leftMinCostP2_reg))-leftMinCostP2_reg); + res = v_add(vx_load_aligned(costs + this->Da - VTraits::vlanes()), v_sub(v_min(v_add(v_min(src_shifted_left, src_shifted_right), P1_reg), v_min(src1_leftBuf, leftMinCostP2_reg)), leftMinCostP2_reg)); leftMinCost = v_reduce_min(v_min(leftMinCost_new_reg,res)); - v_store_aligned(leftBuf+Da-v_int16::nlanes, res); + v_store_aligned(leftBuf+Da-VTraits::vlanes(), res); //process topBuf: src_shifted_left = v_rotate_left<1> (src1_topBuf,src0_topBuf); src_shifted_right = v_rotate_right<1> (src1_topBuf,src2 ); - res = vx_load_aligned(costs+Da-v_int16::nlanes) + (v_min(v_min(src_shifted_left,src_shifted_right) + P1_reg,v_min(src1_topBuf,topMinCostP2_reg))-topMinCostP2_reg); + res = v_add(vx_load_aligned(costs + this->Da - VTraits::vlanes()), v_sub(v_min(v_add(v_min(src_shifted_left, src_shifted_right), P1_reg), v_min(src1_topBuf, topMinCostP2_reg)), topMinCostP2_reg)); topMinCost = v_reduce_min(v_min(topMinCost_new_reg,res)); - v_store_aligned(topBuf+Da-v_int16::nlanes, res); + v_store_aligned(topBuf+Da-VTraits::vlanes(), res); } else { @@ -1903,7 +1903,7 @@ void SGBM3WayMainLoop::accumulateCostsRight(const BufferSGBM3Way &mem, int x, CostType* leftBuf = mem.horPassCostVolume + x; int i = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) v_int16 P1_reg = vx_setall_s16(cv::saturate_cast(P1)); v_int16 rightMinCostP2_reg = vx_setall_s16(cv::saturate_cast(rightMinCost+P2)); @@ -1918,27 +1918,27 @@ void SGBM3WayMainLoop::accumulateCostsRight(const BufferSGBM3Way &mem, int x, v_int16 min_sum_cost_reg = vx_setall_s16(SHRT_MAX); v_int16 min_sum_pos_reg = vx_setall_s16(0); - for(;i::vlanes();i+=VTraits::vlanes()) { //lookahead load: - src2 = vx_load_aligned(rightBuf+i+v_int16::nlanes); + src2 = vx_load_aligned(rightBuf+i+VTraits::vlanes()); //get shifted versions of the current block and add P1: src_shifted_left = v_rotate_left<1> (src1_rightBuf,src0_rightBuf); src_shifted_right = v_rotate_right<1> (src1_rightBuf,src2 ); // process and save current block: - res = vx_load_aligned(costs+i) + (v_min(v_min(src_shifted_left,src_shifted_right) + P1_reg,v_min(src1_rightBuf,rightMinCostP2_reg))-rightMinCostP2_reg); + res = v_add(vx_load_aligned(costs + i), v_sub(v_min(v_add(v_min(src_shifted_left, src_shifted_right), P1_reg), v_min(src1_rightBuf, rightMinCostP2_reg)), rightMinCostP2_reg)); rightMinCost_new_reg = v_min(rightMinCost_new_reg,res); v_store_aligned(rightBuf+i, res); // compute and save total cost: - res = res + vx_load_aligned(leftBuf+i) + vx_load_aligned(topBuf+i); + res = v_add(v_add(res, vx_load_aligned(leftBuf + i)), vx_load_aligned(topBuf + i)); v_store_aligned(leftBuf+i, res); // track disparity value with the minimum cost: min_sum_cost_reg = v_min(min_sum_cost_reg,res); - min_sum_pos_reg = min_sum_pos_reg + ((min_sum_cost_reg == res) & (vx_setall_s16((short)i) - min_sum_pos_reg)); + min_sum_pos_reg = v_add(min_sum_pos_reg, v_and(v_eq(min_sum_cost_reg, res), v_sub(vx_setall_s16((short)i), min_sum_pos_reg))); //update src: src0_rightBuf = src1_rightBuf; @@ -1952,15 +1952,15 @@ void SGBM3WayMainLoop::accumulateCostsRight(const BufferSGBM3Way &mem, int x, src_shifted_left = v_rotate_left<1> (src1_rightBuf,src0_rightBuf); src_shifted_right = v_rotate_right<1> (src1_rightBuf,src2 ); - res = vx_load_aligned(costs+D-v_int16::nlanes) + (v_min(v_min(src_shifted_left,src_shifted_right) + P1_reg,v_min(src1_rightBuf,rightMinCostP2_reg))-rightMinCostP2_reg); + res = v_add(vx_load_aligned(costs + this->D - VTraits::vlanes()), v_sub(v_min(v_add(v_min(src_shifted_left, src_shifted_right), P1_reg), v_min(src1_rightBuf, rightMinCostP2_reg)), rightMinCostP2_reg)); rightMinCost = v_reduce_min(v_min(rightMinCost_new_reg,res)); - v_store_aligned(rightBuf+D-v_int16::nlanes, res); + v_store_aligned(rightBuf+D-VTraits::vlanes(), res); - res = res + vx_load_aligned(leftBuf+D-v_int16::nlanes) + vx_load_aligned(topBuf+D-v_int16::nlanes); - v_store_aligned(leftBuf+D-v_int16::nlanes, res); + res = v_add(v_add(res, vx_load_aligned(leftBuf + this->D - VTraits::vlanes())), vx_load_aligned(topBuf + this->D - VTraits::vlanes())); + v_store_aligned(leftBuf+D-VTraits::vlanes(), res); min_sum_cost_reg = v_min(min_sum_cost_reg,res); - min_sum_pos_reg = min_sum_pos_reg + ((min_sum_cost_reg == res) & (vx_setall_s16((short)D-v_int16::nlanes) - min_sum_pos_reg)); + min_sum_pos_reg = v_add(min_sum_pos_reg, v_and(v_eq(min_sum_cost_reg, res), v_sub(vx_setall_s16((short)(this->D - VTraits::vlanes())), min_sum_pos_reg))); min_pos(min_sum_cost_reg,min_sum_pos_reg, min_cost, optimal_disp); } else @@ -2069,40 +2069,40 @@ void SGBM3WayMainLoop::impl(const Range& range) const if(uniquenessRatio>0) { d = 0; -#if CV_SIMD +#if (CV_SIMD || CV_SIMD_SCALABLE) horPassCostVolume+=x; int thresh = (100*min_cost)/(100-uniquenessRatio); v_int16 thresh_reg = vx_setall_s16((short)(thresh+1)); v_int16 d1 = vx_setall_s16((short)(best_d-1)); v_int16 d2 = vx_setall_s16((short)(best_d+1)); - v_int16 eight_reg = vx_setall_s16(v_int16::nlanes); + v_int16 eight_reg = vx_setall_s16((short)VTraits::vlanes()); v_int16 cur_d = vx_load(idx_row); v_int16 mask; - for( ; d <= D - 2*v_int16::nlanes; d+=2*v_int16::nlanes ) + for( ; d <= D - 2*VTraits::vlanes(); d+=2*VTraits::vlanes() ) { - mask = (vx_load_aligned(horPassCostVolume + d) < thresh_reg) & ( (cur_dd2) ); - cur_d = cur_d+eight_reg; + mask = v_and(v_lt(vx_load_aligned(horPassCostVolume + d), thresh_reg), v_or(v_lt(cur_d, d1), v_gt(cur_d, d2))); + cur_d = v_add(cur_d, eight_reg); if( v_check_any(mask) ) break; - mask = (vx_load_aligned(horPassCostVolume + d + v_int16::nlanes) < thresh_reg) & ( (cur_dd2) ); - cur_d = cur_d+eight_reg; + mask = v_and(v_lt(vx_load_aligned(horPassCostVolume + d + VTraits::vlanes()), thresh_reg), v_or(v_lt(cur_d, d1), v_gt(cur_d, d2))); + cur_d = v_add(cur_d, eight_reg); if( v_check_any(mask) ) break; } - if( d <= D - 2*v_int16::nlanes ) + if( d <= D - 2*VTraits::vlanes() ) { horPassCostVolume-=x; continue; } - if( d <= D - v_int16::nlanes ) + if( d <= D - VTraits::vlanes() ) { - if( v_check_any((vx_load_aligned(horPassCostVolume + d) < thresh_reg) & ((cur_d < d1) | (cur_d > d2))) ) + if( v_check_any(v_and(v_lt(vx_load_aligned(horPassCostVolume + d), thresh_reg), v_or(v_lt(cur_d, d1), v_gt(cur_d, d2)))) ) { horPassCostVolume-=x; continue; } - d+=v_int16::nlanes; + d+=VTraits::vlanes(); } horPassCostVolume-=x; #endif diff --git a/modules/ts/include/opencv2/ts/ts_ext.hpp b/modules/ts/include/opencv2/ts/ts_ext.hpp index efa4860510..e7e01fb3ed 100644 --- a/modules/ts/include/opencv2/ts/ts_ext.hpp +++ b/modules/ts/include/opencv2/ts/ts_ext.hpp @@ -31,6 +31,8 @@ bool checkBigDataTests(); #define CV__TEST_INIT \ CV__TEST_NAMESPACE_CHECK \ + if (setUpSkipped) \ + return; \ ::cvtest::testSetUp(); #define CV__TEST_CLEANUP ::cvtest::testTearDown(); #define CV__TEST_BODY_IMPL(name) \ @@ -47,6 +49,25 @@ bool checkBigDataTests(); } \ } \ +#define CV__TEST_SETUP_IMPL(parent_class) { \ + setUpSkipped = false; \ + try { \ + parent_class::SetUp(); \ + } catch (const cvtest::details::SkipTestExceptionBase& e) { \ + setUpSkipped = true; \ + printf("[ SKIP ] %s\n", e.what()); \ + } \ +} \ + +struct SkipThisTest : public ::testing::Test { + SkipThisTest(const std::string& msg_) : msg(msg_) {} + + virtual void TestBody() CV_OVERRIDE { + printf("[ SKIP ] %s\n", msg.c_str()); + } + + std::string msg; +}; #undef TEST #define TEST_(test_case_name, test_name, parent_class, bodyMethodName, BODY_ATTR, BODY_IMPL) \ @@ -54,12 +75,24 @@ bool checkBigDataTests(); public:\ GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {}\ private:\ + bool setUpSkipped = false; \ virtual void TestBody() CV_OVERRIDE;\ virtual void bodyMethodName() BODY_ATTR;\ + virtual void SetUp() CV_OVERRIDE; \ static ::testing::TestInfo* const test_info_ GTEST_ATTRIBUTE_UNUSED_;\ GTEST_DISALLOW_COPY_AND_ASSIGN_(\ GTEST_TEST_CLASS_NAME_(test_case_name, test_name));\ };\ + class test_case_name##test_name##_factory : public ::testing::internal::TestFactoryBase { \ + public:\ + virtual ::testing::Test* CreateTest() { \ + try { \ + return new GTEST_TEST_CLASS_NAME_(test_case_name, test_name); \ + } catch (const cvtest::details::SkipTestExceptionBase& e) { \ + return new SkipThisTest(e.what()); \ + } \ + } \ + };\ \ ::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_case_name, test_name)\ ::test_info_ =\ @@ -69,9 +102,9 @@ bool checkBigDataTests(); (::testing::internal::GetTestTypeId()), \ parent_class::SetUpTestCase, \ parent_class::TearDownTestCase, \ - new ::testing::internal::TestFactoryImpl<\ - GTEST_TEST_CLASS_NAME_(test_case_name, test_name)>);\ + new test_case_name##test_name##_factory);\ void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() BODY_IMPL( #test_case_name "_" #test_name ) \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::SetUp() CV__TEST_SETUP_IMPL(parent_class) \ void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::bodyMethodName() #define TEST(test_case_name, test_name) TEST_(test_case_name, test_name, ::testing::Test, Body,, CV__TEST_BODY_IMPL) @@ -107,12 +140,24 @@ bool checkBigDataTests(); public:\ GTEST_TEST_CLASS_NAME_(test_fixture, test_name)() {}\ private:\ + bool setUpSkipped = false; \ virtual void TestBody() CV_OVERRIDE;\ virtual void Body(); \ + virtual void SetUp() CV_OVERRIDE; \ static ::testing::TestInfo* const test_info_ GTEST_ATTRIBUTE_UNUSED_;\ GTEST_DISALLOW_COPY_AND_ASSIGN_(\ GTEST_TEST_CLASS_NAME_(test_fixture, test_name));\ };\ + class test_fixture##test_name##_factory : public ::testing::internal::TestFactoryBase { \ + public:\ + virtual ::testing::Test* CreateTest() { \ + try { \ + return new GTEST_TEST_CLASS_NAME_(test_fixture, test_name); \ + } catch (const cvtest::details::SkipTestExceptionBase& e) { \ + return new SkipThisTest(e.what()); \ + } \ + } \ + };\ \ ::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_fixture, test_name)\ ::test_info_ =\ @@ -122,9 +167,9 @@ bool checkBigDataTests(); (::testing::internal::GetTypeId()), \ test_fixture::SetUpTestCase, \ test_fixture::TearDownTestCase, \ - new ::testing::internal::TestFactoryImpl<\ - GTEST_TEST_CLASS_NAME_(test_fixture, test_name)>);\ + new test_fixture##test_name##_factory);\ void GTEST_TEST_CLASS_NAME_(test_fixture, test_name)::TestBody() CV__TEST_BODY_IMPL( #test_fixture "_" #test_name ) \ + void GTEST_TEST_CLASS_NAME_(test_fixture, test_name)::SetUp() CV__TEST_SETUP_IMPL(test_fixture) \ void GTEST_TEST_CLASS_NAME_(test_fixture, test_name)::Body() // Don't use directly @@ -134,8 +179,10 @@ bool checkBigDataTests(); public: \ GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \ private: \ + bool setUpSkipped = false; \ virtual void bodyMethodName() BODY_ATTR; \ virtual void TestBody() CV_OVERRIDE; \ + virtual void SetUp() CV_OVERRIDE; \ static int AddToRegistry() { \ ::testing::UnitTest::GetInstance()->parameterized_test_registry(). \ GetTestCasePatternHolder(\ @@ -157,6 +204,7 @@ bool checkBigDataTests(); test_name)::gtest_registering_dummy_ = \ GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::AddToRegistry(); \ void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody() BODY_IMPL( #test_case_name "_" #test_name ) \ + void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::SetUp() CV__TEST_SETUP_IMPL(test_case_name) \ void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::bodyMethodName() #undef TEST_P diff --git a/modules/video/include/opencv2/video/tracking.hpp b/modules/video/include/opencv2/video/tracking.hpp index eb5a6c7030..7f93a79a72 100644 --- a/modules/video/include/opencv2/video/tracking.hpp +++ b/modules/video/include/opencv2/video/tracking.hpp @@ -887,6 +887,43 @@ public: //bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; }; +/** @brief the VIT tracker is a super lightweight dnn-based general object tracking. + * + * VIT tracker is much faster and extremely lightweight due to special model structure, the model file is about 767KB. + * Model download link: https://github.com/opencv/opencv_zoo/tree/main/models/object_tracking_vittrack + * Author: PengyuLiu, 1872918507@qq.com + */ +class CV_EXPORTS_W TrackerVit : public Tracker +{ +protected: + TrackerVit(); // use ::create() +public: + virtual ~TrackerVit() CV_OVERRIDE; + + struct CV_EXPORTS_W_SIMPLE Params + { + CV_WRAP Params(); + CV_PROP_RW std::string net; + CV_PROP_RW int backend; + CV_PROP_RW int target; + CV_PROP_RW Scalar meanvalue; + CV_PROP_RW Scalar stdvalue; + }; + + /** @brief Constructor + @param parameters vit tracker parameters TrackerVit::Params + */ + static CV_WRAP + Ptr create(const TrackerVit::Params& parameters = TrackerVit::Params()); + + /** @brief Return tracking score + */ + CV_WRAP virtual float getTrackingScore() = 0; + + // void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + // bool update(InputArray image, CV_OUT Rect& boundingBox) CV_OVERRIDE; +}; + //! @} video_track } // cv diff --git a/modules/video/src/tracking/tracker_vit.cpp b/modules/video/src/tracking/tracker_vit.cpp new file mode 100644 index 0000000000..5a6f3e9ac5 --- /dev/null +++ b/modules/video/src/tracking/tracker_vit.cpp @@ -0,0 +1,219 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// Author, PengyuLiu, 1872918507@qq.com + +#include "../precomp.hpp" +#ifdef HAVE_OPENCV_DNN +#include "opencv2/dnn.hpp" +#endif + +namespace cv { + +TrackerVit::TrackerVit() +{ + // nothing +} + +TrackerVit::~TrackerVit() +{ + // nothing +} + +TrackerVit::Params::Params() +{ + net = "vitTracker.onnx"; + meanvalue = Scalar{0.485, 0.456, 0.406}; + stdvalue = Scalar{0.229, 0.224, 0.225}; +#ifdef HAVE_OPENCV_DNN + backend = dnn::DNN_BACKEND_DEFAULT; + target = dnn::DNN_TARGET_CPU; +#else + backend = -1; // invalid value + target = -1; // invalid value +#endif +} + +#ifdef HAVE_OPENCV_DNN + +class TrackerVitImpl : public TrackerVit +{ +public: + TrackerVitImpl(const TrackerVit::Params& parameters) + : params(parameters) + { + net = dnn::readNet(params.net); + CV_Assert(!net.empty()); + } + + void init(InputArray image, const Rect& boundingBox) CV_OVERRIDE; + bool update(InputArray image, Rect& boundingBox) CV_OVERRIDE; + float getTrackingScore() CV_OVERRIDE; + + Rect rect_last; + float tracking_score; + + TrackerVit::Params params; + + +protected: + void preprocess(const Mat& src, Mat& dst, Size size); + + const Size searchSize{256, 256}; + const Size templateSize{128, 128}; + + Mat hanningWindow; + + dnn::Net net; + Mat image; +}; + +static void crop_image(const Mat& src, Mat& dst, Rect box, int factor) +{ + int x = box.x, y = box.y, w = box.width, h = box.height; + int crop_sz = cvCeil(sqrt(w * h) * factor); + + int x1 = x + (w - crop_sz) / 2; + int x2 = x1 + crop_sz; + int y1 = y + (h - crop_sz) / 2; + int y2 = y1 + crop_sz; + + int x1_pad = std::max(0, -x1); + int y1_pad = std::max(0, -y1); + int x2_pad = std::max(x2 - src.size[1] + 1, 0); + int y2_pad = std::max(y2 - src.size[0] + 1, 0); + + Rect roi(x1 + x1_pad, y1 + y1_pad, x2 - x2_pad - x1 - x1_pad, y2 - y2_pad - y1 - y1_pad); + Mat im_crop = src(roi); + copyMakeBorder(im_crop, dst, y1_pad, y2_pad, x1_pad, x2_pad, BORDER_CONSTANT); +} + +void TrackerVitImpl::preprocess(const Mat& src, Mat& dst, Size size) +{ + Mat mean = Mat(size, CV_32FC3, params.meanvalue); + Mat std = Mat(size, CV_32FC3, params.stdvalue); + mean = dnn::blobFromImage(mean, 1.0, Size(), Scalar(), false); + std = dnn::blobFromImage(std, 1.0, Size(), Scalar(), false); + + Mat img; + resize(src, img, size); + + dst = dnn::blobFromImage(img, 1.0, Size(), Scalar(), false); + dst /= 255; + dst = (dst - mean) / std; +} + +static Mat hann1d(int sz, bool centered = true) { + Mat hanningWindow(sz, 1, CV_32FC1); + float* data = hanningWindow.ptr(0); + + if(centered) { + for(int i = 0; i < sz; i++) { + float val = 0.5f * (1.f - std::cos(static_cast(2 * M_PI / (sz + 1)) * (i + 1))); + data[i] = val; + } + } + else { + int half_sz = sz / 2; + for(int i = 0; i <= half_sz; i++) { + float val = 0.5f * (1.f + std::cos(static_cast(2 * M_PI / (sz + 2)) * i)); + data[i] = val; + data[sz - 1 - i] = val; + } + } + + return hanningWindow; +} + +static Mat hann2d(Size size, bool centered = true) { + int rows = size.height; + int cols = size.width; + + Mat hanningWindowRows = hann1d(rows, centered); + Mat hanningWindowCols = hann1d(cols, centered); + + Mat hanningWindow = hanningWindowRows * hanningWindowCols.t(); + + return hanningWindow; +} + +static Rect returnfromcrop(float x, float y, float w, float h, Rect res_Last) +{ + int cropwindowwh = 4 * cvFloor(sqrt(res_Last.width * res_Last.height)); + int x0 = res_Last.x + (res_Last.width - cropwindowwh) / 2; + int y0 = res_Last.y + (res_Last.height - cropwindowwh) / 2; + Rect finalres; + finalres.x = cvFloor(x * cropwindowwh + x0); + finalres.y = cvFloor(y * cropwindowwh + y0); + finalres.width = cvFloor(w * cropwindowwh); + finalres.height = cvFloor(h * cropwindowwh); + return finalres; +} + +void TrackerVitImpl::init(InputArray image_, const Rect &boundingBox_) +{ + image = image_.getMat().clone(); + Mat crop; + crop_image(image, crop, boundingBox_, 2); + Mat blob; + preprocess(crop, blob, templateSize); + net.setInput(blob, "template"); + Size size(16, 16); + hanningWindow = hann2d(size, false); + rect_last = boundingBox_; +} + +bool TrackerVitImpl::update(InputArray image_, Rect &boundingBoxRes) +{ + image = image_.getMat().clone(); + Mat crop; + crop_image(image, crop, rect_last, 4); + Mat blob; + preprocess(crop, blob, searchSize); + net.setInput(blob, "search"); + std::vector outputName = {"output1", "output2", "output3"}; + std::vector outs; + net.forward(outs, outputName); + CV_Assert(outs.size() == 3); + + Mat conf_map = outs[0].reshape(0, {16, 16}); + Mat size_map = outs[1].reshape(0, {2, 16, 16}); + Mat offset_map = outs[2].reshape(0, {2, 16, 16}); + + multiply(conf_map, (1.0 - hanningWindow), conf_map); + + double maxVal; + Point maxLoc; + minMaxLoc(conf_map, nullptr, &maxVal, nullptr, &maxLoc); + tracking_score = static_cast(maxVal); + + float cx = (maxLoc.x + offset_map.at(0, maxLoc.y, maxLoc.x)) / 16; + float cy = (maxLoc.y + offset_map.at(1, maxLoc.y, maxLoc.x)) / 16; + float w = size_map.at(0, maxLoc.y, maxLoc.x); + float h = size_map.at(1, maxLoc.y, maxLoc.x); + + Rect finalres = returnfromcrop(cx - w / 2, cy - h / 2, w, h, rect_last); + rect_last = finalres; + boundingBoxRes = finalres; + return true; +} + +float TrackerVitImpl::getTrackingScore() +{ + return tracking_score; +} + +Ptr TrackerVit::create(const TrackerVit::Params& parameters) +{ + return makePtr(parameters); +} + +#else // OPENCV_HAVE_DNN +Ptr TrackerVit::create(const TrackerVit::Params& parameters) +{ + CV_UNUSED(parameters); + CV_Error(Error::StsNotImplemented, "to use vittrack, the tracking module needs to be built with opencv_dnn !"); +} +#endif // OPENCV_HAVE_DNN +} diff --git a/modules/video/test/test_trackers.cpp b/modules/video/test/test_trackers.cpp index 6ede40896c..7186d0fe6b 100644 --- a/modules/video/test/test_trackers.cpp +++ b/modules/video/test/test_trackers.cpp @@ -160,4 +160,13 @@ TEST(NanoTrack, accuracy_NanoTrack_V2) checkTrackingAccuracy(tracker, 0.69); } +TEST(vittrack, accuracy_vittrack) +{ + std::string model = cvtest::findDataFile("dnn/onnx/models/vitTracker.onnx"); + cv::TrackerVit::Params params; + params.net = model; + cv::Ptr tracker = TrackerVit::create(params); + checkTrackingAccuracy(tracker, 0.67); +} + }} // namespace opencv_test:: diff --git a/modules/videoio/test/test_camera.cpp b/modules/videoio/test/test_camera.cpp index 8b0f0efe83..4466919f50 100644 --- a/modules/videoio/test/test_camera.cpp +++ b/modules/videoio/test/test_camera.cpp @@ -130,7 +130,12 @@ TEST(DISABLED_videoio_camera, msmf_read_yuyv) std::cout << "Capturing FPS: " << capture.get(CAP_PROP_FPS) << std::endl; int fourcc = (int)capture.get(CAP_PROP_FOURCC); std::cout << "FOURCC code: " << cv::format("0x%8x", fourcc) << std::endl; - test_readFrames(capture); + cv::Mat frame; + for (int i = 0; i < 10; i++) + { + capture >> frame; + EXPECT_EQ(2, frame.channels()); + } capture.release(); } diff --git a/platforms/linux/riscv-gnu.toolchain.cmake b/platforms/linux/riscv-gnu.toolchain.cmake index 662fb6bddb..1657bd1681 100644 --- a/platforms/linux/riscv-gnu.toolchain.cmake +++ b/platforms/linux/riscv-gnu.toolchain.cmake @@ -25,22 +25,22 @@ if(NOT DEFINED TOOLCHAIN_COMPILER_LOCATION_HINT) endif() if(NOT DEFINED CMAKE_C_COMPILER) - find_program(CMAKE_C_COMPILER NAMES ${GNU_MACHINE}-gcc${__GCC_VER_SUFFIX} ${TOOLCHAIN_COMPILER_LOCATION_HINT}) + find_program(CMAKE_C_COMPILER NAMES ${GNU_MACHINE}-gcc${__GCC_VER_SUFFIX} PATHS ${TOOLCHAIN_COMPILER_LOCATION_HINT}) else() #message(WARNING "CMAKE_C_COMPILER=${CMAKE_C_COMPILER} is defined") endif() if(NOT DEFINED CMAKE_CXX_COMPILER) - find_program(CMAKE_CXX_COMPILER NAMES ${GNU_MACHINE}-g++${__GCC_VER_SUFFIX} ${TOOLCHAIN_COMPILER_LOCATION_HINT}) + find_program(CMAKE_CXX_COMPILER NAMES ${GNU_MACHINE}-g++${__GCC_VER_SUFFIX} PATHS ${TOOLCHAIN_COMPILER_LOCATION_HINT}) else() #message(WARNING "CMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} is defined") endif() if(NOT DEFINED CMAKE_LINKER) - find_program(CMAKE_LINKER NAMES ${GNU_MACHINE}-ld${__GCC_VER_SUFFIX} ${GNU_MACHINE}-ld ${TOOLCHAIN_COMPILER_LOCATION_HINT}) + find_program(CMAKE_LINKER NAMES ${GNU_MACHINE}-ld${__GCC_VER_SUFFIX} ${GNU_MACHINE}-ld PATHS ${TOOLCHAIN_COMPILER_LOCATION_HINT}) else() #message(WARNING "CMAKE_LINKER=${CMAKE_LINKER} is defined") endif() if(NOT DEFINED CMAKE_AR) - find_program(CMAKE_AR NAMES ${GNU_MACHINE}-ar${__GCC_VER_SUFFIX} ${GNU_MACHINE}-ar ${TOOLCHAIN_COMPILER_LOCATION_HINT}) + find_program(CMAKE_AR NAMES ${GNU_MACHINE}-ar${__GCC_VER_SUFFIX} ${GNU_MACHINE}-ar PATHS ${TOOLCHAIN_COMPILER_LOCATION_HINT}) else() #message(WARNING "CMAKE_AR=${CMAKE_AR} is defined") endif() diff --git a/samples/dnn/vit_tracker.cpp b/samples/dnn/vit_tracker.cpp new file mode 100644 index 0000000000..02e5cea83f --- /dev/null +++ b/samples/dnn/vit_tracker.cpp @@ -0,0 +1,176 @@ +// VitTracker +// model: https://github.com/opencv/opencv_zoo/tree/main/models/object_tracking_vittrack + +#include +#include + +#include +#include +#include +#include + +using namespace cv; +using namespace cv::dnn; + +const char *keys = + "{ help h | | Print help message }" + "{ input i | | Full path to input video folder, the specific camera index. (empty for camera 0) }" + "{ net | vitTracker.onnx | Path to onnx model of vitTracker.onnx}" + "{ backend | 0 | Choose one of computation backends: " + "0: automatically (by default), " + "1: Halide language (http://halide-lang.org/), " + "2: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), " + "3: OpenCV implementation, " + "4: VKCOM, " + "5: CUDA }," + "{ target | 0 | Choose one of target computation devices: " + "0: CPU target (by default), " + "1: OpenCL, " + "2: OpenCL fp16 (half-float precision), " + "3: VPU, " + "4: Vulkan, " + "6: CUDA, " + "7: CUDA fp16 (half-float preprocess) }" +; + +static +int run(int argc, char** argv) +{ + // Parse command line arguments. + CommandLineParser parser(argc, argv, keys); + + if (parser.has("help")) + { + parser.printMessage(); + return 0; + } + + std::string inputName = parser.get("input"); + std::string net = parser.get("net"); + int backend = parser.get("backend"); + int target = parser.get("target"); + + Ptr tracker; + try + { + TrackerVit::Params params; + params.net = samples::findFile(net); + params.backend = backend; + params.target = target; + tracker = TrackerVit::create(params); + } + catch (const cv::Exception& ee) + { + std::cerr << "Exception: " << ee.what() << std::endl; + std::cout << "Can't load the network by using the following files:" << std::endl; + std::cout << "net : " << net << std::endl; + return 2; + } + + const std::string winName = "vitTracker"; + namedWindow(winName, WINDOW_AUTOSIZE); + + // Open a video file or an image file or a camera stream. + VideoCapture cap; + + if (inputName.empty() || (isdigit(inputName[0]) && inputName.size() == 1)) + { + int c = inputName.empty() ? 0 : inputName[0] - '0'; + std::cout << "Trying to open camera #" << c << " ..." << std::endl; + if (!cap.open(c)) + { + std::cout << "Capture from camera #" << c << " didn't work. Specify -i=