diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp index 2cadd01a0b..5e58c6ce0a 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp @@ -159,7 +159,8 @@ INSTANTIATE_TEST_CASE_P(BitwiseNotPerfTestFluid, BitwiseNotPerfTest, INSTANTIATE_TEST_CASE_P(SelectPerfTestFluid, SelectPerfTest, Combine(Values(AbsExact().to_compare_f()), Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1, CV_8UC3, CV_16UC1, CV_16SC1), + Values(CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4, + CV_16UC1, CV_16UC3, CV_16SC1, CV_16SC3), Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(MinPerfTestFluid, MinPerfTest, diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index aeface5046..1f781d518a 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -2195,64 +2195,28 @@ GAPI_FLUID_KERNEL(GFluidInRange, cv::gapi::core::GInRange, false) // //---------------------- -// manually vectored function for important case if RGB/BGR image -static void run_select_row3(int width, uchar out[], uchar in1[], uchar in2[], uchar in3[]) +#if (CV_SIMD || CV_SIMD_SCALABLE) + +template +static CV_ALWAYS_INLINE +typename std::enable_if::value, int>::type +call_select_simd(const SRC in1[], const SRC in2[], const uchar in3[], + SRC out[], int width, int chan) { - int w = 0; // cycle index + return select_simd(in1, in2, in3, out, width, chan); +} -#if CV_SIMD128 - for (; w <= width-16; w+=16) - { - v_uint8x16 a1, b1, c1; - v_uint8x16 a2, b2, c2; - v_uint8x16 mask; - v_uint8x16 a, b, c; - v_load_deinterleave(&in1[3*w], a1, b1, c1); - v_load_deinterleave(&in2[3*w], a2, b2, c2); - - mask = v_load(&in3[w]); - mask = v_ne(mask, v_setzero_u8()); - - a = v_select(mask, a1, a2); - b = v_select(mask, b1, b2); - c = v_select(mask, c1, c2); - - v_store_interleave(&out[3*w], a, b, c); - } +template +static CV_ALWAYS_INLINE +typename std::enable_if::value, int>::type +call_select_simd(const SRC[], const SRC[], const uchar[], + SRC[], int, int) +{ + return 0; +} #endif - for (; w < width; w++) - { - out[3*w ] = in3[w]? in1[3*w ]: in2[3*w ]; - out[3*w + 1] = in3[w]? in1[3*w + 1]: in2[3*w + 1]; - out[3*w + 2] = in3[w]? in1[3*w + 2]: in2[3*w + 2]; - } -} - -// parameter chan is compile-time known constant, normally chan=1..4 -template -static void run_select_row(int width, DST out[], SRC1 in1[], SRC2 in2[], SRC3 in3[]) -{ - if (std::is_same::value && chan==3) - { - // manually vectored function for important case if RGB/BGR image - run_select_row3(width, (uchar*)out, (uchar*)in1, (uchar*)in2, (uchar*)in3); - return; - } - - // because `chan` is template parameter, its value is known at compilation time, - // so that modern compilers would efficiently vectorize this cycle if chan==1 - // (if chan>1, compilers may need help with de-interleaving of the channels) - for (int w=0; w < width; w++) - { - for (int c=0; c < chan; c++) - { - out[w*chan + c] = in3[w]? in1[w*chan + c]: in2[w*chan + c]; - } - } -} - template static void run_select(Buffer &dst, const View &src1, const View &src2, const View &src3) { @@ -2261,7 +2225,6 @@ static void run_select(Buffer &dst, const View &src1, const View &src2, const Vi static_assert(std::is_same::value, "wrong types"); auto *out = dst.OutLine(); - const auto *in1 = src1.InLine(0); const auto *in2 = src2.InLine(0); const auto *in3 = src3.InLine(0); @@ -2269,13 +2232,18 @@ static void run_select(Buffer &dst, const View &src1, const View &src2, const Vi int width = dst.length(); int chan = dst.meta().chan; - switch (chan) + int w = 0; + +#if (CV_SIMD || CV_SIMD_SCALABLE) + w = call_select_simd(in1, in2, in3, out, width, chan); +#endif + + for (; w < width; w++) { - case 1: run_select_row<1>(width, out, in1, in2, in3); break; - case 2: run_select_row<2>(width, out, in1, in2, in3); break; - case 3: run_select_row<3>(width, out, in1, in2, in3); break; - case 4: run_select_row<4>(width, out, in1, in2, in3); break; - default: CV_Error(cv::Error::StsBadArg, "unsupported number of channels"); + for (int c = 0; c < chan; c++) + { + out[w*chan + c] = in3[w] ? in1[w*chan + c] : in2[w*chan + c]; + } } } diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp index 10e88ea44b..8a1e9e1d33 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp @@ -420,8 +420,30 @@ INRANGE_SIMD(short) #undef INRANGE_SIMD +//----------------------------------- +// +// Fluid kernels: Select +// +//----------------------------------- + +#define DISPATCH_SELECT_SIMD(SRC) \ +int select_simd(const SRC in1[], const SRC in2[], const uchar in3[], \ + SRC out[], const int length, const int chan) \ +{ \ + CV_CPU_DISPATCH(select_simd, (in1, in2, in3, out, length, chan), \ + CV_CPU_DISPATCH_MODES_ALL); \ +} + +DISPATCH_SELECT_SIMD(uchar) +DISPATCH_SELECT_SIMD(ushort) +DISPATCH_SELECT_SIMD(short) + +#undef DISPATCH_SELECT_SIMD + } // namespace fluid } // namespace gapi } // namespace cv + + #endif // CV_SIMD #endif // !defined(GAPI_STANDALONE) diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp index 28bffc5bc5..305c1172d3 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp @@ -331,8 +331,26 @@ INRANGE_SIMD(short) #undef INRANGE_SIMD +//----------------------------------- +// +// Fluid kernels: Select +// +//----------------------------------- +#define SELECT_SIMD(SRC) \ +int select_simd(const SRC in1[], const SRC in2[], const uchar in3[], \ + SRC out[], const int length, const int chan); + +SELECT_SIMD(uchar) +SELECT_SIMD(ushort) +SELECT_SIMD(short) + +#undef SELECT_SIMD + } // namespace fluid } // namespace gapi } // namespace cv + + + #endif // !defined(GAPI_STANDALONE) diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index 10e332264e..d0c805cb95 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -347,6 +347,18 @@ int merge4_simd(const uchar in1[], const uchar in2[], const uchar in3[], const uchar in4[], uchar out[], const int width); + +#define DECLARE_SELECT_SIMD(SRC) \ +int select_simd(const SRC in1[], const SRC in2[], const uchar in3[], \ + SRC out[], const int length, const int chan); + +DECLARE_SELECT_SIMD(uchar) +DECLARE_SELECT_SIMD(ushort) +DECLARE_SELECT_SIMD(short) + +#undef DECLARE_SELECT_SIMD + + #ifndef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY #define SRC_SHORT_OR_USHORT std::is_same::value || std::is_same::value @@ -3379,6 +3391,163 @@ INRANGE_SIMD(short) #undef INRANGE_SIMD +//----------------------------------- +// +// Fluid kernels: Select +// +//----------------------------------- + + + +CV_ALWAYS_INLINE int select_simd_impl(const uchar in1[], const uchar in2[], const uchar in3[], + uchar out[], const int length, const int chan) +{ + const int nlanes = VTraits::vlanes(); + if (length < nlanes) return 0; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_uint8 m = vx_load(&in3[x]); + m = v_ne(m, vx_setzero_u8()); + if (chan == 1) + { + v_uint8 a = vx_load(&in1[x]); + v_uint8 b = vx_load(&in2[x]); + v_store(&out[x], v_select(m, a, b)); + } + else if (chan == 2) + { + v_uint8 a0, a1, b0, b1; + v_load_deinterleave(&in1[2 * x], a0, a1); + v_load_deinterleave(&in2[2 * x], b0, b1); + v_store_interleave(&out[2 * x], v_select(m, a0, b0), + v_select(m, a1, b1)); + } + else if (chan == 3) + { + v_uint8 a0, a1, a2, b0, b1, b2; + v_load_deinterleave(&in1[3 * x], a0, a1, a2); + v_load_deinterleave(&in2[3 * x], b0, b1, b2); + v_store_interleave(&out[3 * x], v_select(m, a0, b0), + v_select(m, a1, b1), + v_select(m, a2, b2)); + } + else if (chan == 4) + { + v_uint8 a0, a1, a2, a3, b0, b1, b2, b3; + v_load_deinterleave(&in1[4 * x], a0, a1, a2, a3); + v_load_deinterleave(&in2[4 * x], b0, b1, b2, b3); + v_store_interleave(&out[4 * x], v_select(m, a0, b0), + v_select(m, a1, b1), + v_select(m, a2, b2), + v_select(m, a3, b3)); + } + } + + if (x < length) { x = length - nlanes; continue; } + break; + } + return x; +} + + +CV_ALWAYS_INLINE int select_simd_impl(const ushort in1[], const ushort in2[], const uchar in3[], + ushort out[], const int length, const int chan) +{ + const int nlanes = VTraits::vlanes(); + if (length < nlanes) return 0; + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_uint16 m = vx_load_expand(&in3[x]); + m = v_ne(m, vx_setzero_u16()); + + if (chan == 1) { + v_uint16 a = vx_load(&in1[x]); + v_uint16 b = vx_load(&in2[x]); + v_store(&out[x], v_select(m, a, b)); + } else if (chan == 2) { + v_uint16 a0, a1, b0, b1; + v_load_deinterleave(&in1[2 * x], a0, a1); + v_load_deinterleave(&in2[2 * x], b0, b1); + v_store_interleave(&out[2 * x], v_select(m, a0, b0), v_select(m, a1, b1)); + } else if (chan == 3) { + v_uint16 a0, a1, a2, b0, b1, b2; + v_load_deinterleave(&in1[3 * x], a0, a1, a2); + v_load_deinterleave(&in2[3 * x], b0, b1, b2); + v_store_interleave(&out[3 * x], v_select(m, a0, b0), v_select(m, a1, b1), v_select(m, a2, b2)); + } else if (chan == 4) { + v_uint16 a0, a1, a2, a3, b0, b1, b2, b3; + v_load_deinterleave(&in1[4 * x], a0, a1, a2, a3); + v_load_deinterleave(&in2[4 * x], b0, b1, b2, b3); + v_store_interleave(&out[4 * x], v_select(m, a0, b0), v_select(m, a1, b1), v_select(m, a2, b2), v_select(m, a3, b3)); + } + } + if (x < length) { x = length - nlanes; continue; } + break; + } + return x; +} + +CV_ALWAYS_INLINE int select_simd_impl(const short in1[], const short in2[], const uchar in3[], + short out[], const int length, const int chan) +{ + const int nlanes = VTraits::vlanes(); + if (length < nlanes) return 0; + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_int16 m = v_reinterpret_as_s16(vx_load_expand(&in3[x])); + m = v_ne(m, vx_setzero_s16()); + + if (chan == 1) { + v_int16 a = vx_load(&in1[x]); + v_int16 b = vx_load(&in2[x]); + v_store(&out[x], v_select(m, a, b)); + } else if (chan == 2) { + v_int16 a0, a1, b0, b1; + v_load_deinterleave(&in1[2 * x], a0, a1); + v_load_deinterleave(&in2[2 * x], b0, b1); + v_store_interleave(&out[2 * x], v_select(m, a0, b0), v_select(m, a1, b1)); + } else if (chan == 3) { + v_int16 a0, a1, a2, b0, b1, b2; + v_load_deinterleave(&in1[3 * x], a0, a1, a2); + v_load_deinterleave(&in2[3 * x], b0, b1, b2); + v_store_interleave(&out[3 * x], v_select(m, a0, b0), v_select(m, a1, b1), v_select(m, a2, b2)); + } else if (chan == 4) { + v_int16 a0, a1, a2, a3, b0, b1, b2, b3; + v_load_deinterleave(&in1[4 * x], a0, a1, a2, a3); + v_load_deinterleave(&in2[4 * x], b0, b1, b2, b3); + v_store_interleave(&out[4 * x], v_select(m, a0, b0), v_select(m, a1, b1), v_select(m, a2, b2), v_select(m, a3, b3)); + } + } + if (x < length) { x = length - nlanes; continue; } + break; + } + return x; +} + + +#define SELECT_SIMD(SRC) \ +int select_simd(const SRC in1[], const SRC in2[], const uchar in3[], \ + SRC out[], const int length, const int chan) \ +{ \ + return select_simd_impl(in1, in2, in3, out, length, chan); \ +} + +SELECT_SIMD(uchar) +SELECT_SIMD(ushort) +SELECT_SIMD(short) + +#undef SELECT_SIMD + #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY CV_CPU_OPTIMIZATION_NAMESPACE_END diff --git a/modules/gapi/test/cpu/gapi_core_tests_fluid.cpp b/modules/gapi/test/cpu/gapi_core_tests_fluid.cpp index b61ff95147..2164cdac46 100644 --- a/modules/gapi/test/cpu/gapi_core_tests_fluid.cpp +++ b/modules/gapi/test/cpu/gapi_core_tests_fluid.cpp @@ -240,10 +240,11 @@ INSTANTIATE_TEST_CASE_P(DISABLED_CropTestFluid, CropTest, Values(cv::Rect(10, 8, 20, 35), cv::Rect(4, 10, 37, 50)))); INSTANTIATE_TEST_CASE_P(SelectTestFluid, SelectTest, - Combine(Values(CV_8UC3, CV_8UC1, CV_16UC1, CV_16SC1), - ValuesIn(in_sizes), - Values(-1), - Values(CORE_FLUID))); + Combine(Values(CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4, + CV_16UC1, CV_16UC3, CV_16SC1, CV_16SC3), + ValuesIn(in_sizes), + Values(-1), + Values(CORE_FLUID))); INSTANTIATE_TEST_CASE_P(Polar2CartFluid, Polar2CartTest, Combine(Values(CV_32FC1),