diff --git a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp index 8284896d6c..2cadd01a0b 100644 --- a/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp +++ b/modules/gapi/perf/cpu/gapi_core_perf_tests_fluid.cpp @@ -237,7 +237,9 @@ INSTANTIATE_TEST_CASE_P(ThresholdPerfTestFluid, ThresholdOTPerfTest, INSTANTIATE_TEST_CASE_P(InRangePerfTestFluid, InRangePerfTest, Combine(Values(AbsExact().to_compare_f()), Values(szSmall128, szVGA, sz720p, sz1080p), - Values(CV_8UC1), + Values(CV_8UC1, CV_8UC2, CV_8UC3, CV_8UC4, + CV_16UC1, CV_16UC3, + CV_16SC1, CV_16SC3), Values(cv::compile_args(CORE_FLUID)))); INSTANTIATE_TEST_CASE_P(Split3PerfTestFluid, Split3PerfTest, diff --git a/modules/gapi/src/backends/fluid/gfluidcore.cpp b/modules/gapi/src/backends/fluid/gfluidcore.cpp index 50615b2652..aeface5046 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore.cpp @@ -2084,32 +2084,26 @@ GAPI_FLUID_KERNEL(GFluidThreshold, cv::gapi::core::GThreshold, false) // //------------------------ -static void run_inrange3(uchar out[], const uchar in[], int width, - const uchar lower[], const uchar upper[]) +#if (CV_SIMD || CV_SIMD_SCALABLE) +template +static CV_ALWAYS_INLINE +typename std::enable_if::value, int>::type +call_inrange_simd(const SRC in[], const SRC lower[], const SRC upper[], + uchar out[], int width, int chan) { - int w = 0; // cycle index - -#if CV_SIMD128 - for (; w <= width-16; w+=16) - { - v_uint8x16 i0, i1, i2; - v_load_deinterleave(&in[3*w], i0, i1, i2); - - v_uint8x16 o; - o = v_and(v_and(v_and(v_and(v_and(v_ge(i0, v_setall_u8(lower[0])), v_le(i0, v_setall_u8(upper[0]))), v_ge(i1, v_setall_u8(lower[1]))), v_le(i1, v_setall_u8(upper[1]))), v_ge(i2, v_setall_u8(lower[2]))), v_le(i2, v_setall_u8(upper[2]))); - - v_store(&out[w], o); - } -#endif - - for (; w < width; w++) - { - out[w] = in[3*w ] >= lower[0] && in[3*w ] <= upper[0] && - in[3*w+1] >= lower[1] && in[3*w+1] <= upper[1] && - in[3*w+2] >= lower[2] && in[3*w+2] <= upper[2] ? 255: 0; - } + return inrange_simd(in, lower, upper, out, width, chan); } +template +static CV_ALWAYS_INLINE +typename std::enable_if::value, int>::type +call_inrange_simd(const SRC[], const SRC[], const SRC[], + uchar[], int, int) +{ + return 0; // float: no SIMD, scalar fallback handles everything +} +#endif + template static void run_inrange(Buffer &dst, const View &src, const cv::Scalar &upperb, const cv::Scalar &lowerb) @@ -2123,7 +2117,7 @@ static void run_inrange(Buffer &dst, const View &src, const cv::Scalar &upperb, int chan = src.meta().chan; GAPI_Assert(dst.meta().chan == 1); - SRC lower[4], upper[4]; + SRC lower[4] = {}, upper[4] = {}; for (int c=0; c < chan; c++) { if (std::is_integral::value) @@ -2145,40 +2139,34 @@ static void run_inrange(Buffer &dst, const View &src, const cv::Scalar &upperb, } } - // manually SIMD for important case if RGB/BGR - if (std::is_same::value && chan==3) - { - run_inrange3((uchar*)out, (const uchar*)in, width, - (const uchar*)lower, (const uchar*)upper); - return; - } - - // TODO: please manually SIMD if multiple channels: - // modern compilers would perfectly vectorize this code if one channel, - // but may need help with de-interleaving channels if RGB/BGR image etc + // SIMD path for integral types (uchar, ushort, short); float falls through +int w = 0; +#if (CV_SIMD || CV_SIMD_SCALABLE) + w = call_inrange_simd(in, lower, upper, out, width, chan); +#endif switch (chan) { case 1: - for (int w=0; w < width; w++) - out[w] = in[w] >= lower[0] && in[w] <= upper[0]? 255: 0; + for (; w < width; w++) + out[w] = in[w] >= lower[0] && in[w] <= upper[0] ? 255 : 0; break; case 2: - for (int w=0; w < width; w++) + for (; w < width; w++) out[w] = in[2*w ] >= lower[0] && in[2*w ] <= upper[0] && - in[2*w+1] >= lower[1] && in[2*w+1] <= upper[1] ? 255: 0; + in[2*w+1] >= lower[1] && in[2*w+1] <= upper[1] ? 255 : 0; break; case 3: - for (int w=0; w < width; w++) + for (; w < width; w++) out[w] = in[3*w ] >= lower[0] && in[3*w ] <= upper[0] && in[3*w+1] >= lower[1] && in[3*w+1] <= upper[1] && - in[3*w+2] >= lower[2] && in[3*w+2] <= upper[2] ? 255: 0; + in[3*w+2] >= lower[2] && in[3*w+2] <= upper[2] ? 255 : 0; break; case 4: - for (int w=0; w < width; w++) + for (; w < width; w++) out[w] = in[4*w ] >= lower[0] && in[4*w ] <= upper[0] && in[4*w+1] >= lower[1] && in[4*w+1] <= upper[1] && in[4*w+2] >= lower[2] && in[4*w+2] <= upper[2] && - in[4*w+3] >= lower[3] && in[4*w+3] <= upper[3] ? 255: 0; + in[4*w+3] >= lower[3] && in[4*w+3] <= upper[3] ? 255 : 0; break; default: CV_Error(cv::Error::StsBadArg, "unsupported number of channels"); } diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp index a0ef4b1479..10e88ea44b 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.dispatch.cpp @@ -400,6 +400,26 @@ CONVERTTO_SCALED_SIMD(float, float) #undef CONVERTTO_SCALED_SIMD +//------------------------- +// +// Fluid kernels: InRange +// +//------------------------- + +#define INRANGE_SIMD(SRC) \ +int inrange_simd(const SRC in[], const SRC lower[], const SRC upper[], \ + uchar out[], const int length, const int chan) \ +{ \ + CV_CPU_DISPATCH(inrange_simd, (in, lower, upper, out, length, chan), \ + CV_CPU_DISPATCH_MODES_ALL); \ +} + +INRANGE_SIMD(uchar) +INRANGE_SIMD(ushort) +INRANGE_SIMD(short) + +#undef INRANGE_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp index 0186ea020e..28bffc5bc5 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.hpp @@ -315,6 +315,22 @@ CONVERTTO_SCALED_SIMD(float, float) #undef CONVERTTO_SCALED_SIMD +//------------------------- +// +// Fluid kernels: InRange +// +//------------------------- + +#define INRANGE_SIMD(SRC) \ +int inrange_simd(const SRC in[], const SRC lower[], const SRC upper[], \ + uchar out[], const int length, const int chan); + +INRANGE_SIMD(uchar) +INRANGE_SIMD(ushort) +INRANGE_SIMD(short) + +#undef INRANGE_SIMD + } // namespace fluid } // namespace gapi } // namespace cv diff --git a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp index dead5e30c2..10e332264e 100644 --- a/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp +++ b/modules/gapi/src/backends/fluid/gfluidcore_func.simd.hpp @@ -322,6 +322,16 @@ int split3_simd(const uchar in[], uchar out1[], uchar out2[], int split4_simd(const uchar in[], uchar out1[], uchar out2[], uchar out3[], uchar out4[], const int width); +#define INRANGE_SIMD(SRC) \ +int inrange_simd(const SRC in[], const SRC lower[], const SRC upper[], \ + uchar out[], const int length, const int chan); + +INRANGE_SIMD(uchar) +INRANGE_SIMD(ushort) +INRANGE_SIMD(short) + +#undef INRANGE_SIMD + #define MERGE3_SIMD(T) \ int merge3_simd(const T in1[], const T in2[], const T in3[], \ T out[], const int width); @@ -3154,6 +3164,221 @@ CONVERTTO_SCALED_SIMD(float, float) #undef CONVERTTO_SCALED_SIMD +//------------------------- +// +// Fluid kernels: InRange +// +//------------------------- + +CV_ALWAYS_INLINE int inrange_simd_impl(const uchar in[], + const uchar lower[], const uchar upper[], + uchar out[], const int length, const int chan) +{ + const int nlanes = VTraits::vlanes(); + + if (length < nlanes) + return 0; + + // broadcast scalar bounds into vectors + v_uint8 vlo0 = vx_setall_u8(lower[0]), vhi0 = vx_setall_u8(upper[0]); + v_uint8 vlo1 = vx_setzero_u8(), vhi1 = vx_setzero_u8(); + v_uint8 vlo2 = vx_setzero_u8(), vhi2 = vx_setzero_u8(); + v_uint8 vlo3 = vx_setzero_u8(), vhi3 = vx_setzero_u8(); + if (chan >= 2) { vlo1 = vx_setall_u8(lower[1]); vhi1 = vx_setall_u8(upper[1]); } + if (chan >= 3) { vlo2 = vx_setall_u8(lower[2]); vhi2 = vx_setall_u8(upper[2]); } + if (chan == 4) { vlo3 = vx_setall_u8(lower[3]); vhi3 = vx_setall_u8(upper[3]); } + + int x = 0; // pixel index + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_uint8 mask; + if (chan == 1) + { + v_uint8 a = vx_load(&in[x]); + mask = v_and(v_ge(a, vlo0), v_le(a, vhi0)); + } + else if (chan == 2) + { + v_uint8 a, b; + v_load_deinterleave(&in[2 * x], a, b); + mask = v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))); + } + else if (chan == 3) + { + v_uint8 a, b, c; + v_load_deinterleave(&in[3 * x], a, b, c); + mask = v_and(v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))), + v_and(v_ge(c, vlo2), v_le(c, vhi2))); + } + else // chan == 4 + { + v_uint8 a, b, c, d; + v_load_deinterleave(&in[4 * x], a, b, c, d); + mask = v_and(v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))), + v_and(v_and(v_ge(c, vlo2), v_le(c, vhi2)), + v_and(v_ge(d, vlo3), v_le(d, vhi3)))); + } + vx_store(&out[x], mask); + } + if (x < length) { x = length - nlanes; continue; } + break; + } + return x; +} + +CV_ALWAYS_INLINE int inrange_simd_impl(const ushort in[], + const ushort lower[], const ushort upper[], + uchar out[], const int length, const int chan) +{ + const int nlanes = VTraits::vlanes(); // output pixels per iter + const int nlanes16 = VTraits::vlanes(); // = nlanes / 2 + + if (length < nlanes) + return 0; + + v_uint16 vlo0 = vx_setall_u16(lower[0]), vhi0 = vx_setall_u16(upper[0]); + v_uint16 vlo1 = vx_setzero_u16(), vhi1 = vx_setzero_u16(); + v_uint16 vlo2 = vx_setzero_u16(), vhi2 = vx_setzero_u16(); + v_uint16 vlo3 = vx_setzero_u16(), vhi3 = vx_setzero_u16(); + if (chan >= 2) { vlo1 = vx_setall_u16(lower[1]); vhi1 = vx_setall_u16(upper[1]); } + if (chan >= 3) { vlo2 = vx_setall_u16(lower[2]); vhi2 = vx_setall_u16(upper[2]); } + if (chan == 4) { vlo3 = vx_setall_u16(lower[3]); vhi3 = vx_setall_u16(upper[3]); } + + // Helper lambda: compute mask for nlanes16 pixels starting at in[base*chan] + auto cmp16 = [&](int base) -> v_uint16 + { + if (chan == 1) + { + v_uint16 a = vx_load(&in[base]); + return v_and(v_ge(a, vlo0), v_le(a, vhi0)); + } + else if (chan == 2) + { + v_uint16 a, b; + v_load_deinterleave(&in[2 * base], a, b); + return v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))); + } + else if (chan == 3) + { + v_uint16 a, b, c; + v_load_deinterleave(&in[3 * base], a, b, c); + return v_and(v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))), + v_and(v_ge(c, vlo2), v_le(c, vhi2))); + } + else // chan == 4 + { + v_uint16 a, b, c, d; + v_load_deinterleave(&in[4 * base], a, b, c, d); + return v_and(v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))), + v_and(v_and(v_ge(c, vlo2), v_le(c, vhi2)), + v_and(v_ge(d, vlo3), v_le(d, vhi3)))); + } + }; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_uint16 m1 = cmp16(x); + v_uint16 m2 = cmp16(x + nlanes16); + pack_store_uchar(&out[x], m1, m2); + } + if (x < length) { x = length - nlanes; continue; } + break; + } + return x; +} + +CV_ALWAYS_INLINE int inrange_simd_impl(const short in[], + const short lower[], const short upper[], + uchar out[], const int length, const int chan) +{ + const int nlanes = VTraits::vlanes(); + const int nlanes16 = VTraits::vlanes(); + + if (length < nlanes) + return 0; + + v_int16 vlo0 = vx_setall_s16(lower[0]), vhi0 = vx_setall_s16(upper[0]); + v_int16 vlo1 = vx_setzero_s16(), vhi1 = vx_setzero_s16(); + v_int16 vlo2 = vx_setzero_s16(), vhi2 = vx_setzero_s16(); + v_int16 vlo3 = vx_setzero_s16(), vhi3 = vx_setzero_s16(); + if (chan >= 2) { vlo1 = vx_setall_s16(lower[1]); vhi1 = vx_setall_s16(upper[1]); } + if (chan >= 3) { vlo2 = vx_setall_s16(lower[2]); vhi2 = vx_setall_s16(upper[2]); } + if (chan == 4) { vlo3 = vx_setall_s16(lower[3]); vhi3 = vx_setall_s16(upper[3]); } + + auto cmp16 = [&](int base) -> v_int16 + { + if (chan == 1) + { + v_int16 a = vx_load(&in[base]); + return v_and(v_ge(a, vlo0), v_le(a, vhi0)); + } + else if (chan == 2) + { + v_int16 a, b; + v_load_deinterleave(&in[2 * base], a, b); + return v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))); + } + else if (chan == 3) + { + v_int16 a, b, c; + v_load_deinterleave(&in[3 * base], a, b, c); + return v_and(v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))), + v_and(v_ge(c, vlo2), v_le(c, vhi2))); + } + else // chan == 4 + { + v_int16 a, b, c, d; + v_load_deinterleave(&in[4 * base], a, b, c, d); + return v_and(v_and(v_and(v_ge(a, vlo0), v_le(a, vhi0)), + v_and(v_ge(b, vlo1), v_le(b, vhi1))), + v_and(v_and(v_ge(c, vlo2), v_le(c, vhi2)), + v_and(v_ge(d, vlo3), v_le(d, vhi3)))); + } + }; + + int x = 0; + for (;;) + { + for (; x <= length - nlanes; x += nlanes) + { + v_int16 m1 = cmp16(x); + v_int16 m2 = cmp16(x + nlanes16); + v_uint16 um1 = v_reinterpret_as_u16(m1); + v_uint16 um2 = v_reinterpret_as_u16(m2); + pack_store_uchar(&out[x], um1, um2); + } + if (x < length) { x = length - nlanes; continue; } + break; + } + return x; +} + +#define INRANGE_SIMD(SRC) \ +int inrange_simd(const SRC in[], const SRC lower[], const SRC upper[], \ + uchar out[], const int length, const int chan) \ +{ \ + return inrange_simd_impl(in, lower, upper, out, length, chan); \ +} + +INRANGE_SIMD(uchar) +INRANGE_SIMD(ushort) +INRANGE_SIMD(short) + +#undef INRANGE_SIMD + #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY CV_CPU_OPTIMIZATION_NAMESPACE_END