diff --git a/3rdparty/carotene/hal/tegra_hal.hpp b/3rdparty/carotene/hal/tegra_hal.hpp index 2dfa30b619..7fcca975fe 100644 --- a/3rdparty/carotene/hal/tegra_hal.hpp +++ b/3rdparty/carotene/hal/tegra_hal.hpp @@ -1778,30 +1778,30 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. : CV_HAL_ERROR_NOT_IMPLEMENTED \ ) -#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ +#define TEGRA_CVT2PYUVTOBGR_EX(y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ ( \ CAROTENE_NS::isSupportedConfiguration() ? \ dcn == 3 ? \ uIdx == 0 ? \ (swapBlue ? \ CAROTENE_NS::yuv420i2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420i2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ uIdx == 1 ? \ (swapBlue ? \ CAROTENE_NS::yuv420sp2rgb(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420sp2bgr(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ CV_HAL_ERROR_NOT_IMPLEMENTED : \ @@ -1809,29 +1809,32 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. uIdx == 0 ? \ (swapBlue ? \ CAROTENE_NS::yuv420i2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420i2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ uIdx == 1 ? \ (swapBlue ? \ CAROTENE_NS::yuv420sp2rgbx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step) : \ CAROTENE_NS::yuv420sp2bgrx(CAROTENE_NS::Size2D(dst_width, dst_height), \ - src_data, src_step, \ - src_data + src_step * dst_height, src_step, \ + y_data, y_step, \ + uv_data, uv_step, \ dst_data, dst_step)), \ CV_HAL_ERROR_OK : \ CV_HAL_ERROR_NOT_IMPLEMENTED : \ CV_HAL_ERROR_NOT_IMPLEMENTED \ : CV_HAL_ERROR_NOT_IMPLEMENTED \ ) +#define TEGRA_CVT2PYUVTOBGR(src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx) \ + TEGRA_CVT2PYUVTOBGR_EX(src_data, src_step, src_data + src_step * dst_height, src_step, dst_data, dst_step, \ + dst_width, dst_height, dcn, swapBlue, uIdx); #undef cv_hal_cvtBGRtoBGR #define cv_hal_cvtBGRtoBGR TEGRA_CVTBGRTOBGR @@ -1847,6 +1850,8 @@ TegraCvtColor_Invoker(bgrx2hsvf, bgrx2hsv, src_data + static_cast(range. #define cv_hal_cvtBGRtoHSV TEGRA_CVTBGRTOHSV #undef cv_hal_cvtTwoPlaneYUVtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR TEGRA_CVT2PYUVTOBGR +#undef cv_hal_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtTwoPlaneYUVtoBGREx TEGRA_CVT2PYUVTOBGR_EX #endif // OPENCV_IMGPROC_HAL_INTERFACE_H diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index 764e4b19d8..2eea658178 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/master_20210608 -# Binaries were created for OpenCV: eaa9228a4fdfb9c2465aea65a50ce2d16b55dce0 -ocv_update(FFMPEG_BINARIES_COMMIT "213fcd5d4897319a83207406036c4a5957fba010") -ocv_update(FFMPEG_FILE_HASH_BIN32 "bab661341c30862fa88627130219c0a5") -ocv_update(FFMPEG_FILE_HASH_BIN64 "ac99f9767a83103c31709628af685924") +# Binaries branch name: ffmpeg/master_20211005 +# Binaries were created for OpenCV: 672399c751c431bbe52818b33fd3ca17b51e0e16 +ocv_update(FFMPEG_BINARIES_COMMIT "40b4666d1aa374205fd61373496e15d92ecd5313") +ocv_update(FFMPEG_FILE_HASH_BIN32 "c2f9a897d464a2dce2286f8067ad9d90") +ocv_update(FFMPEG_FILE_HASH_BIN64 "878a4e8fe5a4d68f18c9cdde543b9ead") ocv_update(FFMPEG_FILE_HASH_CMAKE "8862c87496e2e8c375965e1277dee1c7") function(download_win_ffmpeg script_var) diff --git a/3rdparty/libwebp/CMakeLists.txt b/3rdparty/libwebp/CMakeLists.txt index 80ab0b86ab..9160e2024c 100644 --- a/3rdparty/libwebp/CMakeLists.txt +++ b/3rdparty/libwebp/CMakeLists.txt @@ -32,7 +32,9 @@ endif() # Define the library target: # ---------------------------------------------------------------------------------- -add_definitions(-DWEBP_USE_THREAD) +if(NOT OPENCV_DISABLE_THREAD_SUPPORT) + add_definitions(-DWEBP_USE_THREAD) +endif() add_library(${WEBP_LIBRARY} STATIC ${OPENCV_3RDPARTY_EXCLUDE_FROM_ALL} ${lib_srcs} ${lib_hdrs}) if(ANDROID) diff --git a/3rdparty/openvx/hal/openvx_hal.cpp b/3rdparty/openvx/hal/openvx_hal.cpp index d923b28611..b87c38706e 100644 --- a/3rdparty/openvx/hal/openvx_hal.cpp +++ b/3rdparty/openvx/hal/openvx_hal.cpp @@ -923,6 +923,11 @@ int ovx_hal_cvtGraytoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, } int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx) +{ + return ovx_hal_cvtTwoPlaneYUVtoBGREx(a, astep, a + h * astep, astep, b, bstep, w, h, bcn, swapBlue, uIdx); +} + +int ovx_hal_cvtTwoPlaneYUVtoBGREx(const uchar * a, size_t astep, const uchar * b, size_t bstep, uchar * c, size_t cstep, int w, int h, int bcn, bool swapBlue, int uIdx) { if (skipSmallImages(w, h)) return CV_HAL_ERROR_NOT_IMPLEMENTED; @@ -933,8 +938,7 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t if (w & 1 || h & 1) // It's not described in spec but sample implementation unable to convert odd sized images return CV_HAL_ERROR_NOT_IMPLEMENTED; - refineStep(w, h, uIdx ? VX_DF_IMAGE_NV21 : VX_DF_IMAGE_NV12, astep); - refineStep(w, h, bcn == 3 ? VX_DF_IMAGE_RGB : VX_DF_IMAGE_RGBX, bstep); + try { ivx::Context ctx = getOpenVXHALContext(); @@ -943,8 +947,8 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t std::vector ptrs; addr.push_back(ivx::Image::createAddressing(w, h, 1, (vx_int32)astep)); ptrs.push_back((void*)a); - addr.push_back(ivx::Image::createAddressing(w / 2, h / 2, 2, (vx_int32)astep)); - ptrs.push_back((void*)(a + h * astep)); + addr.push_back(ivx::Image::createAddressing(w / 2, h / 2, 2, (vx_int32)bstep)); + ptrs.push_back((void*)b); vxImage ia = ivx::Image::createFromHandle(ctx, uIdx ? VX_DF_IMAGE_NV21 : VX_DF_IMAGE_NV12, addr, ptrs); @@ -952,7 +956,7 @@ int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t return CV_HAL_ERROR_NOT_IMPLEMENTED; // OpenCV store NV12/NV21 as RANGE_RESTRICTED while OpenVX expect RANGE_FULL vxImage ib = ivx::Image::createFromHandle(ctx, bcn == 3 ? VX_DF_IMAGE_RGB : VX_DF_IMAGE_RGBX, - ivx::Image::createAddressing(w, h, bcn, (vx_int32)bstep), b); + ivx::Image::createAddressing(w, h, bcn, (vx_int32)cstep), c); ivx::IVX_CHECK_STATUS(vxuColorConvert(ctx, ia, ib)); } catch (ivx::RuntimeError & e) diff --git a/3rdparty/openvx/hal/openvx_hal.hpp b/3rdparty/openvx/hal/openvx_hal.hpp index 98bb4dc43e..4fba576f1a 100644 --- a/3rdparty/openvx/hal/openvx_hal.hpp +++ b/3rdparty/openvx/hal/openvx_hal.hpp @@ -49,6 +49,7 @@ int ovx_hal_morph(cvhalFilter2D *filter_context, uchar *a, size_t astep, uchar * int ovx_hal_cvtBGRtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int depth, int acn, int bcn, bool swapBlue); int ovx_hal_cvtGraytoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int depth, int bcn); int ovx_hal_cvtTwoPlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx); +int ovx_hal_cvtTwoPlaneYUVtoBGREx(const uchar * a, size_t astep, const uchar * b, size_t bstep, uchar * c, size_t cstep, int w, int h, int bcn, bool swapBlue, int uIdx); int ovx_hal_cvtThreePlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx); int ovx_hal_cvtBGRtoThreePlaneYUV(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int acn, bool swapBlue, int uIdx); int ovx_hal_cvtOnePlaneYUVtoBGR(const uchar * a, size_t astep, uchar * b, size_t bstep, int w, int h, int bcn, bool swapBlue, int uIdx, int ycn); @@ -130,6 +131,8 @@ int ovx_hal_integral(int depth, int sdepth, int, const uchar * a, size_t astep, #define cv_hal_cvtGraytoBGR ovx_hal_cvtGraytoBGR #undef cv_hal_cvtTwoPlaneYUVtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR ovx_hal_cvtTwoPlaneYUVtoBGR +#undef cv_hal_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtTwoPlaneYUVtoBGREx ovx_hal_cvtTwoPlaneYUVtoBGREx #undef cv_hal_cvtThreePlaneYUVtoBGR #define cv_hal_cvtThreePlaneYUVtoBGR ovx_hal_cvtThreePlaneYUVtoBGR #undef cv_hal_cvtBGRtoThreePlaneYUV diff --git a/3rdparty/readme.txt b/3rdparty/readme.txt index 3b961782b0..4e4a6ba0a6 100644 --- a/3rdparty/readme.txt +++ b/3rdparty/readme.txt @@ -31,7 +31,7 @@ libpng Portable Network Graphics library. libtiff Tag Image File Format (TIFF) Software Copyright (c) 1988-1997 Sam Leffler Copyright (c) 1991-1997 Silicon Graphics, Inc. - See libtiff home page http://www.remotesensing.org/libtiff/ + See libtiff home page http://www.libtiff.org/ for details and links to the source code WITH_TIFF CMake option must be ON to add libtiff & zlib support to imgcodecs. @@ -51,7 +51,9 @@ jasper JasPer is a collection of software Copyright (c) 1999-2000 The University of British Columbia Copyright (c) 2001-2003 Michael David Adams - The JasPer license can be found in libjasper. + See JasPer official GitHub repository + https://github.com/jasper-software/jasper.git + for details and links to source code ------------------------------------------------------------------------------------ openexr OpenEXR is a high dynamic-range (HDR) image file format developed by Industrial Light & Magic for use in computer imaging applications. diff --git a/CMakeLists.txt b/CMakeLists.txt index be47c27f4a..690803e53a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -513,6 +513,8 @@ OCV_OPTION(OPENCV_GENERATE_SETUPVARS "Generate setup_vars* scripts" ON IF (NOT OCV_OPTION(ENABLE_CONFIG_VERIFICATION "Fail build if actual configuration doesn't match requested (WITH_XXX != HAVE_XXX)" OFF) OCV_OPTION(OPENCV_ENABLE_MEMALIGN "Enable posix_memalign or memalign usage" ON) OCV_OPTION(OPENCV_DISABLE_FILESYSTEM_SUPPORT "Disable filesystem support" OFF) +OCV_OPTION(OPENCV_DISABLE_THREAD_SUPPORT "Build the library without multi-threaded code." OFF) +OCV_OPTION(OPENCV_SEMIHOSTING "Build the library for semihosting target (Arm). See https://developer.arm.com/documentation/100863/latest." OFF) OCV_OPTION(ENABLE_PYLINT "Add target with Pylint checks" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) OCV_OPTION(ENABLE_FLAKE8 "Add target with Python flake8 checker" (BUILD_DOCS OR BUILD_EXAMPLES) IF (NOT CMAKE_CROSSCOMPILING AND NOT APPLE_FRAMEWORK) ) @@ -662,7 +664,7 @@ if(UNIX) elseif(EMSCRIPTEN) # no need to link to system libs with emscripten elseif(QNXNTO) - # no need to link to system libs with QNX + set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} m) else() set(OPENCV_LINKER_LIBS ${OPENCV_LINKER_LIBS} dl m pthread rt) endif() @@ -670,6 +672,11 @@ if(UNIX) set(HAVE_PTHREAD 1) endif() + # Ensure that libpthread is not listed as one of the libraries to pass to the linker. + if (OPENCV_DISABLE_THREAD_SUPPORT) + list(REMOVE_ITEM OPENCV_LINKER_LIBS pthread) + endif() + if(OPENCV_ENABLE_MEMALIGN) CHECK_SYMBOL_EXISTS(posix_memalign stdlib.h HAVE_POSIX_MEMALIGN) CHECK_INCLUDE_FILE(malloc.h HAVE_MALLOC_H) @@ -914,7 +921,7 @@ add_subdirectory(include) ocv_add_modules_compiler_options() # OpenCV modules -add_subdirectory(modules) +ocv_register_modules() # Generate targets for documentation add_subdirectory(doc) @@ -1244,15 +1251,20 @@ endif(WIN32) # ========================== GUI ========================== status("") -status(" GUI: ") +status(" GUI: " "${OPENCV_HIGHGUI_BUILTIN_BACKEND}") if(WITH_QT OR HAVE_QT) - if(HAVE_QT5) - status(" QT:" "YES (ver ${Qt5Core_VERSION_STRING})") - status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${Qt5OpenGL_LIBRARIES} ${Qt5OpenGL_VERSION_STRING})" ELSE NO) - elseif(HAVE_QT) + if(HAVE_QT) status(" QT:" "YES (ver ${QT_VERSION_MAJOR}.${QT_VERSION_MINOR}.${QT_VERSION_PATCH} ${QT_EDITION})") - status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${QT_QTOPENGL_LIBRARY})" ELSE NO) + if(HAVE_QT_OPENGL) + if(Qt${QT_VERSION_MAJOR}OpenGL_LIBRARIES) + status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${Qt${QT_VERSION_MAJOR}OpenGL_LIBRARIES} ${Qt${QT_VERSION_MAJOR}OpenGL_VERSION_STRING})" ELSE NO) + else() + status(" QT OpenGL support:" HAVE_QT_OPENGL THEN "YES (${QT_QTOPENGL_LIBRARY})" ELSE NO) + endif() + else() + status(" QT OpenGL support:" "NO") + endif() else() status(" QT:" "NO") endif() @@ -1470,6 +1482,15 @@ ocv_build_features_string(parallel_status EXCLUSIVE ELSE "none") status("") status(" Parallel framework:" "${parallel_status}") +if (OPENCV_DISABLE_THREAD_SUPPORT) + status("" "Multi thread code explicitly disabled with OPENCV_DISABLE_THREAD_SUPPORT.") + if(HAVE_PTHREADS_PF OR HAVE_HPX OR HAVE_OPENMP OR HAVE_GCD OR HAVE_CONCURRENCY) + message(FATAL_ERROR "Not all parallel frameworks have been disabled (using ${parallel_status}).") + endif() + if(HAVE_PTHREAD) + message(FATAL_ERROR "Thread execution might be in use in some component.") + endif() +endif() if(CV_TRACE OR OPENCV_TRACE) ocv_build_features_string(trace_status EXCLUSIVE @@ -1521,6 +1542,7 @@ if(WITH_INF_ENGINE OR INF_ENGINE_TARGET) if(INF_ENGINE_TARGET) list(GET INF_ENGINE_TARGET 0 ie_target) set(__msg "YES (${INF_ENGINE_RELEASE} / ${INF_ENGINE_VERSION})") + ocv_get_imported_target(ie_target "${ie_target}") get_target_property(_lib ${ie_target} IMPORTED_LOCATION) get_target_property(_lib_imp_rel ${ie_target} IMPORTED_IMPLIB_RELEASE) get_target_property(_lib_imp_dbg ${ie_target} IMPORTED_IMPLIB_DEBUG) @@ -1544,7 +1566,7 @@ if(WITH_INF_ENGINE OR INF_ENGINE_TARGET) endif() if(WITH_NGRAPH OR HAVE_NGRAPH) if(HAVE_NGRAPH) - set(__target ngraph::ngraph) + ocv_get_imported_target(__target ngraph::ngraph) set(__msg "YES (${ngraph_VERSION})") get_target_property(_lib ${__target} IMPORTED_LOCATION) get_target_property(_lib_imp_rel ${__target} IMPORTED_IMPLIB_RELEASE) diff --git a/apps/model-diagnostics/model_diagnostics.cpp b/apps/model-diagnostics/model_diagnostics.cpp index 2ffeaa1ea5..6970c85071 100644 --- a/apps/model-diagnostics/model_diagnostics.cpp +++ b/apps/model-diagnostics/model_diagnostics.cpp @@ -1,9 +1,10 @@ /************************************************* USAGE: -./model_diagnostics -m +./model_diagnostics -m **************************************************/ #include #include +#include #include @@ -32,7 +33,7 @@ static std::string checkFileExists(const std::string& fileName) } std::string diagnosticKeys = - "{ model m | | Path to the model .onnx file. }" + "{ model m | | Path to the model file. }" "{ config c | | Path to the model configuration file. }" "{ framework f | | [Optional] Name of the model framework. }"; @@ -41,7 +42,7 @@ std::string diagnosticKeys = int main( int argc, const char** argv ) { CommandLineParser argParser(argc, argv, diagnosticKeys); - argParser.about("Use this tool to run the diagnostics of provided ONNX model" + argParser.about("Use this tool to run the diagnostics of provided ONNX/TF model" "to obtain the information about its support (supported layers)."); if (argc == 1) @@ -57,6 +58,7 @@ int main( int argc, const char** argv ) CV_Assert(!model.empty()); enableModelDiagnostics(true); + skipModelImport(true); redirectError(diagnosticsErrorCallback, NULL); Net ocvNet = readNet(model, config, frameworkId); diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index a161b6eb8b..037c7fb5ba 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -178,8 +178,17 @@ if(CV_GCC OR CV_CLANG) add_extra_compiler_option(-Wno-long-long) endif() - # We need pthread's - if(UNIX AND NOT ANDROID AND NOT (APPLE AND CV_CLANG)) # TODO + # We need pthread's, unless we have explicitly disabled multi-thread execution. + if(NOT OPENCV_DISABLE_THREAD_SUPPORT + AND ( + (UNIX + AND NOT ANDROID + AND NOT (APPLE AND CV_CLANG) + AND NOT EMSCRIPTEN + ) + OR (EMSCRIPTEN AND WITH_PTHREADS_PF) # https://github.com/opencv/opencv/issues/20285 + ) + ) # TODO add_extra_compiler_option(-pthread) endif() @@ -398,6 +407,9 @@ if(MSVC) endif() endif() + # Enable [[attribute]] syntax checking to prevent silent failure: "attribute is ignored in this syntactic position" + add_extra_compiler_option("/w15240") + if(NOT ENABLE_NOISY_WARNINGS) ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127) # conditional expression is constant ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4251) # class 'std::XXX' needs to have dll-interface to be used by clients of YYY diff --git a/cmake/OpenCVDetectCUDA.cmake b/cmake/OpenCVDetectCUDA.cmake index d12a9e68ea..10f1288141 100644 --- a/cmake/OpenCVDetectCUDA.cmake +++ b/cmake/OpenCVDetectCUDA.cmake @@ -112,7 +112,7 @@ if(CUDA_FOUND) if(CUDA_GENERATION) if(NOT ";${_generations};" MATCHES ";${CUDA_GENERATION};") string(REPLACE ";" ", " _generations "${_generations}") - message(FATAL_ERROR "ERROR: ${_generations} Generations are suppered.") + message(FATAL_ERROR "ERROR: ${_generations} Generations are supported.") endif() unset(CUDA_ARCH_BIN CACHE) unset(CUDA_ARCH_PTX CACHE) diff --git a/cmake/OpenCVDetectCXXCompiler.cmake b/cmake/OpenCVDetectCXXCompiler.cmake index 1c30cccf4b..7f229cde96 100644 --- a/cmake/OpenCVDetectCXXCompiler.cmake +++ b/cmake/OpenCVDetectCXXCompiler.cmake @@ -171,6 +171,8 @@ elseif(MSVC) set(OpenCV_RUNTIME vc15) elseif(MSVC_VERSION MATCHES "^192[0-9]$") set(OpenCV_RUNTIME vc16) + elseif(MSVC_VERSION MATCHES "^193[0-9]$") + set(OpenCV_RUNTIME vc17) else() message(WARNING "OpenCV does not recognize MSVC_VERSION \"${MSVC_VERSION}\". Cannot set OpenCV_RUNTIME") endif() diff --git a/cmake/OpenCVDetectHalide.cmake b/cmake/OpenCVDetectHalide.cmake index 790f692056..4828c299ae 100644 --- a/cmake/OpenCVDetectHalide.cmake +++ b/cmake/OpenCVDetectHalide.cmake @@ -9,9 +9,14 @@ set(HALIDE_ROOT_DIR "${HALIDE_ROOT_DIR}" CACHE PATH "Halide root directory") if(NOT HAVE_HALIDE) find_package(Halide QUIET) # Try CMake-based config files if(Halide_FOUND) - set(HALIDE_INCLUDE_DIRS "${Halide_INCLUDE_DIRS}" CACHE PATH "Halide include directories" FORCE) - set(HALIDE_LIBRARIES "${Halide_LIBRARIES}" CACHE PATH "Halide libraries" FORCE) - set(HAVE_HALIDE TRUE) + if(TARGET Halide::Halide) # modern Halide scripts defines imported target + set(HALIDE_INCLUDE_DIRS "") + set(HALIDE_LIBRARIES "Halide::Halide") + set(HAVE_HALIDE TRUE) + else() + # using HALIDE_INCLUDE_DIRS / Halide_LIBRARIES + set(HAVE_HALIDE TRUE) + endif() endif() endif() @@ -28,18 +33,15 @@ if(NOT HAVE_HALIDE AND HALIDE_ROOT_DIR) ) if(HALIDE_LIBRARY AND HALIDE_INCLUDE_DIR) # TODO try_compile - set(HALIDE_INCLUDE_DIRS "${HALIDE_INCLUDE_DIR}" CACHE PATH "Halide include directories" FORCE) - set(HALIDE_LIBRARIES "${HALIDE_LIBRARY}" CACHE PATH "Halide libraries" FORCE) + set(HALIDE_INCLUDE_DIRS "${HALIDE_INCLUDE_DIR}") + set(HALIDE_LIBRARIES "${HALIDE_LIBRARY}") set(HAVE_HALIDE TRUE) endif() - if(NOT HAVE_HALIDE) - ocv_clear_vars(HALIDE_LIBRARIES HALIDE_INCLUDE_DIRS CACHE) - endif() endif() if(HAVE_HALIDE) - include_directories(${HALIDE_INCLUDE_DIRS}) + if(HALIDE_INCLUDE_DIRS) + include_directories(${HALIDE_INCLUDE_DIRS}) + endif() list(APPEND OPENCV_LINKER_LIBS ${HALIDE_LIBRARIES}) -else() - ocv_clear_vars(HALIDE_INCLUDE_DIRS HALIDE_LIBRARIES) endif() diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake index 2c0296d634..f77778eb18 100644 --- a/cmake/OpenCVDetectInferenceEngine.cmake +++ b/cmake/OpenCVDetectInferenceEngine.cmake @@ -99,6 +99,20 @@ if(InferenceEngine_FOUND) message(STATUS "Detected InferenceEngine: cmake package (${InferenceEngine_VERSION})") endif() +if(DEFINED InferenceEngine_VERSION) + message(STATUS "InferenceEngine: ${InferenceEngine_VERSION}") + if(NOT INF_ENGINE_RELEASE AND NOT (InferenceEngine_VERSION VERSION_LESS "2021.4")) + math(EXPR INF_ENGINE_RELEASE_INIT "${InferenceEngine_VERSION_MAJOR} * 1000000 + ${InferenceEngine_VERSION_MINOR} * 10000 + ${InferenceEngine_VERSION_PATCH} * 100") + endif() +endif() +if(NOT INF_ENGINE_RELEASE AND NOT INF_ENGINE_RELEASE_INIT) + message(STATUS "WARNING: InferenceEngine version has not been set, 2021.4.1 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") + set(INF_ENGINE_RELEASE_INIT "2021040100") +elseif(DEFINED INF_ENGINE_RELEASE) + set(INF_ENGINE_RELEASE_INIT "${INF_ENGINE_RELEASE}") +endif() +set(INF_ENGINE_RELEASE "${INF_ENGINE_RELEASE_INIT}" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") + if(NOT INF_ENGINE_TARGET AND INF_ENGINE_LIB_DIRS AND INF_ENGINE_INCLUDE_DIRS) find_path(ie_custom_inc "inference_engine.hpp" PATHS "${INF_ENGINE_INCLUDE_DIRS}" NO_DEFAULT_PATH) if(CMAKE_BUILD_TYPE STREQUAL "Debug") @@ -134,16 +148,8 @@ endif() # Add more features to the target if(INF_ENGINE_TARGET) - if(InferenceEngine_VERSION VERSION_GREATER_EQUAL "2021.4") - math(EXPR INF_ENGINE_RELEASE "${InferenceEngine_VERSION_MAJOR} * 1000000 + ${InferenceEngine_VERSION_MINOR} * 10000 + ${InferenceEngine_VERSION_PATCH} * 100") - endif() - if(NOT INF_ENGINE_RELEASE) - message(WARNING "InferenceEngine version has not been set, 2021.3 will be used by default. Set INF_ENGINE_RELEASE variable if you experience build errors.") - set(INF_ENGINE_RELEASE "2021030000") - endif() - set(INF_ENGINE_RELEASE "${INF_ENGINE_RELEASE}" CACHE STRING "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") set_target_properties(${INF_ENGINE_TARGET} PROPERTIES - INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" + INTERFACE_COMPILE_DEFINITIONS "HAVE_INF_ENGINE=1;INF_ENGINE_RELEASE=${INF_ENGINE_RELEASE}" ) endif() diff --git a/cmake/OpenCVDetectVTK.cmake b/cmake/OpenCVDetectVTK.cmake index b8cf36007c..312fd41fe4 100644 --- a/cmake/OpenCVDetectVTK.cmake +++ b/cmake/OpenCVDetectVTK.cmake @@ -1,34 +1,34 @@ -# VTK 9.0 if(NOT VTK_FOUND) - find_package(VTK 9 QUIET NAMES vtk COMPONENTS - FiltersExtraction - FiltersSources - FiltersTexture - IOExport - IOGeometry - IOPLY - InteractionStyle - RenderingCore - RenderingLOD - RenderingOpenGL2 - NO_MODULE) -endif() - -# VTK 6.x components -if(NOT VTK_FOUND) - find_package(VTK QUIET COMPONENTS vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) - IF(VTK_FOUND) - IF(VTK_RENDERING_BACKEND) #in vtk 7, the rendering backend is exported as a var. - find_package(VTK QUIET COMPONENTS vtkRendering${VTK_RENDERING_BACKEND} vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport vtkIOGeometry NO_MODULE) - ELSE(VTK_RENDERING_BACKEND) - find_package(VTK QUIET COMPONENTS vtkRenderingOpenGL vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) - ENDIF(VTK_RENDERING_BACKEND) - ENDIF(VTK_FOUND) -endif() - -# VTK 5.x components -if(NOT VTK_FOUND) - find_package(VTK QUIET COMPONENTS vtkCommon NO_MODULE) + find_package(VTK QUIET NAMES vtk VTK) + if(VTK_FOUND) + if(NOT (VTK_VERSION VERSION_LESS "9.0.0") AND (VTK_VERSION VERSION_LESS "10.0.0")) # VTK 9.x + find_package(VTK 9 QUIET NAMES vtk COMPONENTS + FiltersExtraction + FiltersSources + FiltersTexture + IOExport + IOGeometry + IOPLY + InteractionStyle + RenderingCore + RenderingLOD + RenderingOpenGL2 + NO_MODULE) + elseif(VTK_VERSION VERSION_GREATER "5") # VTK 6.x components + find_package(VTK QUIET COMPONENTS vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) + IF(VTK_FOUND) + IF(VTK_RENDERING_BACKEND) #in vtk 7, the rendering backend is exported as a var. + find_package(VTK QUIET COMPONENTS vtkRendering${VTK_RENDERING_BACKEND} vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport vtkIOGeometry NO_MODULE) + ELSE(VTK_RENDERING_BACKEND) + find_package(VTK QUIET COMPONENTS vtkRenderingOpenGL vtkInteractionStyle vtkRenderingLOD vtkIOPLY vtkFiltersTexture vtkRenderingFreeType vtkIOExport NO_MODULE) + ENDIF(VTK_RENDERING_BACKEND) + ENDIF(VTK_FOUND) + elseif(VTK_VERSION VERSION_EQUAL "5") # VTK 5.x components + find_package(VTK QUIET COMPONENTS vtkCommon NO_MODULE) + else() + set(VTK_FOUND FALSE) + endif() + endif() endif() if(NOT VTK_FOUND) @@ -44,32 +44,15 @@ if(VTK_VERSION VERSION_LESS "5.8.0") endif() # Different Qt versions can't be linked together -if(HAVE_QT5 AND VTK_VERSION VERSION_LESS "6.0.0") - if(VTK_USE_QT) - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt5 and VTK ver.${VTK_VERSION} + Qt4") - endif() -endif() - -# Different Qt versions can't be linked together. VTK 6.0.0 doesn't provide a way to get Qt version it was linked with -if(HAVE_QT5 AND VTK_VERSION VERSION_EQUAL "6.0.0" AND NOT DEFINED FORCE_VTK) - message(STATUS "VTK support is disabled. Possible incompatible combination: OpenCV+Qt5, and VTK ver.${VTK_VERSION} with Qt4") - message(STATUS "If it is known that VTK was compiled without Qt4, please define '-DFORCE_VTK=TRUE' flag in CMake") +if((HAVE_QT AND VTK_USE_QT) + AND NOT DEFINED FORCE_VTK # deprecated + AND NOT DEFINED OPENCV_FORCE_VTK +) + message(STATUS "VTK support is disabled. Possible incompatible combination: OpenCV+Qt, and VTK ver.${VTK_VERSION} with Qt") + message(STATUS "If it is known that VTK was compiled without Qt4, please define '-DOPENCV_FORCE_VTK=TRUE' flag in CMake") return() endif() -# Different Qt versions can't be linked together -if(HAVE_QT AND VTK_VERSION VERSION_GREATER "6.0.0" AND NOT ${VTK_QT_VERSION} STREQUAL "") - if(HAVE_QT5 AND ${VTK_QT_VERSION} EQUAL "4") - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt5 and VTK ver.${VTK_VERSION} + Qt4") - return() - endif() - - if(NOT HAVE_QT5 AND ${VTK_QT_VERSION} EQUAL "5") - message(STATUS "VTK support is disabled. Incompatible combination: OpenCV + Qt4 and VTK ver.${VTK_VERSION} + Qt5") - return() - endif() -endif() - try_compile(VTK_COMPILE_STATUS "${OpenCV_BINARY_DIR}" "${OpenCV_SOURCE_DIR}/cmake/checks/vtk_test.cpp" diff --git a/cmake/OpenCVFindFrameworks.cmake b/cmake/OpenCVFindFrameworks.cmake index 9d673f2697..19f6d66340 100644 --- a/cmake/OpenCVFindFrameworks.cmake +++ b/cmake/OpenCVFindFrameworks.cmake @@ -17,7 +17,7 @@ else() endif() # --- Concurrency --- -if(MSVC AND NOT HAVE_TBB) +if(MSVC AND NOT HAVE_TBB AND NOT OPENCV_DISABLE_THREAD_SUPPORT) set(_fname "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/concurrencytest.cpp") file(WRITE "${_fname}" "#if _MSC_VER < 1600\n#error\n#endif\nint main() { return 0; }\n") try_compile(HAVE_CONCURRENCY "${CMAKE_BINARY_DIR}" "${_fname}") diff --git a/cmake/OpenCVFindLibsGUI.cmake b/cmake/OpenCVFindLibsGUI.cmake index 8030e8b0c0..b33929e539 100644 --- a/cmake/OpenCVFindLibsGUI.cmake +++ b/cmake/OpenCVFindLibsGUI.cmake @@ -2,34 +2,52 @@ # Detect 3rd-party GUI libraries # ---------------------------------------------------------------------------- -#--- Win32 UI --- -ocv_clear_vars(HAVE_WIN32UI) -if(WITH_WIN32UI) - try_compile(HAVE_WIN32UI - "${OpenCV_BINARY_DIR}" - "${OpenCV_SOURCE_DIR}/cmake/checks/win32uitest.cpp" - CMAKE_FLAGS "-DLINK_LIBRARIES:STRING=user32;gdi32") -endif() - # --- QT4/5 --- ocv_clear_vars(HAVE_QT HAVE_QT5) -if(WITH_QT) - if(NOT WITH_QT EQUAL 4) - find_package(Qt5 COMPONENTS Core Gui Widgets Test Concurrent REQUIRED NO_MODULE) - if(Qt5_FOUND) - set(HAVE_QT5 ON) - set(HAVE_QT ON) - find_package(Qt5 COMPONENTS OpenGL QUIET) - if(Qt5OpenGL_FOUND) - set(QT_QTOPENGL_FOUND ON) - endif() - endif() - endif() - if(NOT HAVE_QT) - find_package(Qt4 REQUIRED QtCore QtGui QtTest) - if(QT4_FOUND) - set(HAVE_QT TRUE) +macro(ocv_find_package_Qt4) + find_package(Qt4 COMPONENTS QtCore QtGui QtTest ${ARGN}) + if(QT4_FOUND) + set(QT_FOUND 1) + ocv_assert(QT_VERSION_MAJOR EQUAL 4) + endif() +endmacro() + +macro(ocv_find_package_Qt OCV_QT_VER) + find_package(Qt${OCV_QT_VER} COMPONENTS Core Gui Widgets Test Concurrent ${ARGN} NO_MODULE) + if(Qt${OCV_QT_VER}_FOUND) + set(QT_FOUND 1) + set(QT_VERSION "${Qt${OCV_QT_VER}_VERSION}") + set(QT_VERSION_MAJOR "${Qt${OCV_QT_VER}_VERSION_MAJOR}") + set(QT_VERSION_MINOR "${Qt${OCV_QT_VER}_VERSION_MINOR}") + set(QT_VERSION_PATCH "${Qt${OCV_QT_VER}_VERSION_PATCH}") + set(QT_VERSION_TWEAK "${Qt${OCV_QT_VER}_VERSION_TWEAK}") + set(QT_VERSION_COUNT "${Qt${OCV_QT_VER}_VERSION_COUNT}") + endif() +endmacro() + +if(WITH_QT) + if(NOT WITH_QT GREATER 0) + # BUG: Qt5Config.cmake script can't handle components properly: find_package(QT NAMES Qt6 Qt5 REQUIRED NO_MODULE COMPONENTS Core Gui Widgets Test Concurrent) + ocv_find_package_Qt(6 QUIET) + if(NOT QT_FOUND) + ocv_find_package_Qt(5 QUIET) + endif() + if(NOT QT_FOUND) + ocv_find_package_Qt4(QUIET) + endif() + elseif(WITH_QT EQUAL 4) + ocv_find_package_Qt4(REQUIRED) + else() # WITH_QT= + ocv_find_package_Qt("${WITH_QT}" REQUIRED) + endif() + if(QT_FOUND) + set(HAVE_QT ON) + if(QT_VERSION_MAJOR GREATER 4) + find_package(Qt${QT_VERSION_MAJOR} COMPONENTS OpenGL QUIET) + if(Qt${QT_VERSION_MAJOR}OpenGL_FOUND) + set(QT_QTOPENGL_FOUND ON) # HAVE_QT_OPENGL is defined below + endif() endif() endif() endif() diff --git a/cmake/OpenCVModule.cmake b/cmake/OpenCVModule.cmake index 7c48aad9c2..43384a65fc 100644 --- a/cmake/OpenCVModule.cmake +++ b/cmake/OpenCVModule.cmake @@ -254,7 +254,7 @@ function(_glob_locations out_paths out_names) list(LENGTH paths before) get_filename_component(path "${path}" ABSOLUTE) # Either module itself - if(NOT path STREQUAL CMAKE_CURRENT_SOURCE_DIR AND EXISTS "${path}/CMakeLists.txt") + if(NOT path STREQUAL "${OpenCV_SOURCE_DIR}/modules" AND EXISTS "${path}/CMakeLists.txt") get_filename_component(name "${path}" NAME) list(APPEND paths "${path}") list(APPEND names "${name}") @@ -296,7 +296,7 @@ macro(_add_modules_1 paths names) list(GET ${names} ${i} __name) #message(STATUS "First pass: ${__name} => ${__path}") include("${__path}/cmake/init.cmake" OPTIONAL) - add_subdirectory("${__path}" "${CMAKE_CURRENT_BINARY_DIR}/.firstpass/${__name}") + add_subdirectory("${__path}" "${OpenCV_BINARY_DIR}/modules/.firstpass/${__name}") endforeach() endif() endmacro() @@ -316,7 +316,7 @@ macro(_add_modules_2) endif() string(REGEX REPLACE "^opencv_" "" name "${m}") #message(STATUS "Second pass: ${name} => ${OPENCV_MODULE_${m}_LOCATION}") - add_subdirectory("${OPENCV_MODULE_${m}_LOCATION}" "${CMAKE_CURRENT_BINARY_DIR}/${name}") + add_subdirectory("${OPENCV_MODULE_${m}_LOCATION}" "${OpenCV_BINARY_DIR}/modules/${name}") endif() ocv_cmake_hook(POST_MODULES_CREATE_${the_module}) endforeach() @@ -369,7 +369,6 @@ macro(ocv_glob_modules main_root) __ocv_resolve_dependencies() # create modules - set(OPENCV_INITIAL_PASS OFF PARENT_SCOPE) set(OPENCV_INITIAL_PASS OFF) ocv_cmake_hook(PRE_MODULES_CREATE) _add_modules_2(${OPENCV_MODULES_BUILD}) @@ -377,6 +376,37 @@ macro(ocv_glob_modules main_root) endmacro() +# called by root CMakeLists.txt +macro(ocv_register_modules) + if(NOT OPENCV_MODULES_PATH) + set(OPENCV_MODULES_PATH "${OpenCV_SOURCE_DIR}/modules") + endif() + + ocv_glob_modules(${OPENCV_MODULES_PATH} ${OPENCV_EXTRA_MODULES_PATH}) + + # build lists of modules to be documented + set(OPENCV_MODULES_MAIN "") + set(OPENCV_MODULES_EXTRA "") + + foreach(mod ${OPENCV_MODULES_BUILD} ${OPENCV_MODULES_DISABLED_USER} ${OPENCV_MODULES_DISABLED_AUTO} ${OPENCV_MODULES_DISABLED_FORCE}) + string(REGEX REPLACE "^opencv_" "" mod "${mod}") + if("${OPENCV_MODULE_opencv_${mod}_LOCATION}" STREQUAL "${OpenCV_SOURCE_DIR}/modules/${mod}") + list(APPEND OPENCV_MODULES_MAIN ${mod}) + else() + list(APPEND OPENCV_MODULES_EXTRA ${mod}) + endif() + endforeach() + ocv_list_sort(OPENCV_MODULES_MAIN) + ocv_list_sort(OPENCV_MODULES_EXTRA) + set(FIXED_ORDER_MODULES core imgproc imgcodecs videoio highgui video 3d stereo features2d calib objdetect dnn ml flann photo stitching) + list(REMOVE_ITEM OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES}) + set(OPENCV_MODULES_MAIN ${FIXED_ORDER_MODULES} ${OPENCV_MODULES_MAIN}) + + set(OPENCV_MODULES_MAIN ${OPENCV_MODULES_MAIN} CACHE INTERNAL "List of main modules" FORCE) + set(OPENCV_MODULES_EXTRA ${OPENCV_MODULES_EXTRA} CACHE INTERNAL "List of extra modules" FORCE) +endmacro() + + # disables OpenCV module with missing dependencies function(__ocv_module_turn_off the_module) list(REMOVE_ITEM OPENCV_MODULES_DISABLED_AUTO "${the_module}") @@ -877,6 +907,7 @@ macro(ocv_create_module) endmacro() macro(_ocv_create_module) + add_definitions(-D__OPENCV_BUILD=1) ocv_compiler_optimization_process_sources(OPENCV_MODULE_${the_module}_SOURCES OPENCV_MODULE_${the_module}_DEPS_EXT ${the_module}) set(__module_headers ${OPENCV_MODULE_${the_module}_HEADERS}) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 5d8ab70e0e..21d60cc0f2 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1481,8 +1481,8 @@ function(ocv_target_link_libraries target) if(NOT LINK_PENDING STREQUAL "") __ocv_push_target_link_libraries(${LINK_MODE} ${LINK_PENDING}) set(LINK_PENDING "") - set(LINK_MODE "${dep}") endif() + set(LINK_MODE "${dep}") else() if(BUILD_opencv_world) if(OPENCV_MODULE_${dep}_IS_PART_OF_WORLD) @@ -1619,6 +1619,18 @@ function(ocv_add_external_target name inc link def) endif() endfunction() +# Returns the first non-interface target +function(ocv_get_imported_target imported interface) + set(__result "${interface}") + get_target_property(__type "${__result}" TYPE) + if(__type STREQUAL "INTERFACE_LIBRARY") + get_target_property(__libs "${__result}" INTERFACE_LINK_LIBRARIES) + list(GET __libs 0 __interface) + ocv_get_imported_target(__result "${__interface}") + endif() + set(${imported} "${__result}" PARENT_SCOPE) +endfunction() + macro(ocv_get_libname var_name) get_filename_component(__libname "${ARGN}" NAME) @@ -1999,3 +2011,10 @@ function(ocv_blob2hdr blob_filename hdr_filename cpp_variable) file(WRITE "${hdr_filename}" "${source}") endfunction() + + + +# +# Include configuration override settings +# +include("${CMAKE_CURRENT_LIST_DIR}/vars/EnableModeVars.cmake") diff --git a/cmake/checks/cpu_rvv.cpp b/cmake/checks/cpu_rvv.cpp index a3eab2abc4..684b2ecbeb 100644 --- a/cmake/checks/cpu_rvv.cpp +++ b/cmake/checks/cpu_rvv.cpp @@ -9,7 +9,7 @@ int test() { const float src[] = { 0.0f, 0.0f, 0.0f, 0.0f }; - vfloat32m1_t val = vle32_v_f32m1((const float*)(src)); + vfloat32m1_t val = vle32_v_f32m1((const float*)(src), 4); return (int)vfmv_f_s_f32m1_f32(val); } #else diff --git a/cmake/templates/OpenCVConfig.root-WIN32.cmake.in b/cmake/templates/OpenCVConfig.root-WIN32.cmake.in index 5da438a9ee..b0f254ebe8 100644 --- a/cmake/templates/OpenCVConfig.root-WIN32.cmake.in +++ b/cmake/templates/OpenCVConfig.root-WIN32.cmake.in @@ -137,6 +137,20 @@ elseif(MSVC) set(OpenCV_RUNTIME vc14) # selecting previous compatible runtime version endif() endif() + elseif(MSVC_VERSION MATCHES "^193[0-9]$") + set(OpenCV_RUNTIME vc17) + check_one_config(has_VS2022) + if(NOT has_VS2022) + set(OpenCV_RUNTIME vc16) + check_one_config(has_VS2019) + if(NOT has_VS2019) + set(OpenCV_RUNTIME vc15) # selecting previous compatible runtime version + check_one_config(has_VS2017) + if(NOT has_VS2017) + set(OpenCV_RUNTIME vc14) # selecting previous compatible runtime version + endif() + endif() + endif() endif() elseif(MINGW) set(OpenCV_RUNTIME mingw) diff --git a/cmake/templates/cvconfig.h.in b/cmake/templates/cvconfig.h.in index 4b18a49709..cb936612bd 100644 --- a/cmake/templates/cvconfig.h.in +++ b/cmake/templates/cvconfig.h.in @@ -121,9 +121,6 @@ /* TIFF codec */ #cmakedefine HAVE_TIFF -/* Win32 UI */ -#cmakedefine HAVE_WIN32UI - /* Define if your processor stores words with the most significant byte first (like Motorola and SPARC, unlike Intel and VAX). */ #cmakedefine WORDS_BIGENDIAN diff --git a/cmake/vars/EnableModeVars.cmake b/cmake/vars/EnableModeVars.cmake new file mode 100644 index 0000000000..3f017af496 --- /dev/null +++ b/cmake/vars/EnableModeVars.cmake @@ -0,0 +1,21 @@ +set(__OCV_MODE_VARS_DIR "${CMAKE_CURRENT_LIST_DIR}") + +macro(ocv_change_mode_var) + set(__var "${ARGV0}") + set(__mode "${ARGV1}") + set(__value "${ARGV2}") + if(__mode STREQUAL "MODIFIED_ACCESS" AND __value) + if(NOT __applied_mode_${__var}) + include("${__OCV_MODE_VARS_DIR}/${__var}.cmake") + set(__applied_mode_${__var} 1) + else() + #message("Mode is already applied: ${__var}") + endif() + endif() +endmacro() + +variable_watch(OPENCV_DISABLE_THREAD_SUPPORT ocv_change_mode_var) +set(OPENCV_DISABLE_THREAD_SUPPORT "${OPENCV_DISABLE_THREAD_SUPPORT}") + +variable_watch(OPENCV_SEMIHOSTING ocv_change_mode_var) +set(OPENCV_SEMIHOSTING "${OPENCV_SEMIHOSTING}") diff --git a/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake b/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake new file mode 100644 index 0000000000..5f5fc0204d --- /dev/null +++ b/cmake/vars/OPENCV_DISABLE_THREAD_SUPPORT.cmake @@ -0,0 +1,28 @@ +# Force removal of code conditionally compiled with `#if +# HAVE_PTHREAD`. +ocv_update(HAVE_PTHREAD 0) + +# There components are disabled because they require +# multi-threaded execution. +ocv_update(WITH_PROTOBUF OFF) +ocv_update(WITH_GSTREAMER OFF) +ocv_update(WITH_IPP OFF) +ocv_update(WITH_ITT OFF) +ocv_update(WITH_OPENCL OFF) +ocv_update(WITH_VA OFF) +ocv_update(WITH_VA_INTEL OFF) + +# Disable bindings +ocv_update(BUILD_opencv_python2 OFF) +ocv_update(BUILD_opencv_python3 OFF) +ocv_update(BUILD_JAVA OFF) +ocv_update(BUILD_opencv_java OFF) + +# These modules require `#include +# <[thread|mutex|condition_variable|future]>` and linkage into +# `libpthread` to work. +ocv_update(BUILD_opencv_objdetect OFF) +ocv_update(BUILD_opencv_gapi OFF) +ocv_update(BUILD_opencv_dnn OFF) + +set(OPJ_USE_THREAD "OFF" CACHE INTERNAL "") diff --git a/cmake/vars/OPENCV_SEMIHOSTING.cmake b/cmake/vars/OPENCV_SEMIHOSTING.cmake new file mode 100644 index 0000000000..66f21c7ebd --- /dev/null +++ b/cmake/vars/OPENCV_SEMIHOSTING.cmake @@ -0,0 +1,10 @@ +set(CV_TRACE OFF) + +# These third parties libraries are incompatible with the semihosting +# toolchain. +set(WITH_JPEG OFF) +set(WITH_OPENEXR OFF) +set(WITH_TIFF OFF) + +# Turn off `libpng` for some linking issues. +set(WITH_PNG OFF) diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 26ad42b1e5..ce207d3e31 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -106,7 +106,7 @@ RECURSIVE = YES EXCLUDE = @CMAKE_DOXYGEN_EXCLUDE_LIST@ EXCLUDE_SYMLINKS = NO EXCLUDE_PATTERNS = *.inl.hpp *.impl.hpp *_detail.hpp */cudev/**/detail/*.hpp *.m */opencl/runtime/* */legacy/* *_c.h @DOXYGEN_EXCLUDE_PATTERNS@ -EXCLUDE_SYMBOLS = cv::DataType<*> cv::traits::* int void CV__* T __CV* +EXCLUDE_SYMBOLS = cv::DataType<*> cv::traits::* int void CV__* T __CV* cv::gapi::detail* EXAMPLE_PATH = @CMAKE_DOXYGEN_EXAMPLE_PATH@ EXAMPLE_PATTERNS = * EXAMPLE_RECURSIVE = YES diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown index 22544b280c..8421269587 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contour_features/js_contour_features.markdown @@ -1,6 +1,9 @@ Contour Features {#tutorial_js_contour_features} ================ +@prev_tutorial{tutorial_js_contours_begin} +@next_tutorial{tutorial_js_contour_properties} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown index 1dbb15c4cf..3392283305 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contour_properties/js_contour_properties.markdown @@ -1,6 +1,9 @@ Contour Properties {#tutorial_js_contour_properties} ================== +@prev_tutorial{tutorial_js_contour_features} +@next_tutorial{tutorial_js_contours_more_functions} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown index 9678a7c9f0..3caf17f873 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_begin/js_contours_begin.markdown @@ -1,6 +1,8 @@ Contours : Getting Started {#tutorial_js_contours_begin} ========================== +@next_tutorial{tutorial_js_contour_features} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown index c98628e486..c2e408a962 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_hierarchy/js_contours_hierarchy.markdown @@ -1,6 +1,8 @@ Contours Hierarchy {#tutorial_js_contours_hierarchy} ================== +@prev_tutorial{tutorial_js_contours_more_functions} + Goal ---- diff --git a/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown b/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown index b753116662..941f0c486a 100644 --- a/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown +++ b/doc/js_tutorials/js_imgproc/js_contours/js_contours_more_functions/js_contours_more_functions.markdown @@ -1,6 +1,9 @@ Contours : More Functions {#tutorial_js_contours_more_functions} ========================= +@prev_tutorial{tutorial_js_contour_properties} +@next_tutorial{tutorial_js_contours_hierarchy} + Goal ---- diff --git a/doc/pattern_tools/gen_pattern.py b/doc/pattern_tools/gen_pattern.py index a6ffc7ca7e..83ed115d36 100755 --- a/doc/pattern_tools/gen_pattern.py +++ b/doc/pattern_tools/gen_pattern.py @@ -6,13 +6,14 @@ python gen_pattern.py -o out.svg -r 11 -c 8 -T circles -s 20.0 -R 5.0 -u mm -w 2 -o, --output - output file (default out.svg) -r, --rows - pattern rows (default 11) -c, --columns - pattern columns (default 8) --T, --type - type of pattern, circles, acircles, checkerboard (default circles) +-T, --type - type of pattern, circles, acircles, checkerboard, radon_checkerboard (default circles) -s, --square_size - size of squares in pattern (default 20.0) -R, --radius_rate - circles_radius = square_size/radius_rate (default 5.0) -u, --units - mm, inches, px, m (default mm) -w, --page_width - page width in units (default 216) -h, --page_height - page height in units (default 279) -a, --page_size - page size (default A4), supersedes -h -w arguments +-m, --markers - list of cells with markers for the radon checkerboard -H, --help - show help """ @@ -22,7 +23,7 @@ from svgfig import * class PatternMaker: - def __init__(self, cols, rows, output, units, square_size, radius_rate, page_width, page_height): + def __init__(self, cols, rows, output, units, square_size, radius_rate, page_width, page_height, markers): self.cols = cols self.rows = rows self.output = output @@ -31,6 +32,7 @@ class PatternMaker: self.radius_rate = radius_rate self.width = page_width self.height = page_height + self.markers = markers self.g = SVG("g") # the svg group container def make_circles_pattern(self): @@ -70,6 +72,74 @@ class PatternMaker: height=spacing, fill="black", stroke="none") self.g.append(square) + @staticmethod + def _make_round_rect(x, y, diam, corners=("right", "right", "right", "right")): + rad = diam / 2 + cw_point = ((0, 0), (diam, 0), (diam, diam), (0, diam)) + mid_cw_point = ((0, rad), (rad, 0), (diam, rad), (rad, diam)) + res_str = "M{},{} ".format(x + mid_cw_point[0][0], y + mid_cw_point[0][1]) + n = len(cw_point) + for i in range(n): + if corners[i] == "right": + res_str += "L{},{} L{},{} ".format(x + cw_point[i][0], y + cw_point[i][1], + x + mid_cw_point[(i + 1) % n][0], y + mid_cw_point[(i + 1) % n][1]) + elif corners[i] == "round": + res_str += "A{},{} 0,0,1 {},{} ".format(rad, rad, x + mid_cw_point[(i + 1) % n][0], + y + mid_cw_point[(i + 1) % n][1]) + else: + raise TypeError("unknown corner type") + return res_str + + def _get_type(self, x, y): + corners = ["right", "right", "right", "right"] + is_inside = True + if x == 0: + corners[0] = "round" + corners[3] = "round" + is_inside = False + if y == 0: + corners[0] = "round" + corners[1] = "round" + is_inside = False + if x == self.cols - 1: + corners[1] = "round" + corners[2] = "round" + is_inside = False + if y == self.rows - 1: + corners[2] = "round" + corners[3] = "round" + is_inside = False + return corners, is_inside + + def make_radon_checkerboard_pattern(self): + spacing = self.square_size + xspacing = (self.width - self.cols * self.square_size) / 2.0 + yspacing = (self.height - self.rows * self.square_size) / 2.0 + for x in range(0, self.cols): + for y in range(0, self.rows): + if x % 2 == y % 2: + corner_types, is_inside = self._get_type(x, y) + if is_inside: + square = SVG("rect", x=x * spacing + xspacing, y=y * spacing + yspacing, width=spacing, + height=spacing, fill="black", stroke="none") + else: + square = SVG("path", d=self._make_round_rect(x * spacing + xspacing, y * spacing + yspacing, + spacing, corner_types), fill="black", stroke="none") + self.g.append(square) + if self.markers is not None: + r = self.square_size * 0.17 + pattern_width = ((self.cols - 1.0) * spacing) + (2.0 * r) + pattern_height = ((self.rows - 1.0) * spacing) + (2.0 * r) + x_spacing = (self.width - pattern_width) / 2.0 + y_spacing = (self.height - pattern_height) / 2.0 + for x, y in self.markers: + color = "black" + if x % 2 == y % 2: + color = "white" + dot = SVG("circle", cx=(x * spacing) + x_spacing + r, + cy=(y * spacing) + y_spacing + r, r=r, fill=color, stroke="none") + self.g.append(dot) + def save(self): c = canvas(self.g, width="%d%s" % (self.width, self.units), height="%d%s" % (self.height, self.units), viewBox="0 0 %d %d" % (self.width, self.height)) @@ -85,7 +155,7 @@ def main(): type=int) parser.add_argument("-r", "--rows", help="pattern rows", default="11", action="store", dest="rows", type=int) parser.add_argument("-T", "--type", help="type of pattern", default="circles", action="store", dest="p_type", - choices=["circles", "acircles", "checkerboard"]) + choices=["circles", "acircles", "checkerboard", "radon_checkerboard"]) parser.add_argument("-u", "--units", help="length unit", default="mm", action="store", dest="units", choices=["mm", "inches", "px", "m"]) parser.add_argument("-s", "--square_size", help="size of squares in pattern", default="20.0", action="store", @@ -96,8 +166,12 @@ def main(): dest="page_width", type=float) parser.add_argument("-h", "--page_height", help="page height in units", default=argparse.SUPPRESS, action="store", dest="page_height", type=float) - parser.add_argument("-a", "--page_size", help="page size, superseded if -h and -w are set", default="A4", action="store", - dest="page_size", choices=["A0", "A1", "A2", "A3", "A4", "A5"]) + parser.add_argument("-a", "--page_size", help="page size, superseded if -h and -w are set", default="A4", + action="store", dest="page_size", choices=["A0", "A1", "A2", "A3", "A4", "A5"]) + parser.add_argument("-m", "--markers", help="list of cells with markers for the radon checkerboard. Marker " + "coordinates as list of numbers: -m 1 2 3 4 means markers in cells " + "[1, 2] and [3, 4]", + action="store", dest="markers", nargs="+", type=int) args = parser.parse_args() show_help = args.show_help @@ -121,10 +195,19 @@ def main(): "A5": [148, 210]} page_width = page_sizes[page_size][0] page_height = page_sizes[page_size][1] - pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height) + if len(args.markers) % 2 == 1: + raise ValueError("The length of the markers array={} must be even".format(len(args.markers))) + markers = set() + for x, y in zip(args.markers[::2], args.markers[1::2]): + if x in range(0, columns) and y in range(0, rows): + markers.add((x, y)) + else: + raise ValueError("The marker {},{} is outside the checkerboard".format(x, y)) + + pm = PatternMaker(columns, rows, output, units, square_size, radius_rate, page_width, page_height, markers) # dict for easy lookup of pattern type mp = {"circles": pm.make_circles_pattern, "acircles": pm.make_acircles_pattern, - "checkerboard": pm.make_checkerboard_pattern} + "checkerboard": pm.make_checkerboard_pattern, "radon_checkerboard": pm.make_radon_checkerboard_pattern} mp[p_type]() # this should save pattern to output pm.save() diff --git a/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown b/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown index 2c5eccd0d6..001952deca 100644 --- a/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown +++ b/doc/py_tutorials/py_bindings/py_bindings_basics/py_bindings_basics.markdown @@ -60,6 +60,14 @@ of C++. So this is the basic version of how OpenCV-Python bindings are generated. +@note There is no 1:1 mapping of numpy.ndarray on cv::Mat. For example, cv::Mat has channels field, +which is emulated as last dimension of numpy.ndarray and implicitly converted. +However, such implicit conversion has problem with passing of 3D numpy arrays into C++ code +(the last dimension is implicitly reinterpreted as number of channels). +Refer to the [issue](https://github.com/opencv/opencv/issues/19091) for workarounds if you need to process 3D arrays or ND-arrays with channels. +OpenCV 4.5.4+ has `cv.Mat` wrapper derived from `numpy.ndarray` to explicitly handle the channels behavior. + + How to extend new modules to Python? ------------------------------------ diff --git a/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown b/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown index 4ce61370bd..b1b8a81ca8 100644 --- a/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown +++ b/doc/py_tutorials/py_feature2d/py_fast/py_fast.markdown @@ -98,7 +98,7 @@ import numpy as np import cv2 as cv from matplotlib import pyplot as plt -img = cv.imread('simple.jpg',0) +img = cv.imread('blox.jpg',0) # `/samples/data/blox.jpg` # Initiate FAST object with default values fast = cv.FastFeatureDetector_create() @@ -113,17 +113,17 @@ print( "nonmaxSuppression:{}".format(fast.getNonmaxSuppression()) ) print( "neighborhood: {}".format(fast.getType()) ) print( "Total Keypoints with nonmaxSuppression: {}".format(len(kp)) ) -cv.imwrite('fast_true.png',img2) +cv.imwrite('fast_true.png', img2) # Disable nonmaxSuppression fast.setNonmaxSuppression(0) -kp = fast.detect(img,None) +kp = fast.detect(img, None) print( "Total Keypoints without nonmaxSuppression: {}".format(len(kp)) ) img3 = cv.drawKeypoints(img, kp, None, color=(255,0,0)) -cv.imwrite('fast_false.png',img3) +cv.imwrite('fast_false.png', img3) @endcode See the results. First image shows FAST with nonmaxSuppression and second one without nonmaxSuppression: diff --git a/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown b/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown index cbc2a72eec..d36e5784eb 100644 --- a/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown +++ b/doc/py_tutorials/py_imgproc/py_canny/py_canny.markdown @@ -74,7 +74,7 @@ Canny Edge Detection in OpenCV OpenCV puts all the above in single function, **cv.Canny()**. We will see how to use it. First argument is our input image. Second and third arguments are our minVal and maxVal respectively. -Third argument is aperture_size. It is the size of Sobel kernel used for find image gradients. By +Fourth argument is aperture_size. It is the size of Sobel kernel used for find image gradients. By default it is 3. Last argument is L2gradient which specifies the equation for finding gradient magnitude. If it is True, it uses the equation mentioned above which is more accurate, otherwise it uses this function: \f$Edge\_Gradient \; (G) = |G_x| + |G_y|\f$. By default, it is False. diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown index 5af1e5a1e0..f3c7f6fc31 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_features/py_contour_features.markdown @@ -1,6 +1,9 @@ Contour Features {#tutorial_py_contour_features} ================ +@prev_tutorial{tutorial_py_contours_begin} +@next_tutorial{tutorial_py_contour_properties} + Goal ---- @@ -91,7 +94,7 @@ convexity defects, which are the local maximum deviations of hull from contours. There is a little bit things to discuss about it its syntax: @code{.py} -hull = cv.convexHull(points[, hull[, clockwise[, returnPoints]] +hull = cv.convexHull(points[, hull[, clockwise[, returnPoints]]]) @endcode Arguments details: diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown index 461c870343..282f62ddf9 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contour_properties/py_contour_properties.markdown @@ -1,6 +1,9 @@ Contour Properties {#tutorial_py_contour_properties} ================== +@prev_tutorial{tutorial_py_contour_features} +@next_tutorial{tutorial_py_contours_more_functions} + Here we will learn to extract some frequently used properties of objects like Solidity, Equivalent Diameter, Mask image, Mean Intensity etc. More features can be found at [Matlab regionprops documentation](http://www.mathworks.in/help/images/ref/regionprops.html). diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown index a2b89c1c96..0049d3131d 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_begin/py_contours_begin.markdown @@ -1,6 +1,8 @@ Contours : Getting Started {#tutorial_py_contours_begin} ========================== +@next_tutorial{tutorial_py_contour_features} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown index 2619ea2a70..075e6ec81f 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_hierarchy/py_contours_hierarchy.markdown @@ -1,6 +1,8 @@ Contours Hierarchy {#tutorial_py_contours_hierarchy} ================== +@prev_tutorial{tutorial_py_contours_more_functions} + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown index e50f9dd6c9..65f5b75401 100644 --- a/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown +++ b/doc/py_tutorials/py_imgproc/py_contours/py_contours_more_functions/py_contours_more_functions.markdown @@ -1,6 +1,10 @@ Contours : More Functions {#tutorial_py_contours_more_functions} ========================= +@prev_tutorial{tutorial_py_contour_properties} +@next_tutorial{tutorial_py_contours_hierarchy} + + Goal ---- diff --git a/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown b/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown index 63fde0a130..602ab29ad6 100644 --- a/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown +++ b/doc/py_tutorials/py_imgproc/py_pyramids/py_pyramids.markdown @@ -117,7 +117,7 @@ for i in range(5,0,-1): LS = [] for la,lb in zip(lpA,lpB): rows,cols,dpt = la.shape - ls = np.hstack((la[:,0:cols/2], lb[:,cols/2:])) + ls = np.hstack((la[:,0:cols//2], lb[:,cols//2:])) LS.append(ls) # now reconstruct @@ -127,7 +127,7 @@ for i in range(1,6): ls_ = cv.add(ls_, LS[i]) # image with direct connecting each half -real = np.hstack((A[:,:cols/2],B[:,cols/2:])) +real = np.hstack((A[:,:cols//2],B[:,cols//2:])) cv.imwrite('Pyramid_blending2.jpg',ls_) cv.imwrite('Direct_blending.jpg',real) diff --git a/doc/tutorials/app/video_input_psnr_ssim.markdown b/doc/tutorials/app/video_input_psnr_ssim.markdown index e212c4e46d..1d53c70def 100644 --- a/doc/tutorials/app/video_input_psnr_ssim.markdown +++ b/doc/tutorials/app/video_input_psnr_ssim.markdown @@ -188,7 +188,7 @@ implementation below. This will return a similarity index for each channel of the image. This value is between zero and one, where one corresponds to perfect fit. Unfortunately, the many Gaussian blurring is quite -costly, so while the PSNR may work in a real time like environment (24 frame per second) this will +costly, so while the PSNR may work in a real time like environment (24 frames per second) this will take significantly more than to accomplish similar performance results. Therefore, the source code presented at the start of the tutorial will perform the PSNR measurement diff --git a/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown b/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown index c87f9f95f8..b13f09756b 100644 --- a/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown +++ b/doc/tutorials/calib3d/camera_calibration_pattern/camera_calibration_pattern.markdown @@ -36,6 +36,10 @@ create a circle board pattern in file acircleboard.svg with 7 rows, 5 columns an python gen_pattern.py -o acircleboard.svg --rows 7 --columns 5 --type acircles --square_size 10 --radius_rate 2 +create a radon checkerboard for findChessboardCornersSB() with markers in (7 4), (7 5), (8 5) cells: + + python gen_pattern.py -o radon_checkerboard.svg --rows 10 --columns 15 --type radon_checkerboard -s 12.1 -m 7 4 7 5 8 5 + If you want to change unit use -u option (mm inches, px, m) If you want to change page size use -w and -h options diff --git a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown index da060cf27d..46838dd99b 100644 --- a/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown +++ b/doc/tutorials/core/file_input_output_with_xml_yml/file_input_output_with_xml_yml.markdown @@ -4,7 +4,7 @@ File Input and Output using XML and YAML files {#tutorial_file_input_output_with @tableofcontents @prev_tutorial{tutorial_discrete_fourier_transform} -@next_tutorial{tutorial_how_to_use_OpenCV_parallel_for_} +@next_tutorial{tutorial_how_to_use_OpenCV_parallel_for_new} | | | | -: | :- | diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown new file mode 100644 index 0000000000..57334e2b9b --- /dev/null +++ b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/how_to_use_OpenCV_parallel_for_new.markdown @@ -0,0 +1,166 @@ +How to use the OpenCV parallel_for_ to parallelize your code {#tutorial_how_to_use_OpenCV_parallel_for_new} +================================================================== + +@tableofcontents + +@prev_tutorial{tutorial_file_input_output_with_xml_yml} +@next_tutorial{tutorial_univ_intrin} + +| | | +| -: | :- | +| Compatibility | OpenCV >= 3.0 | + +Goal +---- + +The goal of this tutorial is to demonstrate the use of the OpenCV `parallel_for_` framework to easily parallelize your code. To illustrate the concept, we will write a program to perform convolution operation over an image. +The full tutorial code is [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp). + +Precondition +---- + +### Parallel Frameworks +The first precondition is to have OpenCV built with a parallel framework. +In OpenCV 4.5, the following parallel frameworks are available in that order: + +* Intel Threading Building Blocks (3rdparty library, should be explicitly enabled) +* OpenMP (integrated to compiler, should be explicitly enabled) +* APPLE GCD (system wide, used automatically (APPLE only)) +* Windows RT concurrency (system wide, used automatically (Windows RT only)) +* Windows concurrency (part of runtime, used automatically (Windows only - MSVC++ >= 10)) +* Pthreads + +As you can see, several parallel frameworks can be used in the OpenCV library. Some parallel libraries are third party libraries and have to be explicitly enabled in CMake before building, while others are automatically available with the platform (e.g. APPLE GCD). + +### Race Conditions +Race conditions occur when more than one thread try to write *or* read and write to a particular memory location simultaneously. +Based on that, we can broadly classify algorithms into two categories:- +1. Algorithms in which only a single thread writes data to a particular memory location. + * In *convolution*, for example, even though multiple threads may read from a pixel at a particular time, only a single thread *writes* to a particular pixel. + +2. Algorithms in which multiple threads may write to a single memory location. + * Finding contours, features, etc. Such algorithms may require each thread to add data to a global variable simultaneously. For example, when detecting features, each thread will add features of their respective parts of the image to a common vector, thus creating a race condition. + +Convolution +----------- + +We will use the example of performing a convolution to demonstrate the use of `parallel_for_` to parallelize the computation. This is an example of an algorithm which does not lead to a race condition. + +Theory +------ +Convolution is a simple mathematical operation widely used in image processing. Here, we slide a smaller matrix, called the *kernel*, over an image and a sum of the product of pixel values and corresponding values in the kernel gives us the value of the particular pixel in the output (called the anchor point of the kernel). Based on the values in the kernel, we get different results. +In the example below, we use a 3x3 kernel (anchored at its center) and convolve over a 5x5 matrix to produce a 3x3 matrix. The size of the output can be altered by padding the input with suitable values. +![Convolution Animation](images/convolution-example-matrix.gif) + +For more information about different kernels and what they do, look [here](https://en.wikipedia.org/wiki/Kernel_(image_processing)) + +For the purpose of this tutorial, we will implement the simplest form of the function which takes a grayscale image (1 channel) and an odd length square kernel and produces an output image. +The operation will not be performed in-place. +@note We can store a few of the relevant pixels temporarily to make sure we use the original values during the convolution and then do it in-place. However, the purpose of this tutorial is to introduce parallel_for_ function and an inplace implementation may be too complicated. + +Pseudocode +----------- + + InputImage src, OutputImage dst, kernel(size n) + makeborder(src, n/2) + for each pixel (i, j) strictly inside borders, do: + { + value := 0 + for k := -n/2 to n/2, do: + for l := -n/2 to n/2, do: + value += kernel[n/2 + k][n/2 + l]*src[i + k][j + l] + + dst[i][j] := value + } + +For an *n-sized kernel*, we will add a border of size *n/2* to handle edge cases. +We then run two loops to move along the kernel and add the products to sum + +Implementation +-------------- + +### Sequential implementation + +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-sequential + +We first make an output matrix(dst) with the same size as src and add borders to the src image(to handle edge cases). +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-make-borders + +We then sequentially iterate over the pixels in the src image and compute the value over the kernel and the neighbouring pixel values. +We then fill value to the corresponding pixel in the dst image. +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-kernel-loop + +### Parallel implementation + +When looking at the sequential implementation, we can notice that each pixel depends on multiple neighbouring pixels but only one pixel is edited at a time. Thus, to optimize the computation, we can split the image into stripes and parallely perform convolution on each, by exploiting the multi-core architecture of modern processor. The OpenCV @ref cv::parallel_for_ framework automatically decides how to split the computation efficiently and does most of the work for us. + +@note Although values of a pixel in a particular stripe may depend on pixel values outside the stripe, these are only read only operations and hence will not cause undefined behaviour. + + +We first declare a custom class that inherits from @ref cv::ParallelLoopBody and override the `virtual void operator ()(const cv::Range& range) const`. +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel + +The range in the `operator ()` represents the subset of values that will be treated by an individual thread. Based on the requirement, there may be different ways of splitting the range which in turn changes the computation. + +For example, we can either +1. Split the entire traversal of the image and obtain the [row, col] coordinate in the following way (as shown in the above code): + + @snippet how_to_use_OpenCV_parallel_for_new.cpp overload-full + + We would then call the parallel_for_ function in the following way: + @snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-function +
+ +2. Split the rows and compute for each row: + + @snippet how_to_use_OpenCV_parallel_for_new.cpp overload-row-split + + In this case, we call the parallel_for_ function with a different range: + @snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-function-row + +@note In our case, both implementations perform similarly. Some cases may allow better memory access patterns or other performance benefits. + +To set the number of threads, you can use: @ref cv::setNumThreads. You can also specify the number of splitting using the nstripes parameter in @ref cv::parallel_for_. For instance, if your processor has 4 threads, setting `cv::setNumThreads(2)` or setting `nstripes=2` should be the same as by default it will use all the processor threads available but will split the workload only on two threads. + +@note C++ 11 standard allows to simplify the parallel implementation by get rid of the `parallelConvolution` class and replacing it with lambda expression: + +@snippet how_to_use_OpenCV_parallel_for_new.cpp convolution-parallel-cxx11 + +Results +----------- + +The resulting time taken for execution of the two implementations on a +* *512x512 input* with a *5x5 kernel*: + + This program shows how to use the OpenCV parallel_for_ function and + compares the performance of the sequential and parallel implementations for a + convolution operation + Usage: + ./a.out [image_path -- default lena.jpg] + + Sequential Implementation: 0.0953564s + Parallel Implementation: 0.0246762s + Parallel Implementation(Row Split): 0.0248722s + +
+ +* *512x512 input with a 3x3 kernel* + + This program shows how to use the OpenCV parallel_for_ function and + compares the performance of the sequential and parallel implementations for a + convolution operation + Usage: + ./a.out [image_path -- default lena.jpg] + + Sequential Implementation: 0.0301325s + Parallel Implementation: 0.0117053s + Parallel Implementation(Row Split): 0.0117894s + +The performance of the parallel implementation depends on the type of CPU you have. For instance, on 4 cores - 8 threads CPU, runtime may be 6x to 7x faster than a sequential implementation. There are many factors to explain why we do not achieve a speed-up of 8x: + * the overhead to create and manage the threads, + * background processes running in parallel, + * the difference between 4 hardware cores with 2 logical threads for each core and 8 hardware cores. + +In the tutorial, we used a horizontal gradient filter(as shown in the animation above), which produces an image highlighting the vertical edges. + +![result image](images/resimg.jpg) \ No newline at end of file diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif new file mode 100644 index 0000000000..070208eb3d Binary files /dev/null and b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/convolution-example-matrix.gif differ diff --git a/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg new file mode 100644 index 0000000000..96885c3c51 Binary files /dev/null and b/doc/tutorials/core/how_to_use_OpenCV_parallel_for_new/images/resimg.jpg differ diff --git a/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown b/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown index 4f6f2b8a88..c53296b3bf 100644 --- a/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown +++ b/doc/tutorials/core/mat_the_basic_image_container/mat_the_basic_image_container.markdown @@ -91,8 +91,8 @@ a new header with the new boundaries: Mat D (A, Rect(10, 10, 100, 100) ); // using a rectangle Mat E = A(Range::all(), Range(1,3)); // using row and column boundaries @endcode -Now you may ask -- if the matrix itself may belong to multiple *Mat* objects who takes responsibility -for cleaning it up when it's no longer needed. The short answer is: the last object that used it. +Now you may ask -- if the matrix itself may belong to multiple *Mat* objects, who takes responsibility +for cleaning it up when it's no longer needed? The short answer is: the last object that used it. This is handled by using a reference counting mechanism. Whenever somebody copies a header of a *Mat* object, a counter is increased for the matrix. Whenever a header is cleaned, this counter is decreased. When the counter reaches zero the matrix is freed. Sometimes you will want to copy @@ -102,12 +102,12 @@ Mat F = A.clone(); Mat G; A.copyTo(G); @endcode -Now modifying *F* or *G* will not affect the matrix pointed by the *A*'s header. What you need to +Now modifying *F* or *G* will not affect the matrix pointed to by the *A*'s header. What you need to remember from all this is that: - Output image allocation for OpenCV functions is automatic (unless specified otherwise). - You do not need to think about memory management with OpenCV's C++ interface. -- The assignment operator and the copy constructor only copies the header. +- The assignment operator and the copy constructor only copy the header. - The underlying matrix of an image may be copied using the @ref cv::Mat::clone() and @ref cv::Mat::copyTo() functions. @@ -122,10 +122,10 @@ of these allows us to create many shades of gray. For *colorful* ways we have a lot more methods to choose from. Each of them breaks it down to three or four basic components and we can use the combination of these to create the others. The most popular one is RGB, mainly because this is also how our eye builds up colors. Its base colors are -red, green and blue. To code the transparency of a color sometimes a fourth element: alpha (A) is +red, green and blue. To code the transparency of a color sometimes a fourth element, alpha (A), is added. -There are, however, many other color systems each with their own advantages: +There are, however, many other color systems, each with their own advantages: - RGB is the most common as our eyes use something similar, however keep in mind that OpenCV standard display system composes colors using the BGR color space (red and blue channels are swapped places). @@ -139,11 +139,11 @@ There are, however, many other color systems each with their own advantages: Each of the building components has its own valid domains. This leads to the data type used. How we store a component defines the control we have over its domain. The smallest data type possible is *char*, which means one byte or 8 bits. This may be unsigned (so can store values from 0 to 255) or -signed (values from -127 to +127). Although in case of three components this already gives 16 -million possible colors to represent (like in case of RGB) we may acquire an even finer control by +signed (values from -127 to +127). Although this width, in the case of three components (like RGB), already gives 16 +million possible colors to represent, we may acquire an even finer control by using the float (4 byte = 32 bit) or double (8 byte = 64 bit) data types for each component. Nevertheless, remember that increasing the size of a component also increases the size of the whole -picture in the memory. +picture in memory. Creating a Mat object explicitly ---------------------------------- diff --git a/doc/tutorials/core/table_of_content_core.markdown b/doc/tutorials/core/table_of_content_core.markdown index 4cd77fcdfc..ff142b7f9c 100644 --- a/doc/tutorials/core/table_of_content_core.markdown +++ b/doc/tutorials/core/table_of_content_core.markdown @@ -9,4 +9,5 @@ The Core Functionality (core module) {#tutorial_table_of_content_core} - @subpage tutorial_basic_linear_transform - @subpage tutorial_discrete_fourier_transform - @subpage tutorial_file_input_output_with_xml_yml -- @subpage tutorial_how_to_use_OpenCV_parallel_for_ +- @subpage tutorial_how_to_use_OpenCV_parallel_for_new +- @subpage tutorial_univ_intrin diff --git a/doc/tutorials/core/univ_intrin/univ_intrin.markdown b/doc/tutorials/core/univ_intrin/univ_intrin.markdown new file mode 100644 index 0000000000..7e6a8a7cc8 --- /dev/null +++ b/doc/tutorials/core/univ_intrin/univ_intrin.markdown @@ -0,0 +1,334 @@ +Vectorizing your code using Universal Intrinsics {#tutorial_univ_intrin} +================================================================== + +@tableofcontents + +@prev_tutorial{tutorial_how_to_use_OpenCV_parallel_for_new} + +| | | +| -: | :- | +| Compatibility | OpenCV >= 3.0 | + +Goal +---- + +The goal of this tutorial is to provide a guide to using the @ref core_hal_intrin feature to vectorize your C++ code for a faster runtime. +We'll briefly look into _SIMD intrinsics_ and how to work with wide _registers_, followed by a tutorial on the basic operations using wide registers. + +Theory +------ + +In this section, we will briefly look into a few concepts to better help understand the functionality. + +### Intrinsics +Intrinsics are functions which are separately handled by the compiler. These functions are often optimized to perform in the most efficient ways possible and hence run faster than normal implementations. However, since these functions depend on the compiler, it makes it difficult to write portable applications. + +### SIMD +SIMD stands for **Single Instruction, Multiple Data**. SIMD Intrinsics allow the processor to vectorize calculations. The data is stored in what are known as *registers*. A *register* may be *128-bits*, *256-bits* or *512-bits* wide. Each *register* stores **multiple values** of the **same data type**. The size of the register and the size of each value determines the number of values stored in total. + +Depending on what *Instruction Sets* your CPU supports, you may be able to use the different registers. To learn more, look [here](https://en.wikipedia.org/wiki/Instruction_set_architecture) + +Universal Intrinsics +-------------------- + +OpenCVs universal intrinsics provides an abstraction to SIMD vectorization methods and allows the user to use intrinsics without the need to write system specific code. + +OpenCV Universal Intrinsics support the following instruction sets: +* *128 bit* registers of various types support is implemented for a wide range of architectures including + * x86(SSE/SSE2/SSE4.2), + * ARM(NEON), + * PowerPC(VSX), + * MIPS(MSA). +* *256 bit* registers are supported on x86(AVX2) and +* *512 bit* registers are supported on x86(AVX512) + +**We will now introduce the available structures and functions:** +* Register structures +* Load and store +* Mathematical Operations +* Reduce and Mask + +### Register Structures + +The Universal Intrinsics set implements every register as a structure based on the particular SIMD register. +All types contain the `nlanes` enumeration which gives the exact number of values that the type can hold. This eliminates the need to hardcode the number of values during implementations. + +@note Each register structure is under the `cv` namespace. + +There are **two types** of registers: + +* **Variable sized registers**: These structures do not have a fixed size and their exact bit length is deduced during compilation, based on the available SIMD capabilities. Consequently, the value of the `nlanes` enum is determined in compile time. +
+ + Each structure follows the following convention: + + v_[type of value][size of each value in bits] + + For instance, **v_uint8 holds 8-bit unsigned integers** and **v_float32 holds 32-bit floating point values**. We then declare a register like we would declare any object in C++ + + Based on the available SIMD instruction set, a particular register will hold different number of values. + For example: If your computer supports a maximum of 256bit registers, + * *v_uint8* will hold 32 8-bit unsigned integers + * *v_float64* will hold 4 64-bit floats (doubles) + + v_uint8 a; // a is a register supporting uint8(char) data + int n = a.nlanes; // n holds 32 + + Available data type and sizes: + |Type|Size in bits| + |-:|:-| + |uint| 8, 16, 32, 64| + |int | 8, 16, 32, 64| + |float | 32, 64| + +* **Constant sized registers**: These structures have a fixed bit size and hold a constant number of values. We need to know what SIMD instruction set is supported by the system and select compatible registers. Use these only if exact bit length is necessary. +
+ + Each structure follows the convention: + + v_[type of value][size of each value in bits]x[number of values] + + Suppose we want to store + * 32-bit(*size in bits*) signed integers in a **128 bit register**. Since the register size is already known, we can find out the *number of data points in register* (*128/32 = 4*): + + v_int32x8 reg1 // holds 8 32-bit signed integers. + + * 64-bit floats in 512 bit register: + + v_float64x8 reg2 // reg2.nlanes = 8 + +### Load and Store operations + +Now that we know how registers work, let us look at the functions used for filling these registers with values. + +* **Load**: Load functions allow you to *load* values into a register. + * *Constructors* - When declaring a register structure, we can either provide a memory address from where the register will pick up contiguous values, or provide the values explicitly as multiple arguments (Explicit multiple arguments is available only for Constant Sized Registers): + + float ptr[32] = {1, 2, 3 ..., 32}; // ptr is a pointer to a contiguous memory block of 32 floats + + // Variable Sized Registers // + int x = v_float32().nlanes; // set x as the number of values the register can hold + + v_float32 reg1(ptr); // reg1 stores first x values according to the maximum register size available. + v_float32 reg2(ptr + x); // reg stores the next x values + + // Constant Sized Registers // + v_float32x4 reg1(ptr); // reg1 stores the first 4 floats (1, 2, 3, 4) + v_float32x4 reg2(ptr + 4); // reg2 stores the next 4 floats (5, 6, 7, 8) + + // Or we can explicitly write down the values. + v_float32x4(1, 2, 3, 4); + +
+ * *Load Function* - We can use the load method and provide the memory address of the data: + + float ptr[32] = {1, 2, 3, ..., 32}; + v_float32 reg_var; + reg_var = vx_load(ptr); // loads values from ptr[0] upto ptr[reg_var.nlanes - 1] + + v_float32x4 reg_128; + reg_128 = v_load(ptr); // loads values from ptr[0] upto ptr[3] + + v_float32x8 reg_256; + reg_256 = v256_load(ptr); // loads values from ptr[0] upto ptr[7] + + v_float32x16 reg_512; + reg_512 = v512_load(ptr); // loads values from ptr[0] upto ptr[15] + + @note The load function assumes data is unaligned. If your data is aligned, you may use the `vx_load_aligned()` function. +
+ +* **Store**: Store functions allow you to *store* the values from a register into a particular memory location. + * To store values from a register into a memory location, you may use the *v_store()* function: + + float ptr[4]; + v_store(ptr, reg); // store the first 128 bits(interpreted as 4x32-bit floats) of reg into ptr. +
+@note Ensure **ptr** has the same type as register. You can also cast the register into the proper type before carrying out operations. Simply typecasting the pointer to a particular type will lead wrong interpretation of data. + +### Binary and Unary Operators + +The universal intrinsics set provides element wise binary and unary operations. + +* **Arithmetics**: We can add, subtract, multiply and divide two registers element-wise. The registers must be of the same width and hold the same type. To multiply two registers, for example: + + v_float32 a, b; // {a1, ..., an}, {b1, ..., bn} + v_float32 c; + c = a + b // {a1 + b1, ..., an + bn} + c = a * b; // {a1 * b1, ..., an * bn} + +
+ +* **Bitwise Logic and Shifts**: We can left shift or right shift the bits of each element of the register. We can also apply bitwise &, |, ^ and ~ operators between two registers element-wise: + + v_int32 as; // {a1, ..., an} + v_int32 al = as << 2; // {a1 << 2, ..., an << 2} + v_int32 bl = as >> 2; // {a1 >> 2, ..., an >> 2} + + v_int32 a, b; + v_int32 a_and_b = a & b; // {a1 & b1, ..., an & bn} + +
+ +* **Comparison Operators**: We can compare values between two registers using the <, >, <= , >=, == and != operators. Since each register contains multiple values, we don't get a single bool for these operations. Instead, for true values, all bits are converted to one (0xff for 8 bits, 0xffff for 16 bits, etc), while false values return bits converted to zero. + + // let us consider the following code is run in a 128-bit register + v_uint8 a; // a = {0, 1, 2, ..., 15} + v_uint8 b; // b = {15, 14, 13, ..., 0} + + v_uint8 c = a < b; + + /* + let us look at the first 4 values in binary + + a = |00000000|00000001|00000010|00000011| + b = |00001111|00001110|00001101|00001100| + c = |11111111|11111111|11111111|11111111| + + If we store the values of c and print them as integers, we will get 255 for true values and 0 for false values. + */ + --- + // In a computer supporting 256-bit registers + v_int32 a; // a = {1, 2, 3, 4, 5, 6, 7, 8} + v_int32 b; // b = {8, 7, 6, 5, 4, 3, 2, 1} + + v_int32 c = (a < b); // c = {-1, -1, -1, -1, 0, 0, 0, 0} + + /* + The true values are 0xffffffff, which in signed 32-bit integer representation is equal to -1. + */ +
+ +* **Min/Max operations**: We can use the *v_min()* and *v_max()* functions to return registers containing element-wise min, or max, of the two registers: + + v_int32 a; // {a1, ..., an} + v_int32 b; // {b1, ..., bn} + + v_int32 mn = v_min(a, b); // {min(a1, b1), ..., min(an, bn)} + v_int32 mx = v_max(a, b); // {max(a1, b1), ..., max(an, bn)} +
+ +@note Comparison and Min/Max operators are not available for 64 bit integers. Bitwise shift and logic operators are available only for integer values. Bitwise shift is available only for 16, 32 and 64 bit registers. + +### Reduce and Mask + +* **Reduce Operations**: The *v_reduce_min()*, *v_reduce_max()* and *v_reduce_sum()* return a single value denoting the min, max or sum of the entire register: + + v_int32 a; // a = {a1, ..., a4} + int mn = v_reduce_min(a); // mn = min(a1, ..., an) + int sum = v_reduce_sum(a); // sum = a1 + ... + an +
+ +* **Mask Operations**: Mask operations allow us to replicate conditionals in wide registers. These include: + * *v_check_all()* - Returns a bool, which is true if all the values in the register are less than zero. + * *v_check_any()* - Returns a bool, which is true if any value in the register is less than zero. + * *v_select()* - Returns a register, which blends two registers, based on a mask. + + v_uint8 a; // {a1, .., an} + v_uint8 b; // {b1, ..., bn} + + v_int32x4 mask: // {0xff, 0, 0, 0xff, ..., 0xff, 0} + + v_uint8 Res = v_select(mask, a, b) // {a1, b2, b3, a4, ..., an-1, bn} + + /* + "Res" will contain the value from "a" if mask is true (all bits set to 1), + and value from "b" if mask is false (all bits set to 0) + + We can use comparison operators to generate mask and v_select to obtain results based on conditionals. + It is common to set all values of b to 0. Thus, v_select will give values of "a" or 0 based on the mask. + */ + +## Demonstration +In the following section, we will vectorize a simple convolution function for single channel and compare the results to a scalar implementation. +@note Not all algorithms are improved by manual vectorization. In fact, in certain cases, the compiler may *autovectorize* the code, thus producing faster results for scalar implementations. + +You may learn more about convolution from the previous tutorial. We use the same naive implementation from the previous tutorial and compare it to the vectorized version. + +The full tutorial code is [here](https://github.com/opencv/opencv/tree/master/samples/cpp/tutorial_code/univ_intrin/univ_intrin.cpp). + +### Vectorizing Convolution + +We will first implement a 1-D convolution and then vectorize it. The 2-D vectorized convolution will perform 1-D convolution across the rows to produce the correct results. + +#### 1-D Convolution: Scalar +@snippet univ_intrin.cpp convolution-1D-scalar + +1. We first set up variables and make a border on both sides of the src matrix, to take care of edge cases. + @snippet univ_intrin.cpp convolution-1D-border + +2. For the main loop, we select an index *i* and offset it on both sides along with the kernel, using the k variable. We store the value in *value* and add it to the *dst* matrix. + @snippet univ_intrin.cpp convolution-1D-scalar-main + +#### 1-D Convolution: Vector + +We will now look at the vectorized version of 1-D convolution. +@snippet univ_intrin.cpp convolution-1D-vector + +1. In our case, the kernel is a float. Since the kernel's datatype is the largest, we convert src to float32, forming *src_32*. We also make a border like we did for the naive case. + @snippet univ_intrin.cpp convolution-1D-convert + +2. Now, for each column in the *kernel*, we calculate the scalar product of the value with all *window* vectors of length `step`. We add these values to the already stored values in ans + @snippet univ_intrin.cpp convolution-1D-main + + * We declare a pointer to the src_32 and kernel and run a loop for each kernel element + @snippet univ_intrin.cpp convolution-1D-main-h1 + + * We load a register with the current kernel element. A window is shifted from *0* to *len - step* and its product with the kernel_wide array is added to the values stored in *ans*. We store the values back into *ans* + @snippet univ_intrin.cpp convolution-1D-main-h2 + + * Since the length might not be divisible by steps, we take care of the remaining values directly. The number of *tail* values will always be less than *step* and will not affect the performance significantly. We store all the values to *ans* which is a float pointer. We can also directly store them in a `Mat` object + @snippet univ_intrin.cpp convolution-1D-main-h3 + + * Here is an iterative example: + + For example: + kernel: {k1, k2, k3} + src: ...|a1|a2|a3|a4|... + + + iter1: + for each idx i in (0, len), 'step' idx at a time + kernel_wide: |k1|k1|k1|k1| + window: |a0|a1|a2|a3| + ans: ...| 0| 0| 0| 0|... + sum = ans + window * kernel_wide + = |a0 * k1|a1 * k1|a2 * k1|a3 * k1| + + iter2: + kernel_wide: |k2|k2|k2|k2| + window: |a1|a2|a3|a4| + ans: ...|a0 * k1|a1 * k1|a2 * k1|a3 * k1|... + sum = ans + window * kernel_wide + = |a0 * k1 + a1 * k2|a1 * k1 + a2 * k2|a2 * k1 + a3 * k2|a3 * k1 + a4 * k2| + + iter3: + kernel_wide: |k3|k3|k3|k3| + window: |a2|a3|a4|a5| + ans: ...|a0 * k1 + a1 * k2|a1 * k1 + a2 * k2|a2 * k1 + a3 * k2|a3 * k1 + a4 * k2|... + sum = sum + window * kernel_wide + = |a0*k1 + a1*k2 + a2*k3|a1*k1 + a2*k2 + a3*k3|a2*k1 + a3*k2 + a4*k3|a3*k1 + a4*k2 + a5*k3| + + +@note The function parameters also include *row*, *rowk* and *len*. These values are used when using the function as an intermediate step of 2-D convolution + +#### 2-D Convolution + +Suppose our kernel has *ksize* rows. To compute the values for a particular row, we compute the 1-D convolution of the previous *ksize/2* and the next *ksize/2* rows, with the corresponding kernel row. The final values is simply the sum of the individual 1-D convolutions +@snippet univ_intrin.cpp convolution-2D + +1. We first initialize variables and make a border above and below the *src* matrix. The left and right sides are handled by the 1-D convolution function. + @snippet univ_intrin.cpp convolution-2D-init + +2. For each row, we calculate the 1-D convolution of the rows above and below it. we then add the values to the *dst* matrix. + @snippet univ_intrin.cpp convolution-2D-main + +3. We finally convert the *dst* matrix to a *8-bit* `unsigned char` matrix + @snippet univ_intrin.cpp convolution-2D-conv + +Results +------- + +In the tutorial, we used a horizontal gradient kernel. We obtain the same output image for both methods. + +Improvement in runtime varies and will depend on the SIMD capabilities available in your CPU. diff --git a/doc/tutorials/dnn/dnn_face/dnn_face.markdown b/doc/tutorials/dnn/dnn_face/dnn_face.markdown new file mode 100644 index 0000000000..e5092b8b92 --- /dev/null +++ b/doc/tutorials/dnn/dnn_face/dnn_face.markdown @@ -0,0 +1,95 @@ +# DNN-based Face Detection And Recognition {#tutorial_dnn_face} + +@tableofcontents + +@prev_tutorial{tutorial_dnn_text_spotting} +@next_tutorial{pytorch_cls_tutorial_dnn_conversion} + +| | | +| -: | :- | +| Original Author | Chengrui Wang, Yuantao Feng | +| Compatibility | OpenCV >= 4.5.1 | + +## Introduction + +In this section, we introduce the DNN-based module for face detection and face recognition. Models can be obtained in [Models](#Models). The usage of `FaceDetectorYN` and `FaceRecognizer` are presented in [Usage](#Usage). + +## Models + +There are two models (ONNX format) pre-trained and required for this module: +- [Face Detection](https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx): + - Size: 337KB + - Results on WIDER Face Val set: 0.830(easy), 0.824(medium), 0.708(hard) +- [Face Recognition](https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view?usp=sharing) + - Size: 36.9MB + - Results: + + | Database | Accuracy | Threshold (normL2) | Threshold (cosine) | + | -------- | -------- | ------------------ | ------------------ | + | LFW | 99.60% | 1.128 | 0.363 | + | CALFW | 93.95% | 1.149 | 0.340 | + | CPLFW | 91.05% | 1.204 | 0.275 | + | AgeDB-30 | 94.90% | 1.202 | 0.277 | + | CFP-FP | 94.80% | 1.253 | 0.212 | + +## Usage + +### DNNFaceDetector + +```cpp +// Initialize FaceDetectorYN +Ptr faceDetector = FaceDetectorYN::create(onnx_path, "", image.size(), score_thresh, nms_thresh, top_k); + +// Forward +Mat faces; +faceDetector->detect(image, faces); +``` + +The detection output `faces` is a two-dimension array of type CV_32F, whose rows are the detected face instances, columns are the location of a face and 5 facial landmarks. The format of each row is as follows: + +``` +x1, y1, w, h, x_re, y_re, x_le, y_le, x_nt, y_nt, x_rcm, y_rcm, x_lcm, y_lcm +``` +, where `x1, y1, w, h` are the top-left coordinates, width and height of the face bounding box, `{x, y}_{re, le, nt, rcm, lcm}` stands for the coordinates of right eye, left eye, nose tip, the right corner and left corner of the mouth respectively. + + +### Face Recognition + +Following Face Detection, run codes below to extract face feature from facial image. + +```cpp +// Initialize FaceRecognizer with model path (cv::String) +Ptr faceRecognizer = FaceRecognizer::create(model_path, ""); + +// Aligning and cropping facial image through the first face of faces detected by dnn_face::DNNFaceDetector +Mat aligned_face; +faceRecognizer->alignCrop(image, faces.row(0), aligned_face); + +// Run feature extraction with given aligned_face (cv::Mat) +Mat feature; +faceRecognizer->feature(aligned_face, feature); +feature = feature.clone(); +``` + +After obtaining face features *feature1* and *feature2* of two facial images, run codes below to calculate the identity discrepancy between the two faces. + +```cpp +// Calculating the discrepancy between two face features by using cosine distance. +double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizer::DisType::COSINE); +// Calculating the discrepancy between two face features by using normL2 distance. +double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizer::DisType::NORM_L2); +``` + +For example, two faces have same identity if the cosine distance is greater than or equal to 0.363, or the normL2 distance is less than or equal to 1.128. + +## Reference: + +- https://github.com/ShiqiYu/libfacedetection +- https://github.com/ShiqiYu/libfacedetection.train +- https://github.com/zhongyy/SFace + +## Acknowledgement + +Thanks [Professor Shiqi Yu](https://github.com/ShiqiYu/) and [Yuantao Feng](https://github.com/fengyuentau) for training and providing the face detection model. + +Thanks [Professor Deng](http://www.whdeng.cn/), [PhD Candidate Zhong](https://github.com/zhongyy/) and [Master Candidate Wang](https://github.com/crywang/) for training and providing the face recognition model. diff --git a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown index 5f28b6ce7a..5c465941ca 100644 --- a/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown +++ b/doc/tutorials/dnn/dnn_text_spotting/dnn_text_spotting.markdown @@ -3,7 +3,7 @@ @tableofcontents @prev_tutorial{tutorial_dnn_OCR} -@next_tutorial{pytorch_cls_tutorial_dnn_conversion} +@next_tutorial{tutorial_dnn_face} | | | | -: | :- | @@ -26,6 +26,11 @@ Before recognition, you should `setVocabulary` and `setDecodeType`. - `T` is the sequence length - `B` is the batch size (only support `B=1` in inference) - and `Dim` is the length of vocabulary +1('Blank' of CTC is at the index=0 of Dim). +- "CTC-prefix-beam-search", the output of the text recognition model should be a probability matrix same with "CTC-greedy". + - The algorithm is proposed at Hannun's [paper](https://arxiv.org/abs/1408.2873). + - `setDecodeOptsCTCPrefixBeamSearch` could be used to control the beam size in search step. + - To futher optimize for big vocabulary, a new option `vocPruneSize` is introduced to avoid iterate the whole vocbulary + but only the number of `vocPruneSize` tokens with top probabilty. @ref cv::dnn::TextRecognitionModel::recognize() is the main function for text recognition. - The input image should be a cropped text image or an image with `roiRects` diff --git a/doc/tutorials/dnn/table_of_content_dnn.markdown b/doc/tutorials/dnn/table_of_content_dnn.markdown index 0d5e43ee11..3f74826dac 100644 --- a/doc/tutorials/dnn/table_of_content_dnn.markdown +++ b/doc/tutorials/dnn/table_of_content_dnn.markdown @@ -10,6 +10,7 @@ Deep Neural Networks (dnn module) {#tutorial_table_of_content_dnn} - @subpage tutorial_dnn_custom_layers - @subpage tutorial_dnn_OCR - @subpage tutorial_dnn_text_spotting +- @subpage tutorial_dnn_face #### PyTorch models with OpenCV In this section you will find the guides, which describe how to run classification, segmentation and detection PyTorch DNN models with OpenCV. diff --git a/doc/tutorials/introduction/config_reference/config_reference.markdown b/doc/tutorials/introduction/config_reference/config_reference.markdown index 438cc70288..88ac9bdeb6 100644 --- a/doc/tutorials/introduction/config_reference/config_reference.markdown +++ b/doc/tutorials/introduction/config_reference/config_reference.markdown @@ -589,6 +589,14 @@ Some features have been added specifically for automated build environments, lik | `OPENCV_CMAKE_HOOKS_DIR` | _empty_ | OpenCV allows to customize configuration process by adding custom hook scripts at each stage and substage. cmake scripts with predefined names located in the directory set by this variable will be included before and after various configuration stages. Examples of file names: _CMAKE_INIT.cmake_, _PRE_CMAKE_BOOTSTRAP.cmake_, _POST_CMAKE_BOOTSTRAP.cmake_, etc.. Other names are not documented and can be found in the project cmake files by searching for the _ocv_cmake_hook_ macro calls. | | `OPENCV_DUMP_HOOKS_FLOW` | _OFF_ | Enables a debug message print on each cmake hook script call. | +## Contrib Modules + +Following build options are utilized in `opencv_contrib` modules, as stated [previously](#tutorial_config_reference_general_contrib), these extra modules can be added to your final build by setting `DOPENCV_EXTRA_MODULES_PATH` option. + +| Option | Default | Description | +| ------ | ------- | ----------- | +| `WITH_CLP` | _OFF_ | Will add [coinor](https://projects.coin-or.org/Clp) linear programming library build support which is required in `videostab` module. Make sure to install the development libraries of coinor-clp. | + # Other non-documented options @@ -605,7 +613,6 @@ Some features have been added specifically for automated build environments, lik `WITH_CPUFEATURES` `WITH_EIGEN` `WITH_OPENVX` -`WITH_CLP` `WITH_DIRECTX` `WITH_VA` `WITH_LAPACK` diff --git a/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown b/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown index eb59fea209..ee3f1eb7f9 100644 --- a/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown +++ b/doc/tutorials/introduction/linux_gcc_cmake/linux_gcc_cmake.markdown @@ -1,7 +1,7 @@ Using OpenCV with gcc and CMake {#tutorial_linux_gcc_cmake} =============================== -@prev_tutorial{tutorial_linux_install} +@prev_tutorial{tutorial_linux_gdb_pretty_printer} @next_tutorial{tutorial_linux_eclipse} | | | diff --git a/doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png b/doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png new file mode 100644 index 0000000000..0ec673dcc2 Binary files /dev/null and b/doc/tutorials/introduction/linux_gdb_pretty_printer/images/example.png differ diff --git a/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown b/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown new file mode 100644 index 0000000000..9d64469920 --- /dev/null +++ b/doc/tutorials/introduction/linux_gdb_pretty_printer/linux_gdb_pretty_printer.markdown @@ -0,0 +1,38 @@ +Using OpenCV with gdb-powered IDEs {#tutorial_linux_gdb_pretty_printer} +===================== + +@prev_tutorial{tutorial_linux_install} +@next_tutorial{tutorial_linux_gcc_cmake} + +| | | +| -: | :- | +| Original author | Egor Smirnov | +| Compatibility | OpenCV >= 4.0 | + +@tableofcontents + +# Capabilities {#tutorial_linux_gdb_pretty_printer_capabilities} + +This pretty-printer can show element type, `is_continuous`, `is_submatrix` flags and (possibly truncated) matrix. It is known to work in Clion, VS Code and gdb. + +![Clion example](images/example.png) + + +# Installation {#tutorial_linux_gdb_pretty_printer_installation} + +Move into `opencv/samples/gdb/`. Place `mat_pretty_printer.py` in a convinient place, rename `gdbinit` to `.gdbinit` and move it into your home folder. Change 'source' line of `.gdbinit` to point to your `mat_pretty_printer.py` path. + +In order to check version of python bundled with your gdb, use the following commands from the gdb shell: + + python + import sys + print(sys.version_info) + end + +If the version of python 3 installed in your system doesn't match the version in gdb, create a new virtual environment with the exact same version, install `numpy` and change the path to python3 in `.gdbinit` accordingly. + + +# Usage {#tutorial_linux_gdb_pretty_printer_usage} + +The fields in a debugger prefixed with `view_` are pseudo-fields added for convinience, the rest are left as is. +If you feel that the number of elements in truncated view is too low, you can edit `mat_pretty_printer.py` - `np.set_printoptions` controlls everything matrix display-related. diff --git a/doc/tutorials/introduction/linux_install/linux_install.markdown b/doc/tutorials/introduction/linux_install/linux_install.markdown index 46f67ac61c..f3a60145d0 100644 --- a/doc/tutorials/introduction/linux_install/linux_install.markdown +++ b/doc/tutorials/introduction/linux_install/linux_install.markdown @@ -1,7 +1,7 @@ Installation in Linux {#tutorial_linux_install} ===================== -@next_tutorial{tutorial_linux_gcc_cmake} +@next_tutorial{tutorial_linux_gdb_pretty_printer} | | | | -: | :- | diff --git a/doc/tutorials/introduction/table_of_content_introduction.markdown b/doc/tutorials/introduction/table_of_content_introduction.markdown index d1f2aa3ca3..8fa89d7d7f 100644 --- a/doc/tutorials/introduction/table_of_content_introduction.markdown +++ b/doc/tutorials/introduction/table_of_content_introduction.markdown @@ -6,6 +6,7 @@ Introduction to OpenCV {#tutorial_table_of_content_introduction} ##### Linux - @subpage tutorial_linux_install +- @subpage tutorial_linux_gdb_pretty_printer - @subpage tutorial_linux_gcc_cmake - @subpage tutorial_linux_eclipse diff --git a/doc/tutorials/others/traincascade.markdown b/doc/tutorials/others/traincascade.markdown index 0d95bd003a..fdb38511be 100644 --- a/doc/tutorials/others/traincascade.markdown +++ b/doc/tutorials/others/traincascade.markdown @@ -13,6 +13,8 @@ Working with a boosted cascade of weak classifiers includes two major stages: th To support this tutorial, several official OpenCV applications will be used: [opencv_createsamples](https://github.com/opencv/opencv/tree/master/apps/createsamples), [opencv_annotation](https://github.com/opencv/opencv/tree/master/apps/annotation), [opencv_traincascade](https://github.com/opencv/opencv/tree/master/apps/traincascade) and [opencv_visualisation](https://github.com/opencv/opencv/tree/master/apps/visualisation). +@note Createsamples and traincascade are disabled since OpenCV 4.0. Consider using these apps for training from 3.4 branch for Cascade Classifier. Model format is the same between 3.4 and 4.x. + ### Important notes - If you come across any tutorial mentioning the old opencv_haartraining tool (which is deprecated and still using the OpenCV1.x interface), then please ignore that tutorial and stick to the opencv_traincascade tool. This tool is a newer version, written in C++ in accordance to the OpenCV 2.x and OpenCV 3.x API. The opencv_traincascade supports both HAAR like wavelet features @cite Viola01 and LBP (Local Binary Patterns) @cite Liao2007 features. LBP features yield integer precision in contrast to HAAR features, yielding floating point precision, so both training and detection with LBP are several times faster then with HAAR features. Regarding the LBP and HAAR detection quality, it mainly depends on the training data used and the training parameters selected. It's possible to train a LBP-based classifier that will provide almost the same quality as HAAR-based one, within a percentage of the training time. diff --git a/modules/3d/include/opencv2/3d.hpp b/modules/3d/include/opencv2/3d.hpp index 6750139e37..82ba5e2cdb 100644 --- a/modules/3d/include/opencv2/3d.hpp +++ b/modules/3d/include/opencv2/3d.hpp @@ -1309,7 +1309,6 @@ CV_EXPORTS_W int solvePnPGeneric( InputArray objectPoints, InputArray imagePoint InputArray rvec = noArray(), InputArray tvec = noArray(), OutputArray reprojectionError = noArray() ); - /** @brief Draw axes of the world/object coordinate system from pose estimation. @sa solvePnP @param image Input/output image. It must have 1 or 3 channels. The number of channels is not altered. @@ -1611,6 +1610,76 @@ unit length. */ CV_EXPORTS_W void decomposeEssentialMat( InputArray E, OutputArray R1, OutputArray R2, OutputArray t ); +/** @brief Recovers the relative camera rotation and the translation from corresponding points in two images from two different cameras, using cheirality check. Returns the number of +inliers that pass the check. + +@param points1 Array of N 2D points from the first image. The point coordinates should be +floating-point (single or double precision). +@param points2 Array of the second image points of the same size and format as points1 . +@param cameraMatrix1 Input/output camera matrix for the first camera, the same as in +@ref calibrateCamera. Furthermore, for the stereo case, additional flags may be used, see below. +@param distCoeffs1 Input/output vector of distortion coefficients, the same as in +@ref calibrateCamera. +@param cameraMatrix2 Input/output camera matrix for the first camera, the same as in +@ref calibrateCamera. Furthermore, for the stereo case, additional flags may be used, see below. +@param distCoeffs2 Input/output vector of distortion coefficients, the same as in +@ref calibrateCamera. +@param E The output essential matrix. +@param R Output rotation matrix. Together with the translation vector, this matrix makes up a tuple +that performs a change of basis from the first camera's coordinate system to the second camera's +coordinate system. Note that, in general, t can not be used for this tuple, see the parameter +described below. +@param t Output translation vector. This vector is obtained by @ref decomposeEssentialMat and +therefore is only known up to scale, i.e. t is the direction of the translation vector and has unit +length. +@param method Method for computing an essential matrix. +- @ref RANSAC for the RANSAC algorithm. +- @ref LMEDS for the LMedS algorithm. +@param prob Parameter used for the RANSAC or LMedS methods only. It specifies a desirable level of +confidence (probability) that the estimated matrix is correct. +@param threshold Parameter used for RANSAC. It is the maximum distance from a point to an epipolar +line in pixels, beyond which the point is considered an outlier and is not used for computing the +final fundamental matrix. It can be set to something like 1-3, depending on the accuracy of the +point localization, image resolution, and the image noise. +@param mask Input/output mask for inliers in points1 and points2. If it is not empty, then it marks +inliers in points1 and points2 for then given essential matrix E. Only these inliers will be used to +recover pose. In the output mask only inliers which pass the cheirality check. + +This function decomposes an essential matrix using @ref decomposeEssentialMat and then verifies +possible pose hypotheses by doing cheirality check. The cheirality check means that the +triangulated 3D points should have positive depth. Some details can be found in @cite Nister03. + +This function can be used to process the output E and mask from @ref findEssentialMat. In this +scenario, points1 and points2 are the same input for findEssentialMat.: +@code + // Example. Estimation of fundamental matrix using the RANSAC algorithm + int point_count = 100; + vector points1(point_count); + vector points2(point_count); + + // initialize the points here ... + for( int i = 0; i < point_count; i++ ) + { + points1[i] = ...; + points2[i] = ...; + } + + // Input: camera calibration of both cameras, for example using intrinsic chessboard calibration. + Mat cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2; + + // Output: Essential matrix, relative rotation and relative translation. + Mat E, R, t, mask; + + recoverPose(points1, points2, cameraMatrix1, distCoeffs1, cameraMatrix2, distCoeffs2, E, R, t, mask); +@endcode + */ +CV_EXPORTS_W int recoverPose( InputArray points1, InputArray points2, + InputArray cameraMatrix1, InputArray distCoeffs1, + InputArray cameraMatrix2, InputArray distCoeffs2, + OutputArray E, OutputArray R, OutputArray t, + int method = cv::RANSAC, double prob = 0.999, double threshold = 1.0, + InputOutputArray mask = noArray()); + /** @brief Recovers the relative camera rotation and the translation from an estimated essential matrix and the corresponding points in two images, using cheirality check. Returns the number of inliers that pass the check. @@ -2303,6 +2372,7 @@ void initInverseRectificationMap( InputArray cameraMatrix, InputArray distCoeffs InputArray R, InputArray newCameraMatrix, const Size& size, int m1type, OutputArray map1, OutputArray map2 ); + //! initializes maps for #remap for wide-angle CV_EXPORTS float initWideAngleProjMap(InputArray cameraMatrix, InputArray distCoeffs, diff --git a/modules/3d/src/ap3p.cpp b/modules/3d/src/ap3p.cpp index 3aea8346d4..34ac9c4323 100644 --- a/modules/3d/src/ap3p.cpp +++ b/modules/3d/src/ap3p.cpp @@ -9,9 +9,9 @@ static inline double cbrt(double x) { return (double)cv::cubeRoot((float)x); }; namespace cv { -using namespace std; - -static void solveQuartic(const double *factors, double *realRoots) { +static +void solveQuartic(const double *factors, double *realRoots) +{ const double &a4 = factors[0]; const double &a3 = factors[1]; const double &a2 = factors[2]; @@ -31,29 +31,29 @@ static void solveQuartic(const double *factors, double *realRoots) { double q3 = (72 * r4 * p4 - 2 * p4 * p4 * p4 - 27 * q4 * q4) / 432; // /=2 double t; // *=2 - complex w; + std::complex w; if (q3 >= 0) - w = -sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; + w = -std::sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; else - w = sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; + w = std::sqrt(static_cast >(q3 * q3 - p3 * p3 * p3)) - q3; if (w.imag() == 0.0) { - w.real(cbrt(w.real())); + w.real(std::cbrt(w.real())); t = 2.0 * (w.real() + p3 / w.real()); } else { w = pow(w, 1.0 / 3); t = 4.0 * w.real(); } - complex sqrt_2m = sqrt(static_cast >(-2 * p4 / 3 + t)); + std::complex sqrt_2m = sqrt(static_cast >(-2 * p4 / 3 + t)); double B_4A = -a3 / (4 * a4); double complex1 = 4 * p4 / 3 + t; #if defined(__clang__) && defined(__arm__) && (__clang_major__ == 3 || __clang_major__ == 4) && !defined(__ANDROID__) // details: https://github.com/opencv/opencv/issues/11135 // details: https://github.com/opencv/opencv/issues/11056 - complex complex2 = 2 * q4; - complex2 = complex(complex2.real() / sqrt_2m.real(), 0); + std::complex complex2 = 2 * q4; + complex2 = std::complex(complex2.real() / sqrt_2m.real(), 0); #else - complex complex2 = 2 * q4 / sqrt_2m; + std::complex complex2 = 2 * q4 / sqrt_2m; #endif double sqrt_2m_rh = sqrt_2m.real() / 2; double sqrt1 = sqrt(-(complex1 + complex2)).real() / 2; diff --git a/modules/3d/src/five-point.cpp b/modules/3d/src/five-point.cpp index 084f791f26..746df74423 100644 --- a/modules/3d/src/five-point.cpp +++ b/modules/3d/src/five-point.cpp @@ -399,6 +399,31 @@ protected: } }; + +// Find essential matrix given undistorted points and two cameras. +static Mat findEssentialMat_( InputArray _points1, InputArray _points2, + InputArray cameraMatrix1, InputArray cameraMatrix2, + int method, double prob, double threshold, OutputArray _mask) +{ + // Scale the points back. We use "arithmetic mean" between the supplied two camera matrices. + // Thanks to such 2-stage procedure RANSAC threshold still makes sense, because the undistorted + // and rescaled points have a similar value range to the original ones. + Mat _pointsTransformed1, _pointsTransformed2; + Mat cm1 = cameraMatrix1.getMat(), cm2 = cameraMatrix2.getMat(), cm0; + Mat(cm1 + cm2).convertTo(cm0, CV_64F, 0.5); + CV_Assert(cm0.rows == 3 && cm0.cols == 3); + CV_Assert(std::abs(cm0.at(2, 0)) < 1e-3 && + std::abs(cm0.at(2, 1)) < 1e-3 && + std::abs(cm0.at(2, 2) - 1.) < 1e-3); + Mat affine = cm0.rowRange(0, 2); + + transform(_points1, _pointsTransformed1, affine); + transform(_points2, _pointsTransformed2, affine); + + return findEssentialMat(_pointsTransformed1, _pointsTransformed2, cm0, method, prob, threshold, _mask); +} + + // Input should be a vector of n 2D points or a Nx2 matrix Mat findEssentialMat( InputArray _points1, InputArray _points2, InputArray _cameraMatrix, int method, double prob, double threshold, @@ -485,25 +510,10 @@ Mat findEssentialMat( InputArray _points1, InputArray _points2, CV_INSTRUMENT_REGION(); // Undistort image points, bring them to 3x3 identity "camera matrix" - Mat _pointsUntistorted1, _pointsUntistorted2; - undistortPoints(_points1, _pointsUntistorted1, cameraMatrix1, distCoeffs1); - undistortPoints(_points2, _pointsUntistorted2, cameraMatrix2, distCoeffs2); - - // Scale the points back. We use "arithmetic mean" between the supplied two camera matrices. - // Thanks to such 2-stage procedure RANSAC threshold still makes sense, because the undistorted - // and rescaled points have a similar value range to the original ones. - Mat cm1 = cameraMatrix1.getMat(), cm2 = cameraMatrix2.getMat(), cm0; - Mat(cm1 + cm2).convertTo(cm0, CV_64F, 0.5); - CV_Assert(cm0.rows == 3 && cm0.cols == 3); - CV_Assert(std::abs(cm0.at(2, 0)) < 1e-3 && - std::abs(cm0.at(2, 1)) < 1e-3 && - std::abs(cm0.at(2, 2) - 1.) < 1e-3); - Mat affine = cm0.rowRange(0, 2); - - transform(_pointsUntistorted1, _pointsUntistorted1, affine); - transform(_pointsUntistorted2, _pointsUntistorted2, affine); - - return findEssentialMat(_pointsUntistorted1, _pointsUntistorted2, cm0, method, prob, threshold, _mask); + Mat _pointsUndistorted1, _pointsUndistorted2; + undistortPoints(_points1, _pointsUndistorted1, cameraMatrix1, distCoeffs1); + undistortPoints(_points2, _pointsUndistorted2, cameraMatrix2, distCoeffs2); + return findEssentialMat_(_pointsUndistorted1, _pointsUndistorted2, cameraMatrix1, cameraMatrix2, method, prob, threshold, _mask); } Mat findEssentialMat( InputArray points1, InputArray points2, @@ -520,9 +530,33 @@ Mat findEssentialMat( InputArray points1, InputArray points2, } +int recoverPose( InputArray _points1, InputArray _points2, + InputArray cameraMatrix1, InputArray distCoeffs1, + InputArray cameraMatrix2, InputArray distCoeffs2, + OutputArray E, OutputArray R, OutputArray t, + int method, double prob, double threshold, + InputOutputArray _mask) +{ + CV_INSTRUMENT_REGION(); + + // Undistort image points, bring them to 3x3 identity "camera matrix" + Mat _pointsUndistorted1, _pointsUndistorted2; + undistortPoints(_points1, _pointsUndistorted1, cameraMatrix1, distCoeffs1); + undistortPoints(_points2, _pointsUndistorted2, cameraMatrix2, distCoeffs2); + + // Get essential matrix. + Mat _E = findEssentialMat_(_pointsUndistorted1, _pointsUndistorted2, cameraMatrix1, cameraMatrix2, + method, prob, threshold, _mask); + CV_Assert(_E.cols == 3 && _E.rows == 3); + E.create(3, 3, _E.type()); + _E.copyTo(E); + + return recoverPose(_E, _pointsUndistorted1, _pointsUndistorted2, Mat::eye(3,3, CV_64F), R, t, _mask); +} + int recoverPose( InputArray E, InputArray _points1, InputArray _points2, - InputArray _cameraMatrix, OutputArray _R, OutputArray _t, double distanceThresh, - InputOutputArray _mask, OutputArray triangulatedPoints) + InputArray _cameraMatrix, OutputArray _R, OutputArray _t, double distanceThresh, + InputOutputArray _mask, OutputArray triangulatedPoints) { CV_INSTRUMENT_REGION(); diff --git a/modules/3d/test/test_undistort.cpp b/modules/3d/test/test_undistort.cpp index 92d1050c9b..6e0ccae6df 100644 --- a/modules/3d/test/test_undistort.cpp +++ b/modules/3d/test/test_undistort.cpp @@ -897,7 +897,7 @@ void CV_InitInverseRectificationMapTest::prepare_to_validation(int/* test_case_i Mat _new_cam0 = zero_new_cam ? test_mat[INPUT][0] : test_mat[INPUT][3]; Mat _mapx(img_size, CV_32F), _mapy(img_size, CV_32F); - double a[9], d[5]={0,0,0,0,0}, R[9]={1, 0, 0, 0, 1, 0, 0, 0, 1}, a1[9]; + double a[9], d[5]={0., 0., 0., 0. , 0.}, R[9]={1., 0., 0., 0., 1., 0., 0., 0., 1.}, a1[9]; Mat _a(3, 3, CV_64F, a), _a1(3, 3, CV_64F, a1); Mat _d(_d0.rows,_d0.cols, CV_MAKETYPE(CV_64F,_d0.channels()),d); Mat _R(3, 3, CV_64F, R); @@ -951,9 +951,9 @@ void CV_InitInverseRectificationMapTest::prepare_to_validation(int/* test_case_i // Undistort double x2 = x*x, y2 = y*y; double r2 = x2 + y2; - double cdist = 1./(1 + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1 + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; - double x_ = x*cdist - d[2]*2*x*y + d[3]*(r2 + 2*x2); - double y_ = y*cdist - d[3]*2*x*y + d[2]*(r2 + 2*y2); + double cdist = 1./(1. + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1. + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; + double x_ = (x - (d[2]*2.*x*y + d[3]*(r2 + 2.*x2)))*cdist; + double y_ = (y - (d[3]*2.*x*y + d[2]*(r2 + 2.*y2)))*cdist; // Rectify double X = R[0]*x_ + R[1]*y_ + R[2]; @@ -1807,4 +1807,78 @@ TEST(Calib3d_initUndistortRectifyMap, regression_14467) EXPECT_LE(cvtest::norm(dst, mesh_uv, NORM_INF), 1e-3); } +TEST(Calib3d_initInverseRectificationMap, regression_20165) +{ + Size size_w_h(1280, 800); + Mat dst(size_w_h, CV_32FC2); // Reference for validation + Mat mapxy; // Output of initInverseRectificationMap() + + // Camera Matrix + double k[9]={ + 1.5393951443032472e+03, 0., 6.7491727003047140e+02, + 0., 1.5400748240626747e+03, 5.1226968329123963e+02, + 0., 0., 1. + }; + Mat _K(3, 3, CV_64F, k); + + // Distortion + // double d[5]={0,0,0,0,0}; // Zero Distortion + double d[5]={ // Non-zero distortion + -3.4134571357400023e-03, 2.9733267766101856e-03, // K1, K2 + 3.6653586399031184e-03, -3.1960714017365702e-03, // P1, P2 + 0. // K3 + }; + Mat _d(1, 5, CV_64F, d); + + // Rotation + //double R[9]={1., 0., 0., 0., 1., 0., 0., 0., 1.}; // Identity transform (none) + double R[9]={ // Random transform + 9.6625486010428052e-01, 1.6055789378989216e-02, 2.5708706103628531e-01, + -8.0300261706161002e-03, 9.9944797497929860e-01, -3.2237617614807819e-02, + -2.5746274294459848e-01, 2.9085338870243265e-02, 9.6585039165403186e-01 + }; + Mat _R(3, 3, CV_64F, R); + + // --- Validation --- // + initInverseRectificationMap(_K, _d, _R, _K, size_w_h, CV_32FC2, mapxy, noArray()); + + // Copy camera matrix + double fx, fy, cx, cy, ifx, ify, cxn, cyn; + fx = k[0]; fy = k[4]; cx = k[2]; cy = k[5]; + + // Copy new camera matrix + ifx = k[0]; ify = k[4]; cxn = k[2]; cyn = k[5]; + + // Distort Points + for( int v = 0; v < size_w_h.height; v++ ) + { + for( int u = 0; u < size_w_h.width; u++ ) + { + // Convert from image to pin-hole coordinates + double x = (u - cx)/fx; + double y = (v - cy)/fy; + + // Undistort + double x2 = x*x, y2 = y*y; + double r2 = x2 + y2; + double cdist = 1./(1. + (d[0] + (d[1] + d[4]*r2)*r2)*r2); // (1. + (d[5] + (d[6] + d[7]*r2)*r2)*r2) == 1 as d[5-7]=0; + double x_ = (x - (d[2]*2.*x*y + d[3]*(r2 + 2.*x2)))*cdist; + double y_ = (y - (d[3]*2.*x*y + d[2]*(r2 + 2.*y2)))*cdist; + + // Rectify + double X = R[0]*x_ + R[1]*y_ + R[2]; + double Y = R[3]*x_ + R[4]*y_ + R[5]; + double Z = R[6]*x_ + R[7]*y_ + R[8]; + double x__ = X/Z; + double y__ = Y/Z; + + // Convert from pin-hole to image coordinates + dst.at(v, u) = Vec2f((float)(x__*ifx + cxn), (float)(y__*ify + cyn)); + } + } + + // Check Result + EXPECT_LE(cvtest::norm(dst, mapxy, NORM_INF), 2e-1); +} + }} // namespace diff --git a/modules/calib/include/opencv2/calib.hpp b/modules/calib/include/opencv2/calib.hpp index 460ac2657f..00c5a19d9d 100644 --- a/modules/calib/include/opencv2/calib.hpp +++ b/modules/calib/include/opencv2/calib.hpp @@ -525,6 +525,8 @@ Sample usage of detecting and drawing chessboard corners: : the board to make the detection more robust in various environments. Otherwise, if there is no border and the background is dark, the outer black squares cannot be segmented properly and so the square grouping and ordering algorithm fails. + +Use gen_pattern.py (@ref tutorial_camera_calibration_pattern) to create checkerboard. */ CV_EXPORTS_W bool findChessboardCorners( InputArray image, Size patternSize, OutputArray corners, int flags = CALIB_CB_ADAPTIVE_THRESH + CALIB_CB_NORMALIZE_IMAGE ); @@ -581,6 +583,8 @@ transformation it is beneficial to use round corners for the field corners which are located on the outside of the board. The following figure illustrates a sample checkerboard optimized for the detection. However, any other checkerboard can be used as well. + +Use gen_pattern.py (@ref tutorial_camera_calibration_pattern) to create checkerboard. ![Checkerboard](pics/checkerboard_radon.png) */ CV_EXPORTS_AS(findChessboardCornersSBWithMeta) diff --git a/modules/calib/src/chessboard.cpp b/modules/calib/src/chessboard.cpp index 53ace9df2b..5ac9c2d6ac 100644 --- a/modules/calib/src/chessboard.cpp +++ b/modules/calib/src/chessboard.cpp @@ -1707,10 +1707,10 @@ void Chessboard::Board::normalizeOrientation(bool bblack) iter_bottom_left.getCell()->empty() || iter_bottom_right.getCell()->empty()) return; - float d1 = pow(top_left->top_left->x,2)+pow(top_left->top_left->y,2); - float d2 = pow((*iter_top_right)->x,2)+pow((*iter_top_right)->y,2); - float d3 = pow((*iter_bottom_left)->x,2)+pow((*iter_bottom_left)->y,2); - float d4 = pow((*iter_bottom_right)->x,2)+pow((*iter_bottom_right)->y,2); + float d1 = top_left->top_left->dot(*top_left->top_left); + float d2 = (*iter_top_right)->dot(*(*iter_top_right)); + float d3 = (*iter_bottom_left)->dot(*(*iter_bottom_left)); + float d4 = (*iter_bottom_right)->dot(*(*iter_bottom_right)); if(d2 <= d1 && d2 <= d3 && d2 <= d4) // top left is top right rotateLeft(); else if(d3 <= d1 && d3 <= d2 && d3 <= d4) // top left is bottom left @@ -3924,7 +3924,7 @@ bool cv::findChessboardCornersSB(InputArray image_, Size pattern_size, { meta_.create(int(board.rowCount()),int(board.colCount()),CV_8UC1); cv::Mat meta = meta_.getMat(); - meta = 0; + meta.setTo(cv::Scalar::all(0)); for(int row =0;row < meta.rows-1;++row) { for(int col=0;col< meta.cols-1;++col) diff --git a/modules/calib/test/test_cameracalibration.cpp b/modules/calib/test/test_cameracalibration.cpp index f1d0f873ed..60a4f91414 100644 --- a/modules/calib/test/test_cameracalibration.cpp +++ b/modules/calib/test/test_cameracalibration.cpp @@ -2144,7 +2144,17 @@ TEST(CV_RecoverPoseTest, regression_15341) // camera matrix with both focal lengths = 1, and principal point = (0, 0) const Mat cameraMatrix = Mat::eye(3, 3, CV_64F); - const Mat zeroDistCoeffs = Mat::zeros(1, 5, CV_64F); + + // camera matrix with focal lengths 0.5 and 0.6 respectively and principal point = (100, 200) + double cameraMatrix2Data[] = { 0.5, 0, 100, + 0, 0.6, 200, + 0, 0, 1 }; + const Mat cameraMatrix2( 3, 3, CV_64F, cameraMatrix2Data ); + + // zero and nonzero distortion coefficients + double nonZeroDistCoeffsData[] = { 0.01, 0.0001, 0, 0, 1e-04, 0.2, 0.02, 0.0002 }; // k1, k2, p1, p2, k3, k4, k5, k6 + vector distCoeffsList = {Mat::zeros(1, 5, CV_64F), Mat{1, 8, CV_64F, nonZeroDistCoeffsData}}; + const auto &zeroDistCoeffs = distCoeffsList[0]; int Inliers = 0; @@ -2160,14 +2170,26 @@ TEST(CV_RecoverPoseTest, regression_15341) // Estimation of fundamental matrix using the RANSAC algorithm Mat E, E2, R, t; + + // Check pose when camera matrices are different. + for (const auto &distCoeffs: distCoeffsList) + { + E = findEssentialMat(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, RANSAC, 0.999, 1.0, mask); + recoverPose(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, E2, R, t, RANSAC, 0.999, 1.0, mask); + EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << + "Two big difference between the same essential matrices computed using different functions with different cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function failed with different cameras, testcase " << testcase; + } + + // Check pose when camera matrices are the same. E = findEssentialMat(points1, points2, cameraMatrix, RANSAC, 0.999, 1.0, mask); E2 = findEssentialMat(points1, points2, cameraMatrix, zeroDistCoeffs, cameraMatrix, zeroDistCoeffs, RANSAC, 0.999, 1.0, mask); EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << - "Two big difference between the same essential matrices computed using different functions, testcase " << testcase; - EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function findEssentialMat failed, testcase " << testcase; + "Two big difference between the same essential matrices computed using different functions with same cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask[13]) << "Detecting outliers in function findEssentialMat failed with same cameras, testcase " << testcase; points2[12] = Point2f(0.0f, 0.0f); // provoke another outlier detection for recover Pose Inliers = recoverPose(E, points1, points2, cameraMatrix, R, t, mask); - EXPECT_EQ(0, (int)mask[12]) << "Detecting outliers in function failed, testcase " << testcase; + EXPECT_EQ(0, (int)mask[12]) << "Detecting outliers in function failed with same cameras, testcase " << testcase; } else // testcase with mat input data { @@ -2187,14 +2209,26 @@ TEST(CV_RecoverPoseTest, regression_15341) // Estimation of fundamental matrix using the RANSAC algorithm Mat E, E2, R, t; + + // Check pose when camera matrices are different. + for (const auto &distCoeffs: distCoeffsList) + { + E = findEssentialMat(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, RANSAC, 0.999, 1.0, mask); + recoverPose(points1, points2, cameraMatrix, distCoeffs, cameraMatrix2, distCoeffs, E2, R, t, RANSAC, 0.999, 1.0, mask); + EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << + "Two big difference between the same essential matrices computed using different functions with different cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function failed with different cameras, testcase " << testcase; + } + + // Check pose when camera matrices are the same. E = findEssentialMat(points1, points2, cameraMatrix, RANSAC, 0.999, 1.0, mask); E2 = findEssentialMat(points1, points2, cameraMatrix, zeroDistCoeffs, cameraMatrix, zeroDistCoeffs, RANSAC, 0.999, 1.0, mask); EXPECT_LT(cv::norm(E, E2, NORM_INF), 1e-4) << - "Two big difference between the same essential matrices computed using different functions, testcase " << testcase; - EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function findEssentialMat failed, testcase " << testcase; + "Two big difference between the same essential matrices computed using different functions with same cameras, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(13)) << "Detecting outliers in function findEssentialMat failed with same cameras, testcase " << testcase; points2.at(12) = Point2f(0.0f, 0.0f); // provoke an outlier detection Inliers = recoverPose(E, points1, points2, cameraMatrix, R, t, mask); - EXPECT_EQ(0, (int)mask.at(12)) << "Detecting outliers in function failed, testcase " << testcase; + EXPECT_EQ(0, (int)mask.at(12)) << "Detecting outliers in function failed with same cameras, testcase " << testcase; } EXPECT_EQ(Inliers, point_count - invalid_point_count) << "Number of inliers differs from expected number of inliers, testcase " << testcase; diff --git a/modules/core/CMakeLists.txt b/modules/core/CMakeLists.txt index b2797ab31f..13d0af4db8 100644 --- a/modules/core/CMakeLists.txt +++ b/modules/core/CMakeLists.txt @@ -153,6 +153,14 @@ if(OPENCV_CORE_EXCLUDE_C_API) ocv_target_compile_definitions(${the_module} PRIVATE "OPENCV_EXCLUDE_C_API=1") endif() +if(OPENCV_DISABLE_THREAD_SUPPORT) + ocv_target_compile_definitions(${the_module} PUBLIC "OPENCV_DISABLE_THREAD_SUPPORT=1") +endif() + +if(OPENCV_SEMIHOSTING) + ocv_target_compile_definitions(${the_module} PRIVATE "-DOPENCV_SEMIHOSTING") +endif(OPENCV_SEMIHOSTING) + if(HAVE_HPX) ocv_target_link_libraries(${the_module} LINK_PRIVATE "${HPX_LIBRARIES}") endif() diff --git a/modules/core/include/opencv2/core/bindings_utils.hpp b/modules/core/include/opencv2/core/bindings_utils.hpp index cf8bcdd622..f9c73dd4ff 100644 --- a/modules/core/include/opencv2/core/bindings_utils.hpp +++ b/modules/core/include/opencv2/core/bindings_utils.hpp @@ -116,6 +116,65 @@ String dumpRange(const Range& argument) } } +CV_WRAP static inline +int testOverwriteNativeMethod(int argument) +{ + return argument; +} + +CV_WRAP static inline +String testReservedKeywordConversion(int positional_argument, int lambda = 2, int from = 3) +{ + return format("arg=%d, lambda=%d, from=%d", positional_argument, lambda, from); +} + +CV_EXPORTS_W String dumpVectorOfInt(const std::vector& vec); + +CV_EXPORTS_W String dumpVectorOfDouble(const std::vector& vec); + +CV_EXPORTS_W String dumpVectorOfRect(const std::vector& vec); + +CV_WRAP static inline +void generateVectorOfRect(size_t len, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(12345); + Mat tmp(static_cast(len), 1, CV_32SC4); + rng.fill(tmp, RNG::UNIFORM, 10, 20); + tmp.copyTo(vec); + } +} + +CV_WRAP static inline +void generateVectorOfInt(size_t len, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(554433); + Mat tmp(static_cast(len), 1, CV_32SC1); + rng.fill(tmp, RNG::UNIFORM, -10, 10); + tmp.copyTo(vec); + } +} + +CV_WRAP static inline +void generateVectorOfMat(size_t len, int rows, int cols, int dtype, CV_OUT std::vector& vec) +{ + vec.resize(len); + if (len > 0) + { + RNG rng(65431); + for (size_t i = 0; i < len; ++i) + { + vec[i].create(rows, cols, dtype); + rng.fill(vec[i], RNG::UNIFORM, 0, 10); + } + } +} + CV_WRAP static inline void testRaiseGeneralException() { diff --git a/modules/core/include/opencv2/core/cv_cpu_dispatch.h b/modules/core/include/opencv2/core/cv_cpu_dispatch.h index 8365b10ba9..ab5a67d4c8 100644 --- a/modules/core/include/opencv2/core/cv_cpu_dispatch.h +++ b/modules/core/include/opencv2/core/cv_cpu_dispatch.h @@ -55,7 +55,7 @@ # ifdef _MSC_VER # include # if defined(_M_X64) -# define CV_POPCNT_U64 _mm_popcnt_u64 +# define CV_POPCNT_U64 (int)_mm_popcnt_u64 # endif # define CV_POPCNT_U32 _mm_popcnt_u32 # else diff --git a/modules/core/include/opencv2/core/cvdef.h b/modules/core/include/opencv2/core/cvdef.h index 86e4a30ef1..8d51009f4f 100644 --- a/modules/core/include/opencv2/core/cvdef.h +++ b/modules/core/include/opencv2/core/cvdef.h @@ -707,14 +707,47 @@ __CV_ENUM_FLAGS_BITWISE_XOR_EQ (EnumType, EnumType) # endif #endif +/****************************************************************************************\ +* CV_NODISCARD_STD attribute (C++17) * +* encourages the compiler to issue a warning if the return value is discarded * +\****************************************************************************************/ +#ifndef CV_NODISCARD_STD +# ifndef __has_cpp_attribute +// workaround preprocessor non-compliance https://reviews.llvm.org/D57851 +# define __has_cpp_attribute(__x) 0 +# endif +# if __has_cpp_attribute(nodiscard) +# define CV_NODISCARD_STD [[nodiscard]] +# elif __cplusplus >= 201703L +// available when compiler is C++17 compliant +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(_MSC_VER) && _MSC_VER >= 1911 && _MSVC_LANG >= 201703L +// available with VS2017 v15.3+ with /std:c++17 or higher; works on functions and classes +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(__GNUC__) && (((__GNUC__ * 100) + __GNUC_MINOR__) >= 700) && (__cplusplus >= 201103L) +// available with GCC 7.0+; works on functions, works or silently fails on classes +# define CV_NODISCARD_STD [[nodiscard]] +# elif defined(__GNUC__) && (((__GNUC__ * 100) + __GNUC_MINOR__) >= 408) && (__cplusplus >= 201103L) +// available with GCC 4.8+ but it usually does nothing and can fail noisily -- therefore not used +// define CV_NODISCARD_STD [[gnu::warn_unused_result]] +# endif +#endif +#ifndef CV_NODISCARD_STD +# define CV_NODISCARD_STD /* nothing by default */ +#endif + /****************************************************************************************\ -* CV_NODISCARD attribute * -* encourages the compiler to issue a warning if the return value is discarded (C++17) * +* CV_NODISCARD attribute (deprecated, GCC only) * +* DONT USE: use instead the standard CV_NODISCARD_STD macro above * +* this legacy method silently fails to issue warning until some version * +* after gcc 6.3.0. Yet with gcc 7+ you can use the above standard method * +* which makes this method useless. Don't use it. * +* @deprecated use instead CV_NODISCARD_STD * \****************************************************************************************/ #ifndef CV_NODISCARD # if defined(__GNUC__) -# define CV_NODISCARD __attribute__((__warn_unused_result__)) // at least available with GCC 3.4 +# define CV_NODISCARD __attribute__((__warn_unused_result__)) # elif defined(__clang__) && defined(__has_attribute) # if __has_attribute(__warn_unused_result__) # define CV_NODISCARD __attribute__((__warn_unused_result__)) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp index cb2140df58..55e0d88f59 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv.hpp @@ -151,12 +151,14 @@ struct vint8mf4_t }; #define OPENCV_HAL_IMPL_RVV_NATIVE_LOADSTORE_MF2(_Tpvec, _Tp, suffix, width, n) \ -inline _Tpvec vle##width##_v_##suffix##mf2(const _Tp* ptr) \ +inline _Tpvec vle##width##_v_##suffix##mf2(const _Tp* ptr, size_t vl) \ { \ + CV_UNUSED(vl); \ return _Tpvec(ptr); \ } \ -inline void vse##width##_v_##suffix##mf2(_Tp* ptr, _Tpvec v) \ +inline void vse##width##_v_##suffix##mf2(_Tp* ptr, _Tpvec v, size_t vl) \ { \ + CV_UNUSED(vl); \ for (int i = 0; i < n; ++i) \ { \ ptr[i] = v.val[i]; \ @@ -176,15 +178,14 @@ OPENCV_HAL_IMPL_RVV_NATIVE_LOADSTORE_MF2(vfloat64mf2_t, float64_t, f64, 64, 1) #define OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(_Tpwvec, _Tpvec, _wTp, wcvt, suffix, width, n) \ -inline _Tpwvec wcvt (_Tpvec v) \ +inline _Tpwvec wcvt (_Tpvec v, size_t vl) \ { \ _wTp tmp[n]; \ for (int i = 0; i < n; ++i) \ { \ tmp[i] = (_wTp)v.val[i]; \ } \ - vsetvlmax_e##width##m1(); \ - return vle##width##_v_##suffix##m1(tmp); \ + return vle##width##_v_##suffix##m1(tmp, vl); \ } OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vuint16m1_t, vuint8mf2_t, ushort, vwcvtu_x_x_v_u16m1, u16, 16, 8) @@ -194,32 +195,34 @@ OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vint32m1_t, vint16mf2_t, int, vwcvt_x_x_v_i32m1, OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vuint64m1_t, vuint32mf2_t, uint64, vwcvtu_x_x_v_u64m1, u64, 64, 2) OPENCV_HAL_IMPL_RVV_NATIVE_WCVT(vint64m1_t, vint32mf2_t, int64, vwcvt_x_x_v_i64m1, i64, 64, 2) -inline vuint8mf4_t vle8_v_u8mf4 (const uint8_t *base) +inline vuint8mf4_t vle8_v_u8mf4 (const uint8_t *base, size_t vl) { + CV_UNUSED(vl); return vuint8mf4_t(base); } -inline vint8mf4_t vle8_v_i8mf4 (const int8_t *base) +inline vint8mf4_t vle8_v_i8mf4 (const int8_t *base, size_t vl) { + CV_UNUSED(vl); return vint8mf4_t(base); } -inline vuint16mf2_t vwcvtu_x_x_v_u16mf2 (vuint8mf4_t src) +inline vuint16mf2_t vwcvtu_x_x_v_u16mf2 (vuint8mf4_t src, size_t vl) { ushort tmp[4]; for (int i = 0; i < 4; ++i) { tmp[i] = (ushort)src.val[i]; } - return vle16_v_u16mf2(tmp); + return vle16_v_u16mf2(tmp, vl); } -inline vint16mf2_t vwcvt_x_x_v_i16mf2 (vint8mf4_t src) +inline vint16mf2_t vwcvt_x_x_v_i16mf2 (vint8mf4_t src, size_t vl) { short tmp[4]; for (int i = 0; i < 4; ++i) { tmp[i] = (short)src.val[i]; } - return vle16_v_i16mf2(tmp); + return vle16_v_i16mf2(tmp, vl); } //////////// Types //////////// @@ -232,8 +235,7 @@ struct v_uint8x16 v_uint8x16() {} explicit v_uint8x16(vuint8m1_t v) { - vsetvlmax_e8m1(); - vse8_v_u8m1(val, v); + vse8_v_u8m1(val, v, nlanes); } v_uint8x16(uchar v0, uchar v1, uchar v2, uchar v3, uchar v4, uchar v5, uchar v6, uchar v7, uchar v8, uchar v9, uchar v10, uchar v11, uchar v12, uchar v13, uchar v14, uchar v15) @@ -246,8 +248,7 @@ struct v_uint8x16 } operator vuint8m1_t() const { - vsetvlmax_e8m1(); - return vle8_v_u8m1(val); + return vle8_v_u8m1(val, nlanes); } uchar get0() const { @@ -265,8 +266,7 @@ struct v_int8x16 v_int8x16() {} explicit v_int8x16(vint8m1_t v) { - vsetvlmax_e8m1(); - vse8_v_i8m1(val, v); + vse8_v_i8m1(val, v, nlanes); } v_int8x16(schar v0, schar v1, schar v2, schar v3, schar v4, schar v5, schar v6, schar v7, schar v8, schar v9, schar v10, schar v11, schar v12, schar v13, schar v14, schar v15) @@ -279,8 +279,7 @@ struct v_int8x16 } operator vint8m1_t() const { - vsetvlmax_e8m1(); - return vle8_v_i8m1(val); + return vle8_v_i8m1(val, nlanes); } schar get0() const { @@ -298,8 +297,7 @@ struct v_uint16x8 v_uint16x8() {} explicit v_uint16x8(vuint16m1_t v) { - vsetvlmax_e16m1(); - vse16_v_u16m1(val, v); + vse16_v_u16m1(val, v, nlanes); } v_uint16x8(ushort v0, ushort v1, ushort v2, ushort v3, ushort v4, ushort v5, ushort v6, ushort v7) { @@ -311,8 +309,7 @@ struct v_uint16x8 } operator vuint16m1_t() const { - vsetvlmax_e16m1(); - return vle16_v_u16m1(val); + return vle16_v_u16m1(val, nlanes); } ushort get0() const { @@ -330,8 +327,7 @@ struct v_int16x8 v_int16x8() {} explicit v_int16x8(vint16m1_t v) { - vsetvlmax_e16m1(); - vse16_v_i16m1(val, v); + vse16_v_i16m1(val, v, nlanes); } v_int16x8(short v0, short v1, short v2, short v3, short v4, short v5, short v6, short v7) { @@ -343,8 +339,7 @@ struct v_int16x8 } operator vint16m1_t() const { - vsetvlmax_e16m1(); - return vle16_v_i16m1(val); + return vle16_v_i16m1(val, nlanes); } short get0() const { @@ -362,8 +357,7 @@ struct v_uint32x4 v_uint32x4() {} explicit v_uint32x4(vuint32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_u32m1(val, v); + vse32_v_u32m1(val, v, nlanes); } v_uint32x4(unsigned v0, unsigned v1, unsigned v2, unsigned v3) { @@ -375,8 +369,7 @@ struct v_uint32x4 } operator vuint32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_u32m1(val); + return vle32_v_u32m1(val, nlanes); } unsigned get0() const { @@ -394,8 +387,7 @@ struct v_int32x4 v_int32x4() {} explicit v_int32x4(vint32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_i32m1(val, v); + vse32_v_i32m1(val, v, nlanes); } v_int32x4(int v0, int v1, int v2, int v3) { @@ -407,8 +399,7 @@ struct v_int32x4 } operator vint32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_i32m1(val); + return vle32_v_i32m1(val, nlanes); } int get0() const { @@ -425,8 +416,7 @@ struct v_float32x4 v_float32x4() {} explicit v_float32x4(vfloat32m1_t v) { - vsetvlmax_e32m1(); - vse32_v_f32m1(val, v); + vse32_v_f32m1(val, v, nlanes); } v_float32x4(float v0, float v1, float v2, float v3) { @@ -438,8 +428,7 @@ struct v_float32x4 } operator vfloat32m1_t() const { - vsetvlmax_e32m1(); - return vle32_v_f32m1(val); + return vle32_v_f32m1(val, nlanes); } float get0() const { @@ -456,8 +445,7 @@ struct v_uint64x2 v_uint64x2() {} explicit v_uint64x2(vuint64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_u64m1(val, v); + vse64_v_u64m1(val, v, nlanes); } v_uint64x2(uint64 v0, uint64 v1) { @@ -469,8 +457,7 @@ struct v_uint64x2 } operator vuint64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_u64m1(val); + return vle64_v_u64m1(val, nlanes); } uint64 get0() const { @@ -488,8 +475,7 @@ struct v_int64x2 v_int64x2() {} explicit v_int64x2(vint64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_i64m1(val, v); + vse64_v_i64m1(val, v, nlanes); } v_int64x2(int64 v0, int64 v1) { @@ -501,8 +487,7 @@ struct v_int64x2 } operator vint64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_i64m1(val); + return vle64_v_i64m1(val, nlanes); } int64 get0() const { @@ -521,8 +506,7 @@ struct v_float64x2 v_float64x2() {} explicit v_float64x2(vfloat64m1_t v) { - vsetvlmax_e64m1(); - vse64_v_f64m1(val, v); + vse64_v_f64m1(val, v, nlanes); } v_float64x2(double v0, double v1) { @@ -534,8 +518,7 @@ struct v_float64x2 } operator vfloat64m1_t() const { - vsetvlmax_e64m1(); - return vle64_v_f64m1(val); + return vle64_v_f64m1(val, nlanes); } double get0() const { @@ -549,42 +532,38 @@ struct v_float64x2 //////////// Initial //////////// -#define OPENCV_HAL_IMPL_RVV_INIT_INTEGER(_Tpvec, _Tp, width, suffix1, suffix2) \ +#define OPENCV_HAL_IMPL_RVV_INIT_INTEGER(_Tpvec, _Tp, suffix1, suffix2, vl) \ inline v_##_Tpvec v_setzero_##suffix1() \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpvec(vzero_##suffix2##m1()); \ + return v_##_Tpvec(vmv_v_x_##suffix2##m1(0, vl)); \ } \ inline v_##_Tpvec v_setall_##suffix1(_Tp v) \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpvec(vmv_v_x_##suffix2##m1(v)); \ + return v_##_Tpvec(vmv_v_x_##suffix2##m1(v, vl)); \ } -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint8x16, uchar, 8, u8, u8) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int8x16, schar, 8, s8, i8) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint16x8, ushort, 16, u16, u16) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int16x8, short, 16, s16, i16) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint32x4, unsigned, 32, u32, u32) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int32x4, int, 32, s32, i32) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint64x2, uint64, 64, u64, u64) -OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int64x2, int64, 64, s64, i64) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint8x16, uchar, u8, u8, 16) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int8x16, schar, s8, i8, 16) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint16x8, ushort, u16, u16, 8) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int16x8, short, s16, i16, 8) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint32x4, unsigned, u32, u32, 4) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int32x4, int, s32, i32, 4) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(uint64x2, uint64, u64, u64, 2) +OPENCV_HAL_IMPL_RVV_INIT_INTEGER(int64x2, int64, s64, i64, 2) -#define OPENCV_HAL_IMPL_RVV_INIT_FP(_Tpv, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_INIT_FP(_Tpv, _Tp, suffix, vl) \ inline v_##_Tpv v_setzero_##suffix() \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpv(vzero_##suffix##m1()); \ + return v_##_Tpv(vfmv_v_f_##suffix##m1(0, vl)); \ } \ inline v_##_Tpv v_setall_##suffix(_Tp v) \ { \ - vsetvlmax_e##width##m1(); \ - return v_##_Tpv(vfmv_v_f_##suffix##m1(v)); \ + return v_##_Tpv(vfmv_v_f_##suffix##m1(v, vl)); \ } -OPENCV_HAL_IMPL_RVV_INIT_FP(float32x4, float, 32, f32) +OPENCV_HAL_IMPL_RVV_INIT_FP(float32x4, float, f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_INIT_FP(float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_INIT_FP(float64x2, double, f64, 2) #endif //////////// Reinterpret //////////// @@ -605,232 +584,214 @@ OPENCV_HAL_IMPL_RVV_SELF_REINTERPRET(int64x2, s64) OPENCV_HAL_IMPL_RVV_SELF_REINTERPRET(float64x2, f64) #endif -#define OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(_Tpvec1, _Tpvec2, _nTpvec1, _nTpvec2, suffix1, suffix2, nsuffix1, nsuffix2, width1, width2) \ +#define OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(_Tpvec1, _Tpvec2, _nTpvec1, _nTpvec2, suffix1, suffix2, nsuffix1, nsuffix2, width1, width2, vl1, vl2) \ inline v_##_Tpvec1 v_reinterpret_as_##suffix1(const v_##_Tpvec2& v) \ { \ - vsetvlmax_e##width2##m1(); \ - return v_##_Tpvec1((_nTpvec1)vle##width2##_v_##nsuffix2##m1(v.val)); \ + return v_##_Tpvec1((_nTpvec1)vle##width2##_v_##nsuffix2##m1(v.val, vl2)); \ } \ inline v_##_Tpvec2 v_reinterpret_as_##suffix2(const v_##_Tpvec1& v) \ { \ - vsetvlmax_e##width1##m1(); \ - return v_##_Tpvec2((_nTpvec2)vle##width1##_v_##nsuffix1##m1(v.val)); \ + return v_##_Tpvec2((_nTpvec2)vle##width1##_v_##nsuffix1##m1(v.val, vl1)); \ } -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int8x16, vuint8m1_t, vint8m1_t, u8, s8, u8, i8, 8, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int16x8, vuint16m1_t, vint16m1_t, u16, s16, u16, i16, 16, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int32x4, vuint32m1_t, vint32m1_t, u32, s32, u32, i32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float32x4, vuint32m1_t, vfloat32m1_t, u32, f32, u32, f32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float32x4, vint32m1_t, vfloat32m1_t, s32, f32, i32, f32, 32, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int64x2, vuint64m1_t, vint64m1_t, u64, s64, u64, i64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint16x8, vuint8m1_t, vuint16m1_t, u8, u16, u8, u16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint32x4, vuint8m1_t, vuint32m1_t, u8, u32, u8, u32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint64x2, vuint8m1_t, vuint64m1_t, u8, u64, u8, u64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint32x4, vuint16m1_t, vuint32m1_t, u16, u32, u16, u32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint64x2, vuint16m1_t, vuint64m1_t, u16, u64, u16, u64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, uint64x2, vuint32m1_t, vuint64m1_t, u32, u64, u32, u64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int16x8, vint8m1_t, vint16m1_t, s8, s16, i8, i16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int32x4, vint8m1_t, vint32m1_t, s8, s32, i8, i32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int64x2, vint8m1_t, vint64m1_t, s8, s64, i8, i64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int32x4, vint16m1_t, vint32m1_t, s16, s32, i16, i32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int64x2, vint16m1_t, vint64m1_t, s16, s64, i16, i64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, int64x2, vint32m1_t, vint64m1_t, s32, s64, i32, i64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int16x8, vuint8m1_t, vint16m1_t, u8, s16, u8, i16, 8, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int32x4, vuint8m1_t, vint32m1_t, u8, s32, u8, i32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int64x2, vuint8m1_t, vint64m1_t, u8, s64, u8, i64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int8x16, vuint16m1_t, vint8m1_t, u16, s8, u16, i8, 16, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int32x4, vuint16m1_t, vint32m1_t, u16, s32, u16, i32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int64x2, vuint16m1_t, vint64m1_t, u16, s64, u16, i64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int8x16, vuint32m1_t, vint8m1_t, u32, s8, u32, i8, 32, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int16x8, vuint32m1_t, vint16m1_t, u32, s16, u32, i16, 32, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int64x2, vuint32m1_t, vint64m1_t, u32, s64, u32, i64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int8x16, vuint64m1_t, vint8m1_t, u64, s8, u64, i8, 64, 8) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int16x8, vuint64m1_t, vint16m1_t, u64, s16, u64, i16, 64, 16) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int32x4, vuint64m1_t, vint32m1_t, u64, s32, u64, i32, 64, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float32x4, vuint8m1_t, vfloat32m1_t, u8, f32, u8, f32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float32x4, vuint16m1_t, vfloat32m1_t, u16, f32, u16, f32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float32x4, vuint64m1_t, vfloat32m1_t, u64, f32, u64, f32, 64, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float32x4, vint8m1_t, vfloat32m1_t, s8, f32, i8, f32, 8, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float32x4, vint16m1_t, vfloat32m1_t, s16, f32, i16, f32, 16, 32) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float32x4, vint64m1_t, vfloat32m1_t, s64, f32, i64, f32, 64, 32) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int8x16, vuint8m1_t, vint8m1_t, u8, s8, u8, i8, 8, 8, 16, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int16x8, vuint16m1_t, vint16m1_t, u16, s16, u16, i16, 16, 16, 8, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int32x4, vuint32m1_t, vint32m1_t, u32, s32, u32, i32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float32x4, vuint32m1_t, vfloat32m1_t, u32, f32, u32, f32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float32x4, vint32m1_t, vfloat32m1_t, s32, f32, i32, f32, 32, 32, 4, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int64x2, vuint64m1_t, vint64m1_t, u64, s64, u64, i64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint16x8, vuint8m1_t, vuint16m1_t, u8, u16, u8, u16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint32x4, vuint8m1_t, vuint32m1_t, u8, u32, u8, u32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, uint64x2, vuint8m1_t, vuint64m1_t, u8, u64, u8, u64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint32x4, vuint16m1_t, vuint32m1_t, u16, u32, u16, u32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, uint64x2, vuint16m1_t, vuint64m1_t, u16, u64, u16, u64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, uint64x2, vuint32m1_t, vuint64m1_t, u32, u64, u32, u64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int16x8, vint8m1_t, vint16m1_t, s8, s16, i8, i16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int32x4, vint8m1_t, vint32m1_t, s8, s32, i8, i32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, int64x2, vint8m1_t, vint64m1_t, s8, s64, i8, i64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int32x4, vint16m1_t, vint32m1_t, s16, s32, i16, i32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, int64x2, vint16m1_t, vint64m1_t, s16, s64, i16, i64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, int64x2, vint32m1_t, vint64m1_t, s32, s64, i32, i64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int16x8, vuint8m1_t, vint16m1_t, u8, s16, u8, i16, 8, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int32x4, vuint8m1_t, vint32m1_t, u8, s32, u8, i32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, int64x2, vuint8m1_t, vint64m1_t, u8, s64, u8, i64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int8x16, vuint16m1_t, vint8m1_t, u16, s8, u16, i8, 16, 8, 8, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int32x4, vuint16m1_t, vint32m1_t, u16, s32, u16, i32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, int64x2, vuint16m1_t, vint64m1_t, u16, s64, u16, i64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int8x16, vuint32m1_t, vint8m1_t, u32, s8, u32, i8, 32, 8, 4, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int16x8, vuint32m1_t, vint16m1_t, u32, s16, u32, i16, 32, 16, 4, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, int64x2, vuint32m1_t, vint64m1_t, u32, s64, u32, i64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int8x16, vuint64m1_t, vint8m1_t, u64, s8, u64, i8, 64, 8, 2, 16) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int16x8, vuint64m1_t, vint16m1_t, u64, s16, u64, i16, 64, 16, 2, 8) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, int32x4, vuint64m1_t, vint32m1_t, u64, s32, u64, i32, 64, 32, 2, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float32x4, vuint8m1_t, vfloat32m1_t, u8, f32, u8, f32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float32x4, vuint16m1_t, vfloat32m1_t, u16, f32, u16, f32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float32x4, vuint64m1_t, vfloat32m1_t, u64, f32, u64, f32, 64, 32, 2, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float32x4, vint8m1_t, vfloat32m1_t, s8, f32, i8, f32, 8, 32, 16, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float32x4, vint16m1_t, vfloat32m1_t, s16, f32, i16, f32, 16, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float32x4, vint64m1_t, vfloat32m1_t, s64, f32, i64, f32, 64, 32, 2, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float64x2, vuint64m1_t, vfloat64m1_t, u64, f64, u64, f64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float64x2, vint64m1_t, vfloat64m1_t, s64, f64, i64, f64, 64, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float64x2, vuint8m1_t, vfloat64m1_t, u8, f64, u8, f64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float64x2, vuint16m1_t, vfloat64m1_t, u16, f64, u16, f64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float64x2, vuint32m1_t, vfloat64m1_t, u32, f64, u32, f64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float64x2, vint8m1_t, vfloat64m1_t, s8, f64, i8, f64, 8, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float64x2, vint16m1_t, vfloat64m1_t, s16, f64, i16, f64, 16, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float64x2, vint32m1_t, vfloat64m1_t, s32, f64, i32, f64, 32, 64) -OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(float32x4, float64x2, vfloat32m1_t, vfloat64m1_t, f32, f64, f32, f64, 32, 64) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint64x2, float64x2, vuint64m1_t, vfloat64m1_t, u64, f64, u64, f64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int64x2, float64x2, vint64m1_t, vfloat64m1_t, s64, f64, i64, f64, 64, 64, 2, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint8x16, float64x2, vuint8m1_t, vfloat64m1_t, u8, f64, u8, f64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint16x8, float64x2, vuint16m1_t, vfloat64m1_t, u16, f64, u16, f64, 16, 64, 6, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(uint32x4, float64x2, vuint32m1_t, vfloat64m1_t, u32, f64, u32, f64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int8x16, float64x2, vint8m1_t, vfloat64m1_t, s8, f64, i8, f64, 8, 64, 16, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int16x8, float64x2, vint16m1_t, vfloat64m1_t, s16, f64, i16, f64, 16, 64, 8, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(int32x4, float64x2, vint32m1_t, vfloat64m1_t, s32, f64, i32, f64, 32, 64, 4, 2) +OPENCV_HAL_IMPL_RVV_ONE_TIME_REINTERPRET(float32x4, float64x2, vfloat32m1_t, vfloat64m1_t, f32, f64, f32, f64, 32, 64, 4, 2) #endif ////////////// Extract ////////////// -#define OPENCV_HAL_IMPL_RVV_EXTRACT(_Tpvec, _Tp, suffix, width, vmv) \ +#define OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(_Tpvec, _Tp, suffix, vmv, vl) \ template \ inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, s), b, _Tpvec::nlanes - s)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, s, vl), b, _Tpvec::nlanes - s, vl)); \ } \ template inline _Tp v_extract_n(_Tpvec v) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tp(vmv(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), v, i))); \ + return _Tp(vmv(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), v, i, vl))); \ } -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint8x16, uchar, u8, 8, vmv_x_s_u8m1_u8) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int8x16, schar, i8, 8, vmv_x_s_i8m1_i8) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint16x8, ushort, u16, 16, vmv_x_s_u16m1_u16) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int16x8, short, i16, 16, vmv_x_s_i16m1_i16) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint32x4, uint, u32, 32, vmv_x_s_u32m1_u32) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int32x4, int, i32, 32, vmv_x_s_i32m1_i32) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_uint64x2, uint64, u64, 64, vmv_x_s_u64m1_u64) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_int64x2, int64, i64, 64, vmv_x_s_i64m1_i64) -OPENCV_HAL_IMPL_RVV_EXTRACT(v_float32x4, float, f32, 32, vfmv_f_s_f32m1_f32) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint8x16, uchar, u8, vmv_x_s_u8m1_u8, 16) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int8x16, schar, i8, vmv_x_s_i8m1_i8, 16) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint16x8, ushort, u16, vmv_x_s_u16m1_u16, 8) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int16x8, short, i16, vmv_x_s_i16m1_i16, 8) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint32x4, uint, u32, vmv_x_s_u32m1_u32, 4) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int32x4, int, i32, vmv_x_s_i32m1_i32, 4) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_uint64x2, uint64, u64, vmv_x_s_u64m1_u64, 2) +OPENCV_HAL_IMPL_RVV_EXTRACT_INTEGER(v_int64x2, int64, i64, vmv_x_s_i64m1_i64, 2) + +#define OPENCV_HAL_IMPL_RVV_EXTRACT_FP(_Tpvec, _Tp, suffix, vmv, vl) \ +template \ +inline _Tpvec v_extract(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, s, vl), b, _Tpvec::nlanes - s, vl)); \ +} \ +template inline _Tp v_extract_n(_Tpvec v) \ +{ \ + return _Tp(vmv(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), v, i, vl))); \ +} + +OPENCV_HAL_IMPL_RVV_EXTRACT_FP(v_float32x4, float, f32, vfmv_f_s_f32m1_f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_EXTRACT(v_float64x2, double, f64, 64, vfmv_f_s_f64m1_f64) +OPENCV_HAL_IMPL_RVV_EXTRACT_FP(v_float64x2, double, f64, vfmv_f_s_f64m1_f64, 2) #endif ////////////// Load/Store ////////////// -#define OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(_Tpvec, _nTpvec, _Tp, hvl, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(_Tpvec, _nTpvec, _Tp, hvl, vl, width, suffix, vmv) \ inline _Tpvec v_load(const _Tp* ptr) \ { \ - vsetvlmax_e8m1(); \ - return _Tpvec((_nTpvec)vle8_v_u8m1((uchar*)ptr)); \ + return _Tpvec((_nTpvec)vle8_v_u8m1((uchar*)ptr, 16)); \ } \ inline _Tpvec v_load_aligned(const _Tp* ptr) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vle##width##_v_##suffix##m1(ptr)); \ + return _Tpvec(vle##width##_v_##suffix##m1(ptr, vl)); \ } \ inline _Tpvec v_load_low(const _Tp* ptr) \ { \ - vsetvl_e##width##m1(hvl); \ - _Tpvec res = _Tpvec(vle##width##_v_##suffix##m1(ptr)); \ - vsetvlmax_e##width##m1(); \ + _Tpvec res = _Tpvec(vle##width##_v_##suffix##m1(ptr, hvl)); \ return res; \ } \ inline void v_store(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e8m1(); \ - vse8_v_u8m1((uchar*)ptr, vle8_v_u8m1((uchar*)a.val)); \ + vse8_v_u8m1((uchar*)ptr, vle8_v_u8m1((uchar*)a.val, 16), 16); \ } \ inline void v_store_aligned(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store_aligned_nocache(_Tp* ptr, const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store(_Tp* ptr, const _Tpvec& a, hal::StoreMode /*mode*/) \ { \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(ptr, a); \ + vse##width##_v_##suffix##m1(ptr, a, vl); \ } \ inline void v_store_low(_Tp* ptr, const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) tmp_ptr[_Tpvec::nlanes] = {0}; \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(tmp_ptr, a); \ - for(int i = 0; i < _Tpvec::nlanes/2; ++i) \ - { \ - ptr[i] = tmp_ptr[i]; \ - } \ + vse##width##_v_##suffix##m1(ptr, a, hvl); \ } \ inline void v_store_high(_Tp* ptr, const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) tmp_ptr[_Tpvec::nlanes] = {0}; \ - vsetvlmax_e##width##m1(); \ - vse##width##_v_##suffix##m1(tmp_ptr, a); \ - for(int i = 0; i < _Tpvec::nlanes/2; ++i) \ - { \ - ptr[i] = tmp_ptr[i+_Tpvec::nlanes/2]; \ - } \ + vse##width##_v_##suffix##m1(ptr, vslidedown_vx_##suffix##m1(vmv(0, vl), a, hvl, vl), hvl); \ } -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint8x16, vuint8m1_t, uchar, 8, 8, u8) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int8x16, vint8m1_t, schar, 8, 8, i8) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint16x8, vuint16m1_t, ushort, 4, 16, u16) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int16x8, vint16m1_t, short, 4, 16, i16) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint32x4, vuint32m1_t, unsigned, 2, 32, u32) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int32x4, vint32m1_t, int, 2, 32, i32) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint64x2, vuint64m1_t, uint64, 1, 64, u64) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int64x2, vint64m1_t, int64, 1, 64, i64) -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float32x4, vfloat32m1_t, float, 2, 32, f32) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint8x16, vuint8m1_t, uchar, 8, 16, 8, u8, vmv_v_x_u8m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int8x16, vint8m1_t, schar, 8, 16, 8, i8, vmv_v_x_i8m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint16x8, vuint16m1_t, ushort, 4, 8, 16, u16, vmv_v_x_u16m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int16x8, vint16m1_t, short, 4, 8, 16, i16, vmv_v_x_i16m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint32x4, vuint32m1_t, unsigned, 2, 4, 32, u32, vmv_v_x_u32m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int32x4, vint32m1_t, int, 2, 4, 32, i32, vmv_v_x_i32m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_uint64x2, vuint64m1_t, uint64, 1, 2, 64, u64, vmv_v_x_u64m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_int64x2, vint64m1_t, int64, 1, 2, 64, i64, vmv_v_x_i64m1) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float32x4, vfloat32m1_t, float, 2, 4, 32, f32, vfmv_v_f_f32m1) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64x2, vfloat64m1_t, double, 1, 64, f64) +OPENCV_HAL_IMPL_RVV_LOADSTORE_OP(v_float64x2, vfloat64m1_t, double, 1, 2, 64, f64, vfmv_v_f_f64m1) #endif inline v_int8x16 v_load_halves(const schar* ptr0, const schar* ptr1) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr0[4], ptr0[5], ptr0[6], ptr0[7], ptr1[0], ptr1[1], ptr1[2], ptr1[3], ptr1[4], ptr1[5], ptr1[6], ptr1[7] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_uint8x16 v_load_halves(const uchar* ptr0, const uchar* ptr1) { return v_reinterpret_as_u8(v_load_halves((schar*)ptr0, (schar*)ptr1)); } inline v_int16x8 v_load_halves(const short* ptr0, const short* ptr1) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { ptr0[0], ptr0[1], ptr0[2], ptr0[3], ptr1[0], ptr1[1], ptr1[2], ptr1[3] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_uint16x8 v_load_halves(const ushort* ptr0, const ushort* ptr1) { return v_reinterpret_as_u16(v_load_halves((short*)ptr0, (short*)ptr1)); } inline v_int32x4 v_load_halves(const int* ptr0, const int* ptr1) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_float32x4 v_load_halves(const float* ptr0, const float* ptr1) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { ptr0[0], ptr0[1], ptr1[0], ptr1[1] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_uint32x4 v_load_halves(const unsigned* ptr0, const unsigned* ptr1) { return v_reinterpret_as_u32(v_load_halves((int*)ptr0, (int*)ptr1)); } inline v_int64x2 v_load_halves(const int64* ptr0, const int64* ptr1) { - int64 CV_DECL_ALIGNED(32) elems[2] = + int64 elems[2] = { ptr0[0], ptr1[0] }; - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(elems)); + return v_int64x2(vle64_v_i64m1(elems, 2)); } inline v_uint64x2 v_load_halves(const uint64* ptr0, const uint64* ptr1) { return v_reinterpret_as_u64(v_load_halves((int64*)ptr0, (int64*)ptr1)); } #if CV_SIMD128_64F inline v_float64x2 v_load_halves(const double* ptr0, const double* ptr1) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { ptr0[0], ptr1[0] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } #endif @@ -839,7 +800,7 @@ inline v_float64x2 v_load_halves(const double* ptr0, const double* ptr1) inline v_int8x16 v_lut(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[ 0]], tab[idx[ 1]], @@ -858,12 +819,11 @@ inline v_int8x16 v_lut(const schar* tab, const int* idx) tab[idx[14]], tab[idx[15]] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[0]], tab[idx[0] + 1], @@ -882,12 +842,11 @@ inline v_int8x16 v_lut_pairs(const schar* tab, const int* idx) tab[idx[7]], tab[idx[7] + 1] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) { - schar CV_DECL_ALIGNED(32) elems[16] = + schar elems[16] = { tab[idx[0]], tab[idx[0] + 1], @@ -906,8 +865,7 @@ inline v_int8x16 v_lut_quads(const schar* tab, const int* idx) tab[idx[3] + 2], tab[idx[3] + 3] }; - vsetvlmax_e8m1(); - return v_int8x16(vle8_v_i8m1(elems)); + return v_int8x16(vle8_v_i8m1(elems, 16)); } inline v_uint8x16 v_lut(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut((schar*)tab, idx)); } inline v_uint8x16 v_lut_pairs(const uchar* tab, const int* idx) { return v_reinterpret_as_u8(v_lut_pairs((schar*)tab, idx)); } @@ -915,7 +873,7 @@ inline v_uint8x16 v_lut_quads(const uchar* tab, const int* idx) { return v_reint inline v_int16x8 v_lut(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[1]], @@ -926,12 +884,11 @@ inline v_int16x8 v_lut(const short* tab, const int* idx) tab[idx[6]], tab[idx[7]] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[0] + 1], @@ -942,12 +899,11 @@ inline v_int16x8 v_lut_pairs(const short* tab, const int* idx) tab[idx[3]], tab[idx[3] + 1] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_int16x8 v_lut_quads(const short* tab, const int* idx) { - short CV_DECL_ALIGNED(32) elems[8] = + short elems[8] = { tab[idx[0]], tab[idx[0] + 1], @@ -958,8 +914,7 @@ inline v_int16x8 v_lut_quads(const short* tab, const int* idx) tab[idx[1] + 2], tab[idx[1] + 3] }; - vsetvlmax_e16m1(); - return v_int16x8(vle16_v_i16m1(elems)); + return v_int16x8(vle16_v_i16m1(elems, 8)); } inline v_uint16x8 v_lut(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut((short*)tab, idx)); } inline v_uint16x8 v_lut_pairs(const ushort* tab, const int* idx) { return v_reinterpret_as_u16(v_lut_pairs((short*)tab, idx)); } @@ -967,32 +922,29 @@ inline v_uint16x8 v_lut_quads(const ushort* tab, const int* idx) { return v_rein inline v_int32x4 v_lut(const int* tab, const int* idx) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_int32x4 v_lut_pairs(const int* tab, const int* idx) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[idx[0]], tab[idx[0] + 1], tab[idx[1]], tab[idx[1] + 1] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_int32x4 v_lut_quads(const int* tab, const int* idx) { - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(tab + idx[0])); + return v_int32x4(vle32_v_i32m1(tab + idx[0], 4)); } inline v_uint32x4 v_lut(const unsigned* tab, const int* idx) { return v_reinterpret_as_u32(v_lut((int*)tab, idx)); } @@ -1001,94 +953,86 @@ inline v_uint32x4 v_lut_quads(const unsigned* tab, const int* idx) { return v_re inline v_int64x2 v_lut(const int64_t* tab, const int* idx) { - int64_t CV_DECL_ALIGNED(32) elems[2] = + int64_t elems[2] = { tab[idx[0]], tab[idx[1]] }; - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(elems)); + return v_int64x2(vle64_v_i64m1(elems, 2)); } inline v_int64x2 v_lut_pairs(const int64* tab, const int* idx) { - vsetvlmax_e64m1(); - return v_int64x2(vle64_v_i64m1(tab + idx[0])); + return v_int64x2(vle64_v_i64m1(tab + idx[0], 2)); } inline v_uint64x2 v_lut(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut((const int64_t *)tab, idx)); } inline v_uint64x2 v_lut_pairs(const uint64* tab, const int* idx) { return v_reinterpret_as_u64(v_lut_pairs((const int64_t *)tab, idx)); } inline v_float32x4 v_lut(const float* tab, const int* idx) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_float32x4 v_lut_pairs(const float* tab, const int* idx) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[idx[0]], tab[idx[0] + 1], tab[idx[1]], tab[idx[1] + 1] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline v_float32x4 v_lut_quads(const float* tab, const int* idx) { - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(tab + idx[0])); + return v_float32x4(vle32_v_f32m1(tab + idx[0], 4)); } inline v_int32x4 v_lut(const int* tab, const v_int32x4& idxvec) { - int CV_DECL_ALIGNED(32) elems[4] = + int elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_int32x4(vle32_v_i32m1(elems)); + return v_int32x4(vle32_v_i32m1(elems, 4)); } inline v_uint32x4 v_lut(const unsigned* tab, const v_int32x4& idxvec) { - unsigned CV_DECL_ALIGNED(32) elems[4] = + unsigned elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_uint32x4(vle32_v_u32m1(elems)); + return v_uint32x4(vle32_v_u32m1(elems, 4)); } inline v_float32x4 v_lut(const float* tab, const v_int32x4& idxvec) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)], tab[v_extract_n<2>(idxvec)], tab[v_extract_n<3>(idxvec)] }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_float32x4& x, v_float32x4& y) { - int CV_DECL_ALIGNED(32) idx[4]; + int idx[4]; v_store_aligned(idx, idxvec); x = v_float32x4(tab[idx[0]], tab[idx[1]], tab[idx[2]], tab[idx[3]]); @@ -1098,35 +1042,32 @@ inline void v_lut_deinterleave(const float* tab, const v_int32x4& idxvec, v_floa #if CV_SIMD128_64F inline v_float64x2 v_lut(const double* tab, const int* idx) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { tab[idx[0]], tab[idx[1]] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_lut_pairs(const double* tab, const int* idx) { - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(tab + idx[0])); + return v_float64x2(vle64_v_f64m1(tab + idx[0], 2)); } inline v_float64x2 v_lut(const double* tab, const v_int32x4& idxvec) { - double CV_DECL_ALIGNED(32) elems[2] = + double elems[2] = { tab[v_extract_n<0>(idxvec)], tab[v_extract_n<1>(idxvec)] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_float64x2& x, v_float64x2& y) { - int CV_DECL_ALIGNED(32) idx[4] = {0}; + int idx[4] = {0}; v_store_aligned(idx, idxvec); x = v_float64x2(tab[idx[0]], tab[idx[1]]); @@ -1138,30 +1079,28 @@ inline void v_lut_deinterleave(const double* tab, const v_int32x4& idxvec, v_flo inline v_uint8x16 v_pack_b(const v_uint16x8& a, const v_uint16x8& b) { - ushort CV_DECL_ALIGNED(32) ptr[16] = {0}; + ushort ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 8, b); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vle16_v_u16m2(ptr), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vle16_v_u16m2(ptr, 16), 0, 16)); } inline v_uint8x16 v_pack_b(const v_uint32x4& a, const v_uint32x4& b, const v_uint32x4& c, const v_uint32x4& d) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 4, b); v_store(ptr + 8, c); v_store(ptr + 12, d); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vle32_v_u32m4(ptr), 0), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vle32_v_u32m4(ptr, 16), 0, 16), 0, 16)); } inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uint64x2& c, const v_uint64x2& d, const v_uint64x2& e, const v_uint64x2& f, const v_uint64x2& g, const v_uint64x2& h) { - uint64 CV_DECL_ALIGNED(32) ptr[16] = {0}; + uint64 ptr[16] = {0}; v_store(ptr, a); v_store(ptr + 2, b); v_store(ptr + 4, c); @@ -1170,95 +1109,89 @@ inline v_uint8x16 v_pack_b(const v_uint64x2& a, const v_uint64x2& b, const v_uin v_store(ptr + 10, f); v_store(ptr + 12, g); v_store(ptr + 14, h); - vsetvlmax_e8m1(); - return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vnsrl_wx_u32m4(vle64_v_u64m8(ptr), 0), 0), 0)); + return v_uint8x16(vnsrl_wx_u8m1(vnsrl_wx_u16m2(vnsrl_wx_u32m4(vle64_v_u64m8(ptr, 16), 0, 16), 0, 16), 0, 16)); } ////////////// Arithmetics ////////////// -#define OPENCV_HAL_IMPL_RVV_BIN_OP(bin_op, _Tpvec, intrin, width) \ +#define OPENCV_HAL_IMPL_RVV_BIN_OP(bin_op, _Tpvec, intrin, vl) \ inline _Tpvec operator bin_op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(intrin(a, b)); \ + return _Tpvec(intrin(a, b, vl)); \ } \ inline _Tpvec& operator bin_op##= (_Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - a = _Tpvec(intrin(a, b)); \ + a = _Tpvec(intrin(a, b, vl)); \ return a; \ } -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint8x16, vsaddu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint8x16, vssubu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint8x16, vdivu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int8x16, vsadd_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int8x16, vssub_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int8x16, vdiv_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint16x8, vsaddu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint16x8, vssubu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint16x8, vdivu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int16x8, vsadd_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int16x8, vssub_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int16x8, vdiv_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint32x4, vadd_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint32x4, vsub_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint32x4, vmul_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint32x4, vdivu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int32x4, vadd_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int32x4, vsub_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int32x4, vmul_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int32x4, vdiv_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float32x4, vfadd_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float32x4, vfsub_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float32x4, vfmul_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float32x4, vfdiv_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint64x2, vadd_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint64x2, vsub_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint64x2, vmul_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint64x2, vdivu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int64x2, vadd_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int64x2, vsub_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int64x2, vmul_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int64x2, vdiv_vv_i64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint8x16, vsaddu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint8x16, vssubu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint8x16, vdivu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int8x16, vsadd_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int8x16, vssub_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int8x16, vdiv_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint16x8, vsaddu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint16x8, vssubu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint16x8, vdivu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int16x8, vsadd_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int16x8, vssub_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int16x8, vdiv_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint32x4, vadd_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint32x4, vsub_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint32x4, vmul_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint32x4, vdivu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int32x4, vadd_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int32x4, vsub_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int32x4, vmul_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int32x4, vdiv_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float32x4, vfadd_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float32x4, vfsub_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float32x4, vfmul_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float32x4, vfdiv_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_uint64x2, vadd_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_uint64x2, vsub_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_uint64x2, vmul_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_uint64x2, vdivu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_int64x2, vadd_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_int64x2, vsub_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_int64x2, vmul_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_int64x2, vdiv_vv_i64m1, 2) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float64x2, vfadd_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float64x2, vfsub_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float64x2, vfmul_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float64x2, vfdiv_vv_f64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_OP(+, v_float64x2, vfadd_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(-, v_float64x2, vfsub_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(*, v_float64x2, vfmul_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_OP(/, v_float64x2, vfdiv_vv_f64m1, 2) #endif ////////////// Bitwise logic ////////////// -#define OPENCV_HAL_IMPL_RVV_LOGIC_OP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(&, _Tpvec, vand_vv_##suffix##m1, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(|, _Tpvec, vor_vv_##suffix##m1, width) \ -OPENCV_HAL_IMPL_RVV_BIN_OP(^, _Tpvec, vxor_vv_##suffix##m1, width) \ +#define OPENCV_HAL_IMPL_RVV_LOGIC_OP(_Tpvec, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(&, _Tpvec, vand_vv_##suffix##m1, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(|, _Tpvec, vor_vv_##suffix##m1, vl) \ +OPENCV_HAL_IMPL_RVV_BIN_OP(^, _Tpvec, vxor_vv_##suffix##m1, vl) \ inline _Tpvec operator ~ (const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vnot_v_##suffix##m1(a)); \ + return _Tpvec(vnot_v_##suffix##m1(a, vl)); \ } -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int64x2, i64, 64) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_LOGIC_OP(v_int64x2, i64, 2) #define OPENCV_HAL_IMPL_RVV_FLT_BIT_OP(bin_op, intrin) \ inline v_float32x4 operator bin_op (const v_float32x4& a, const v_float32x4& b) \ { \ - vsetvlmax_e32m1(); \ - return v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b)))); \ + return v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b), 4))); \ } \ inline v_float32x4& operator bin_op##= (v_float32x4& a, const v_float32x4& b) \ { \ - vsetvlmax_e32m1(); \ - a = v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b)))); \ + a = v_float32x4(vreinterpret_v_i32m1_f32m1(intrin(vreinterpret_v_f32m1_i32m1(a), vreinterpret_v_f32m1_i32m1(b), 4))); \ return a; \ } @@ -1268,21 +1201,18 @@ OPENCV_HAL_IMPL_RVV_FLT_BIT_OP(^, vxor_vv_i32m1) inline v_float32x4 operator ~ (const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_float32x4(vreinterpret_v_i32m1_f32m1(vnot_v_i32m1(vreinterpret_v_f32m1_i32m1(a)))); + return v_float32x4(vreinterpret_v_i32m1_f32m1(vnot_v_i32m1(vreinterpret_v_f32m1_i32m1(a), 4))); } #if CV_SIMD128_64F #define OPENCV_HAL_IMPL_RVV_FLT64_BIT_OP(bin_op, intrin) \ inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \ { \ - vsetvlmax_e64m1(); \ - return v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b)))); \ + return v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b), 2))); \ } \ inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \ { \ - vsetvlmax_e64m1(); \ - a = v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b)))); \ + a = v_float64x2(vreinterpret_v_i64m1_f64m1(intrin(vreinterpret_v_f64m1_i64m1(a), vreinterpret_v_f64m1_i64m1(b), 2))); \ return a; \ } @@ -1292,119 +1222,110 @@ OPENCV_HAL_IMPL_RVV_FLT64_BIT_OP(^, vxor_vv_i64m1) inline v_float64x2 operator ~ (const v_float64x2& a) { - vsetvlmax_e64m1(); - return v_float64x2(vreinterpret_v_i64m1_f64m1(vnot_v_i64m1(vreinterpret_v_f64m1_i64m1(a)))); + return v_float64x2(vreinterpret_v_i64m1_f64m1(vnot_v_i64m1(vreinterpret_v_f64m1_i64m1(a), 2))); } #endif ////////////// Bitwise shifts ////////////// -#define OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(_Tpvec, suffix, vl) \ inline _Tpvec operator << (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ inline _Tpvec operator >> (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shl(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shr(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsrl_vx_##suffix##m1(a, uint8_t(n), vl)); \ } -#define OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(_Tpvec, suffix, vl) \ inline _Tpvec operator << (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ inline _Tpvec operator >> (const _Tpvec& a, int n) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shl(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsll_vx_##suffix##m1(a, uint8_t(n), vl)); \ } \ template inline _Tpvec v_shr(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n))); \ + return _Tpvec(vsra_vx_##suffix##m1(a, uint8_t(n), vl)); \ } -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int64x2, i64, 64) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_UNSIGNED_SHIFT_OP(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_SIGNED_SHIFT_OP(v_int64x2, i64, 2) ////////////// Comparison ////////////// -#define OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, op, intrin, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b), vzero_##suffix##m1(), 1)); \ + uint64_t ones = -1; \ + return _Tpvec(vmerge_vxm_##suffix##m1(intrin(a, b, vl), vmv_v_x_##suffix##m1(0, vl), ones, vl)); \ } -#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, op, intrin, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, op, intrin, suffix, vl) \ inline _Tpvec operator op (const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b), vzero_##suffix##m1(), 1)); \ + union { uint64 u; double d; } ones; ones.u = -1; \ + return _Tpvec(vfmerge_vfm_##suffix##m1(intrin(a, b, vl), vfmv_v_f_##suffix##m1(0, vl), ones.d, vl)); \ } -#define OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmsltu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgtu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsleu_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsgeu_vv_##suffix##m1_b##width, suffix, width) +#define OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmsltu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgtu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsleu_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsgeu_vv_##suffix##m1_b##width, suffix, vl) -#define OPENCV_HAL_IMPL_RVV_SIGNED_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmslt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsle_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsge_vv_##suffix##m1_b##width, suffix, width) +#define OPENCV_HAL_IMPL_RVV_SIGNED_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, ==, vmseq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, !=, vmsne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <, vmslt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >, vmsgt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, <=, vmsle_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_INT_CMP_OP(_Tpvec, >=, vmsge_vv_##suffix##m1_b##width, suffix, vl) -#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP(_Tpvec, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, ==, vmfeq_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, !=, vmfne_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <, vmflt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >, vmfgt_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <=, vmfle_vv_##suffix##m1_b##width, suffix, width) \ -OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >=, vmfge_vv_##suffix##m1_b##width, suffix, width) +#define OPENCV_HAL_IMPL_RVV_FLOAT_CMP(_Tpvec, suffix, width, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, ==, vmfeq_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, !=, vmfne_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <, vmflt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >, vmfgt_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, <=, vmfle_vv_##suffix##m1_b##width, suffix, vl) \ +OPENCV_HAL_IMPL_RVV_FLOAT_CMP_OP(_Tpvec, >=, vmfge_vv_##suffix##m1_b##width, suffix, vl) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int64x2, i64, 64) -OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float32x4, f32, 32) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint8x16, u8, 8, 16) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint16x8, u16, 16, 8) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint32x4, u32, 32, 4) +OPENCV_HAL_IMPL_RVV_UNSIGNED_CMP(v_uint64x2, u64, 64, 2) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int8x16, i8, 8, 16) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int16x8, i16, 16, 8) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int32x4, i32, 32, 4) +OPENCV_HAL_IMPL_RVV_SIGNED_CMP(v_int64x2, i64, 64, 2) +OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float32x4, f32, 32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float64x2, f64, 64) +OPENCV_HAL_IMPL_RVV_FLOAT_CMP(v_float64x2, f64, 64, 2) #endif inline v_float32x4 v_not_nan(const v_float32x4& a) @@ -1417,121 +1338,126 @@ inline v_float64x2 v_not_nan(const v_float64x2& a) ////////////// Min/Max ////////////// -#define OPENCV_HAL_IMPL_RVV_BIN_FUNC(_Tpvec, func, intrin, width) \ +#define OPENCV_HAL_IMPL_RVV_BIN_FUNC(_Tpvec, func, intrin, vl) \ inline _Tpvec func(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(intrin(a, b)); \ + return _Tpvec(intrin(a, b, vl)); \ } -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_min, vminu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_max, vmaxu_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_min, vmin_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_max, vmax_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_min, vminu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_max, vmaxu_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_min, vmin_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_max, vmax_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_min, vminu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_max, vmaxu_vv_u32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_min, vmin_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_max, vmax_vv_i32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_min, vfmin_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_max, vfmax_vv_f32m1, 32) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_min, vminu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_max, vmaxu_vv_u64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_min, vmin_vv_i64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_max, vmax_vv_i64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_min, vminu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_max, vmaxu_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_min, vmin_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_max, vmax_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_min, vminu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_max, vmaxu_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_min, vmin_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_max, vmax_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_min, vminu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint32x4, v_max, vmaxu_vv_u32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_min, vmin_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int32x4, v_max, vmax_vv_i32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_min, vfmin_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float32x4, v_max, vfmax_vv_f32m1, 4) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_min, vminu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint64x2, v_max, vmaxu_vv_u64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_min, vmin_vv_i64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int64x2, v_max, vmax_vv_i64m1, 2) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_min, vfmin_vv_f64m1, 64) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_max, vfmax_vv_f64m1, 64) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_min, vfmin_vv_f64m1, 2) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_float64x2, v_max, vfmax_vv_f64m1, 2) #endif ////////////// Arithmetics wrap ////////////// -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_add_wrap, vadd_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_add_wrap, vadd_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_add_wrap, vadd_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_add_wrap, vadd_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_sub_wrap, vsub_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_sub_wrap, vsub_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_sub_wrap, vsub_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_sub_wrap, vsub_vv_i16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_mul_wrap, vmul_vv_u8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_mul_wrap, vmul_vv_i8m1, 8) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_mul_wrap, vmul_vv_u16m1, 16) -OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_mul_wrap, vmul_vv_i16m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_add_wrap, vadd_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_add_wrap, vadd_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_add_wrap, vadd_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_add_wrap, vadd_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_sub_wrap, vsub_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_sub_wrap, vsub_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_sub_wrap, vsub_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_sub_wrap, vsub_vv_i16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint8x16, v_mul_wrap, vmul_vv_u8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int8x16, v_mul_wrap, vmul_vv_i8m1, 16) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_uint16x8, v_mul_wrap, vmul_vv_u16m1, 8) +OPENCV_HAL_IMPL_RVV_BIN_FUNC(v_int16x8, v_mul_wrap, vmul_vv_i16m1, 8) ////////////// Reduce ////////////// -#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, wwidth, red) \ +#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, vl, red) \ inline scalartype v_reduce_sum(const _Tpvec& a) \ { \ - vsetvlmax_e##wwidth##m1(); \ - _nwTpvec zero = vzero_##wsuffix##m1(); \ - _nwTpvec res = vzero_##wsuffix##m1(); \ - res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero); \ + _nwTpvec zero = vmv_v_x_##wsuffix##m1(0, vl); \ + _nwTpvec res = vmv_v_x_##wsuffix##m1(0, vl); \ + res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero, vl); \ return (scalartype)(_wTpvec(res).get0()); \ } OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint8x16, v_uint16x8, vuint16m1_t, unsigned, u8, u16, 16, wredsumu) OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int8x16, v_int16x8, vint16m1_t, int, i8, i16, 16, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint16x8, v_uint32x4, vuint32m1_t, unsigned, u16, u32, 32, wredsumu) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int16x8, v_int32x4, vint32m1_t, int, i16, i32, 32, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint32x4, v_uint64x2, vuint64m1_t, unsigned, u32, u64, 64, wredsumu) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int32x4, v_int64x2, vint64m1_t, int, i32, i64, 64, wredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 32, fredsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 64, redsum) -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 64, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint16x8, v_uint32x4, vuint32m1_t, unsigned, u16, u32, 8, wredsumu) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int16x8, v_int32x4, vint32m1_t, int, i16, i32, 8, wredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint32x4, v_uint64x2, vuint64m1_t, unsigned, u32, u64, 4, wredsumu) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int32x4, v_int64x2, vint64m1_t, int, i32, i64, 4, wredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_uint64x2, v_uint64x2, vuint64m1_t, uint64, u64, u64, 2, redsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_int64x2, v_int64x2, vint64m1_t, int64, i64, i64, 2, redsum) + +#define OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(_Tpvec, _wTpvec, _nwTpvec, scalartype, suffix, wsuffix, vl, red) \ +inline scalartype v_reduce_sum(const _Tpvec& a) \ +{ \ + _nwTpvec zero = vfmv_v_f_##wsuffix##m1(0, vl); \ + _nwTpvec res = vfmv_v_f_##wsuffix##m1(0, vl); \ + res = v##red##_vs_##suffix##m1_##wsuffix##m1(res, a, zero, vl); \ + return (scalartype)(_wTpvec(res).get0()); \ +} + +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float32x4, v_float32x4, vfloat32m1_t, float, f32, f32, 4, fredsum) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REDUCE_SUM(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 64, fredsum) +OPENCV_HAL_IMPL_RVV_REDUCE_SUM_FP(v_float64x2, v_float64x2, vfloat64m1_t, double, f64, f64, 2, fredsum) #endif -#define OPENCV_HAL_IMPL_RVV_REDUCE(_Tpvec, func, scalartype, suffix, width, red) \ +#define OPENCV_HAL_IMPL_RVV_REDUCE(_Tpvec, func, scalartype, suffix, vl, red) \ inline scalartype v_reduce_##func(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - _Tpvec res = _Tpvec(v##red##_vs_##suffix##m1_##suffix##m1(a, a, a)); \ + _Tpvec res = _Tpvec(v##red##_vs_##suffix##m1_##suffix##m1(a, a, a, vl)); \ return scalartype(res.get0()); \ } -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, min, uchar, u8, 8, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, min, schar, i8, 8, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, min, ushort, u16, 16, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, min, short, i16, 16, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, min, unsigned, u32, 32, redminu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, min, int, i32, 32, redmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, min, float, f32, 32, fredmin) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, max, uchar, u8, 8, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, max, schar, i8, 8, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, max, ushort, u16, 16, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, max, short, i16, 16, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, max, unsigned, u32, 32, redmaxu) -OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, max, int, i32, 32, redmax) -OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, max, float, f32, 32, fredmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, min, uchar, u8, 16, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, min, schar, i8, 16, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, min, ushort, u16, 8, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, min, short, i16, 8, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, min, unsigned, u32, 4, redminu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, min, int, i32, 4, redmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, min, float, f32, 4, fredmin) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint8x16, max, uchar, u8, 16, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int8x16, max, schar, i8, 16, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint16x8, max, ushort, u16, 8, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int16x8, max, short, i16, 8, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_uint32x4, max, unsigned, u32, 4, redmaxu) +OPENCV_HAL_IMPL_RVV_REDUCE(v_int32x4, max, int, i32, 4, redmax) +OPENCV_HAL_IMPL_RVV_REDUCE(v_float32x4, max, float, f32, 4, fredmax) inline v_float32x4 v_reduce_sum4(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c, const v_float32x4& d) { - float CV_DECL_ALIGNED(32) elems[4] = + float elems[4] = { v_reduce_sum(a), v_reduce_sum(b), v_reduce_sum(c), v_reduce_sum(d) }; - vsetvlmax_e32m1(); - return v_float32x4(vle32_v_f32m1(elems)); + return v_float32x4(vle32_v_f32m1(elems, 4)); } ////////////// Square-Root ////////////// inline v_float32x4 v_sqrt(const v_float32x4& x) { - vsetvlmax_e32m1(); - return v_float32x4(vfsqrt_v_f32m1(x)); + return v_float32x4(vfsqrt_v_f32m1(x, 4)); } inline v_float32x4 v_invsqrt(const v_float32x4& x) @@ -1543,8 +1469,7 @@ inline v_float32x4 v_invsqrt(const v_float32x4& x) #if CV_SIMD128_64F inline v_float64x2 v_sqrt(const v_float64x2& x) { - vsetvlmax_e64m1(); - return v_float64x2(vfsqrt_v_f64m1(x)); + return v_float64x2(vfsqrt_v_f64m1(x, 4)); } inline v_float64x2 v_invsqrt(const v_float64x2& x) @@ -1556,29 +1481,25 @@ inline v_float64x2 v_invsqrt(const v_float64x2& x) inline v_float32x4 v_magnitude(const v_float32x4& a, const v_float32x4& b) { - vsetvlmax_e32m1(); - v_float32x4 x(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a), b, b)); + v_float32x4 x(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a, 4), b, b, 4)); return v_sqrt(x); } inline v_float32x4 v_sqr_magnitude(const v_float32x4& a, const v_float32x4& b) { - vsetvlmax_e32m1(); - return v_float32x4(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a), b, b)); + return v_float32x4(vfmacc_vv_f32m1(vfmul_vv_f32m1(a, a, 4), b, b, 4)); } #if CV_SIMD128_64F inline v_float64x2 v_magnitude(const v_float64x2& a, const v_float64x2& b) { - vsetvlmax_e64m1(); - v_float64x2 x(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a), b, b)); + v_float64x2 x(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a, 2), b, b, 2)); return v_sqrt(x); } inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b) { - vsetvlmax_e64m1(); - return v_float64x2(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a), b, b)); + return v_float64x2(vfmacc_vv_f64m1(vfmul_vv_f64m1(a, a, 2), b, b, 2)); } #endif @@ -1586,13 +1507,11 @@ inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b) inline v_float32x4 v_fma(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) { - vsetvlmax_e32m1(); - return v_float32x4(vfmacc_vv_f32m1(c, a, b)); + return v_float32x4(vfmacc_vv_f32m1(c, a, b, 4)); } inline v_int32x4 v_fma(const v_int32x4& a, const v_int32x4& b, const v_int32x4& c) { - vsetvlmax_e32m1(); - return v_int32x4(vmacc_vv_i32m1(c, a, b)); + return v_int32x4(vmacc_vv_i32m1(c, a, b, 4)); } inline v_float32x4 v_muladd(const v_float32x4& a, const v_float32x4& b, const v_float32x4& c) @@ -1608,8 +1527,7 @@ inline v_int32x4 v_muladd(const v_int32x4& a, const v_int32x4& b, const v_int32x #if CV_SIMD128_64F inline v_float64x2 v_fma(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) { - vsetvlmax_e64m1(); - return v_float64x2(vfmacc_vv_f64m1(c, a, b)); + return v_float64x2(vfmacc_vv_f64m1(c, a, b, 2)); } inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) @@ -1620,24 +1538,22 @@ inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_ ////////////// Check all/any ////////////// -#define OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(_Tpvec, suffix, shift, width) \ +#define OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(_Tpvec, suffix, shift, vl) \ inline bool v_check_all(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(vnot_v_##suffix##m1(a), shift)); \ + v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(vnot_v_##suffix##m1(a, vl), shift, vl)); \ return (v.val[0] | v.val[1]) == 0; \ } \ inline bool v_check_any(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(a, shift)); \ + v_uint64x2 v = v_uint64x2((vuint64m1_t)vsrl_vx_##suffix##m1(a, shift, vl)); \ return (v.val[0] | v.val[1]) != 0; \ } -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint8x16, u8, 7, 8) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint16x8, u16, 15, 16) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint32x4, u32, 31, 32) -OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint64x2, u64, 63, 64) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint8x16, u8, 7, 16) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint16x8, u16, 15, 8) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint32x4, u32, 31, 4) +OPENCV_HAL_IMPL_RVV_CHECK_ALLANY(v_uint64x2, u64, 63, 2) inline bool v_check_all(const v_int8x16& a) @@ -1690,16 +1606,15 @@ OPENCV_HAL_IMPL_RVV_ABSDIFF(v_float64x2, absdiff) OPENCV_HAL_IMPL_RVV_ABSDIFF(v_int8x16, absdiffs) OPENCV_HAL_IMPL_RVV_ABSDIFF(v_int16x8, absdiffs) -#define OPENCV_HAL_IMPL_RVV_ABSDIFF_S(_Tpvec, _rTpvec, _nwTpvec, sub, rshr, width) \ +#define OPENCV_HAL_IMPL_RVV_ABSDIFF_S(_Tpvec, _rTpvec, _nwTpvec, sub, rshr, vl) \ inline _rTpvec v_absdiff(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _rTpvec(rshr((_nwTpvec)sub(v_max(a, b), v_min(a, b)), 0)); \ + return _rTpvec(rshr((_nwTpvec)sub(v_max(a, b), v_min(a, b), vl), 0, vl)); \ } -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int8x16, v_uint8x16, vuint16m2_t, vwsub_vv_i16m2, vnclipu_wx_u8m1, 8) -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int16x8, v_uint16x8, vuint32m2_t, vwsub_vv_i32m2, vnclipu_wx_u16m1, 16) -OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int32x4, v_uint32x4, vuint64m2_t, vwsub_vv_i64m2, vnclipu_wx_u32m1, 32) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int8x16, v_uint8x16, vuint16m2_t, vwsub_vv_i16m2, vnclipu_wx_u8m1, 16) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int16x8, v_uint16x8, vuint32m2_t, vwsub_vv_i32m2, vnclipu_wx_u16m1, 8) +OPENCV_HAL_IMPL_RVV_ABSDIFF_S(v_int32x4, v_uint32x4, vuint64m2_t, vwsub_vv_i64m2, vnclipu_wx_u32m1, 4) #define OPENCV_HAL_IMPL_RVV_ABS(_Tprvec, _Tpvec, suffix) \ inline _Tprvec v_abs(const _Tpvec& a) \ @@ -1732,149 +1647,152 @@ OPENCV_HAL_IMPL_RVV_REDUCE_SAD(v_float32x4, float) ////////////// Select ////////////// -#define OPENCV_HAL_IMPL_RVV_SELECT(_Tpvec, merge, ne, width) \ +#define OPENCV_HAL_IMPL_RVV_SELECT(_Tpvec, merge, ne, vl) \ inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(merge(ne(mask, 0), b, a)); \ + return _Tpvec(merge(ne(mask, 0, vl), b, a, vl)); \ } -OPENCV_HAL_IMPL_RVV_SELECT(v_uint8x16, vmerge_vvm_u8m1, vmsne_vx_u8m1_b8, 8) -OPENCV_HAL_IMPL_RVV_SELECT(v_int8x16, vmerge_vvm_i8m1, vmsne_vx_i8m1_b8, 8) -OPENCV_HAL_IMPL_RVV_SELECT(v_uint16x8, vmerge_vvm_u16m1, vmsne_vx_u16m1_b16, 16) -OPENCV_HAL_IMPL_RVV_SELECT(v_int16x8, vmerge_vvm_i16m1, vmsne_vx_i16m1_b16, 16) -OPENCV_HAL_IMPL_RVV_SELECT(v_uint32x4, vmerge_vvm_u32m1, vmsne_vx_u32m1_b32, 32) -OPENCV_HAL_IMPL_RVV_SELECT(v_int32x4, vmerge_vvm_i32m1, vmsne_vx_i32m1_b32, 32) -OPENCV_HAL_IMPL_RVV_SELECT(v_float32x4, vmerge_vvm_f32m1, vmfne_vf_f32m1_b32, 32) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint8x16, vmerge_vvm_u8m1, vmsne_vx_u8m1_b8, 16) +OPENCV_HAL_IMPL_RVV_SELECT(v_int8x16, vmerge_vvm_i8m1, vmsne_vx_i8m1_b8, 16) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint16x8, vmerge_vvm_u16m1, vmsne_vx_u16m1_b16, 8) +OPENCV_HAL_IMPL_RVV_SELECT(v_int16x8, vmerge_vvm_i16m1, vmsne_vx_i16m1_b16, 8) +OPENCV_HAL_IMPL_RVV_SELECT(v_uint32x4, vmerge_vvm_u32m1, vmsne_vx_u32m1_b32, 4) +OPENCV_HAL_IMPL_RVV_SELECT(v_int32x4, vmerge_vvm_i32m1, vmsne_vx_i32m1_b32, 4) +OPENCV_HAL_IMPL_RVV_SELECT(v_float32x4, vmerge_vvm_f32m1, vmfne_vf_f32m1_b32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_SELECT(v_float64x2, vmerge_vvm_f64m1, vmfne_vf_f64m1_b64, 64) +OPENCV_HAL_IMPL_RVV_SELECT(v_float64x2, vmerge_vvm_f64m1, vmfne_vf_f64m1_b64, 2) #endif ////////////// Rotate shift ////////////// -#define OPENCV_HAL_IMPL_RVV_ROTATE_OP(_Tpvec, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(_Tpvec, suffix, vl) \ template inline _Tpvec v_rotate_right(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, n)); \ + return _Tpvec(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl)); \ } \ template inline _Tpvec v_rotate_left(const _Tpvec& a) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vzero_##suffix##m1(), a, n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl)); \ } \ template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \ { return a; } \ template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), a, n), b, _Tpvec::nlanes - n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), a, n, vl), b, _Tpvec::nlanes - n, vl)); \ } \ template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ { \ - vsetvlmax_e##width##m1(); \ - return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vzero_##suffix##m1(), b, _Tpvec::nlanes - n), a, n)); \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vmv_v_x_##suffix##m1(0, vl), b, _Tpvec::nlanes - n, vl), a, n, vl)); \ } \ template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \ { CV_UNUSED(b); return a; } +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint8x16, u8, 16) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int8x16, i8, 16) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint16x8, u16, 8) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int16x8, i16, 8) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint32x4, u32, 4) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int32x4, i32, 4) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_uint64x2, u64, 2) +OPENCV_HAL_IMPL_RVV_ROTATE_INTEGER(v_int64x2, i64, 2) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint8x16, u8, 8) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int8x16, i8, 8) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint16x8, u16, 16) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int16x8, i16, 16) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint32x4, u32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int32x4, i32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_float32x4, f32, 32) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_uint64x2, u64, 64) -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_int64x2, i64, 64) +#define OPENCV_HAL_IMPL_RVV_ROTATE_FP(_Tpvec, suffix, vl) \ +template inline _Tpvec v_rotate_right(const _Tpvec& a) \ +{ \ + return _Tpvec(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl)); \ +} \ +template inline _Tpvec v_rotate_left(const _Tpvec& a) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl)); \ +} \ +template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a) \ +{ return a; } \ +template inline _Tpvec v_rotate_right(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), a, n, vl), b, _Tpvec::nlanes - n, vl)); \ +} \ +template inline _Tpvec v_rotate_left(const _Tpvec& a, const _Tpvec& b) \ +{ \ + return _Tpvec(vslideup_vx_##suffix##m1(vslidedown_vx_##suffix##m1(vfmv_v_f_##suffix##m1(0, vl), b, _Tpvec::nlanes - n, vl), a, n, vl)); \ +} \ +template<> inline _Tpvec v_rotate_left<0>(const _Tpvec& a, const _Tpvec& b) \ +{ CV_UNUSED(b); return a; } + +OPENCV_HAL_IMPL_RVV_ROTATE_FP(v_float32x4, f32, 4) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_ROTATE_OP(v_float64x2, f64, 64) +OPENCV_HAL_IMPL_RVV_ROTATE_FP(v_float64x2, f64, 2) #endif ////////////// Convert to float ////////////// inline v_float32x4 v_cvt_f32(const v_int32x4& a) { - vsetvlmax_e32m1(); - return v_float32x4(vfcvt_f_x_v_f32m1(a)); + return v_float32x4(vfcvt_f_x_v_f32m1(a, 4)); } #if CV_SIMD128_64F inline v_float32x4 v_cvt_f32(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - vsetvlmax_e32m1(); - return v_float32x4(vfncvt_f_f_w_f32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_float32x4(vfncvt_f_f_w_f32m1(tmp, 4)); } inline v_float32x4 v_cvt_f32(const v_float64x2& a, const v_float64x2& b) { double arr[4] = {a.val[0], a.val[1], b.val[0], b.val[1]}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - vsetvlmax_e32m1(); - return v_float32x4(vfncvt_f_f_w_f32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_float32x4(vfncvt_f_f_w_f32m1(tmp, 4)); } inline v_float64x2 v_cvt_f64(const v_int32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a)); - double CV_DECL_ALIGNED(32) elems[2] = + double ptr[4] = {0}; + vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); + double elems[2] = { ptr[0], ptr[1] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a)); - double CV_DECL_ALIGNED(32) elems[2] = + double ptr[4] = {0}; + vse64_v_f64m2(ptr, vfwcvt_f_x_v_f64m2(a, 4), 4); + double elems[2] = { ptr[2], ptr[3] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64(const v_float32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a)); - double CV_DECL_ALIGNED(32) elems[2] = + double ptr[4] = {0}; + vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); + double elems[2] = { ptr[0], ptr[1] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) { - double CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a)); - double CV_DECL_ALIGNED(32) elems[2] = + double ptr[4] = {0}; + vse64_v_f64m2(ptr, vfwcvt_f_f_v_f64m2(a, 4), 4); + double elems[2] = { ptr[2], ptr[3] }; - vsetvlmax_e64m1(); - return v_float64x2(vle64_v_f64m1(elems)); + return v_float64x2(vle64_v_f64m1(elems, 2)); } inline v_float64x2 v_cvt_f64(const v_int64x2& a) { - vsetvlmax_e64m1(); - return v_float64x2(vfcvt_f_x_v_f64m1(a)); + return v_float64x2(vfcvt_f_x_v_f64m1(a, 2)); } #endif @@ -1907,7 +1825,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_##_Tpvec& b0, v_##_Tpvec& b1, \ v_##_Tpvec& b2, v_##_Tpvec& b3) \ { \ - _Tp CV_DECL_ALIGNED(32) elems0[4] = \ + _Tp elems0[4] = \ { \ v_extract_n<0>(a0), \ v_extract_n<0>(a1), \ @@ -1915,7 +1833,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<0>(a3) \ }; \ b0 = v_load(elems0); \ - _Tp CV_DECL_ALIGNED(32) elems1[4] = \ + _Tp elems1[4] = \ { \ v_extract_n<1>(a0), \ v_extract_n<1>(a1), \ @@ -1923,7 +1841,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<1>(a3) \ }; \ b1 = v_load(elems1); \ - _Tp CV_DECL_ALIGNED(32) elems2[4] = \ + _Tp elems2[4] = \ { \ v_extract_n<2>(a0), \ v_extract_n<2>(a1), \ @@ -1931,7 +1849,7 @@ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ v_extract_n<2>(a3) \ }; \ b2 = v_load(elems2); \ - _Tp CV_DECL_ALIGNED(32) elems3[4] = \ + _Tp elems3[4] = \ { \ v_extract_n<3>(a0), \ v_extract_n<3>(a1), \ @@ -1947,11 +1865,11 @@ OPENCV_HAL_IMPL_RVV_TRANSPOSE4x4(float32x4, float, f32) ////////////// Reverse ////////////// -#define OPENCV_HAL_IMPL_RVV_REVERSE(_Tpvec, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_REVERSE(_Tpvec, _Tp, suffix) \ inline _Tpvec v_reverse(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptra[_Tpvec::nlanes] = {0}; \ + _Tp ptr[_Tpvec::nlanes] = {0}; \ + _Tp ptra[_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ for (int i = 0; i < _Tpvec::nlanes; i++) \ { \ @@ -1960,159 +1878,149 @@ inline _Tpvec v_reverse(const _Tpvec& a) \ return v_load(ptr); \ } -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint8x16, uchar, 8, u8) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int8x16, schar, 8, i8) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint16x8, ushort, 16, u16) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int16x8, short, 16, i16) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint32x4, unsigned, 32, u32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int32x4, int, 32, i32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_float32x4, float, 32, f32) -OPENCV_HAL_IMPL_RVV_REVERSE(v_uint64x2, uint64, 64, u64) -OPENCV_HAL_IMPL_RVV_REVERSE(v_int64x2, int64, 64, i64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint8x16, uchar, u8) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int8x16, schar, i8) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint16x8, ushort, u16) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int16x8, short, i16) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint32x4, unsigned, u32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int32x4, int, i32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_float32x4, float, f32) +OPENCV_HAL_IMPL_RVV_REVERSE(v_uint64x2, uint64, u64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_int64x2, int64, i64) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_REVERSE(v_float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_REVERSE(v_float64x2, double, f64) #endif //////////// Value reordering //////////// -#define OPENCV_HAL_IMPL_RVV_EXPAND(_Tpwvec, _Tp, _Tpvec, width, suffix, wcvt) \ +#define OPENCV_HAL_IMPL_RVV_EXPAND(_Tpwvec, _Tp, _Tpvec, width, suffix, wcvt, vl) \ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ { \ - _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ + _Tp lptr[_Tpvec::nlanes/2] = {0}; \ + _Tp hptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ v_store_high(hptr, a); \ - b0 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr))); \ - b1 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr))); \ + b0 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ + b1 = _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr, vl), vl)); \ } \ inline _Tpwvec v_expand_low(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) lptr[_Tpvec::nlanes/2] = {0}; \ + _Tp lptr[_Tpvec::nlanes/2] = {0}; \ v_store_low(lptr, a); \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(lptr, vl), vl)); \ } \ inline _Tpwvec v_expand_high(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) hptr[_Tpvec::nlanes/2] = {0}; \ + _Tp hptr[_Tpvec::nlanes/2] = {0}; \ v_store_high(hptr, a); \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(hptr, vl), vl)); \ } \ inline _Tpwvec v_load_expand(const _Tp* ptr) \ { \ - return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(ptr))); \ + return _Tpwvec(wcvt(vle##width##_v_##suffix##mf2(ptr, vl), vl)); \ } -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint16x8, uchar, v_uint8x16, 8, u8, vwcvtu_x_x_v_u16m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int16x8, schar, v_int8x16, 8, i8, vwcvt_x_x_v_i16m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint32x4, ushort, v_uint16x8, 16, u16, vwcvtu_x_x_v_u32m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int32x4, short, v_int16x8, 16, i16, vwcvt_x_x_v_i32m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_uint64x2, uint, v_uint32x4, 32, u32, vwcvtu_x_x_v_u64m1) -OPENCV_HAL_IMPL_RVV_EXPAND(v_int64x2, int, v_int32x4, 32, i32, vwcvt_x_x_v_i64m1) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint16x8, uchar, v_uint8x16, 8, u8, vwcvtu_x_x_v_u16m1, 8) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int16x8, schar, v_int8x16, 8, i8, vwcvt_x_x_v_i16m1, 8) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint32x4, ushort, v_uint16x8, 16, u16, vwcvtu_x_x_v_u32m1, 4) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int32x4, short, v_int16x8, 16, i16, vwcvt_x_x_v_i32m1, 4) +OPENCV_HAL_IMPL_RVV_EXPAND(v_uint64x2, uint, v_uint32x4, 32, u32, vwcvtu_x_x_v_u64m1, 2) +OPENCV_HAL_IMPL_RVV_EXPAND(v_int64x2, int, v_int32x4, 32, i32, vwcvt_x_x_v_i64m1, 2) inline v_uint32x4 v_load_expand_q(const uchar* ptr) { - vsetvlmax_e32m1(); - return v_uint32x4(vwcvtu_x_x_v_u32m1(vwcvtu_x_x_v_u16mf2(vle8_v_u8mf4(ptr)))); + return v_uint32x4(vwcvtu_x_x_v_u32m1(vwcvtu_x_x_v_u16mf2(vle8_v_u8mf4(ptr, 4), 4), 4)); } inline v_int32x4 v_load_expand_q(const schar* ptr) { - vsetvlmax_e32m1(); - return v_int32x4(vwcvt_x_x_v_i32m1(vwcvt_x_x_v_i16mf2(vle8_v_i8mf4(ptr)))); + return v_int32x4(vwcvt_x_x_v_i32m1(vwcvt_x_x_v_i16mf2(vle8_v_i8mf4(ptr, 4), 4), 4)); } -#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, shr) \ +#define OPENCV_HAL_IMPL_RVV_PACK(_Tpvec, _Tp, _wTpvec, _wTp, hwidth, width, hsuffix, suffix, rshr, shr, hvl, vl) \ inline _Tpvec v_pack(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(shr(vle##width##_v_##suffix##m2(arr), 0)); \ + return _Tpvec(shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl)); \ } \ inline void v_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(shr(vle##width##_v_##suffix##m2(arr), 0))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, shr(vle##width##_v_##suffix##m2(arr, vl), 0, vl), hvl); \ } \ template inline \ _Tpvec v_rshr_pack(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(vle##width##_v_##suffix##m2(arr), n)); \ + return _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl)); \ } \ template inline \ void v_rshr_pack_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr), n))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, _Tpvec(rshr(vle##width##_v_##suffix##m2(arr, vl), n, vl)), hvl); \ } -OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 16, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 16, i16, vnclip_wx_i8m1, vnclip_wx_i8m1) -OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 32, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 32, i32, vnclip_wx_i16m1, vnclip_wx_i16m1) -OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 64, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1) -OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 64, i64, vnclip_wx_i32m1, vnsra_wx_i32m1) +OPENCV_HAL_IMPL_RVV_PACK(v_uint8x16, uchar, v_uint16x8, ushort, 8, 16, u8, u16, vnclipu_wx_u8m1, vnclipu_wx_u8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_int8x16, schar, v_int16x8, short, 8, 16, i8, i16, vnclip_wx_i8m1, vnclip_wx_i8m1, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK(v_uint16x8, ushort, v_uint32x4, unsigned, 16, 32, u16, u32, vnclipu_wx_u16m1, vnclipu_wx_u16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_int16x8, short, v_int32x4, int, 16, 32, i16, i32, vnclip_wx_i16m1, vnclip_wx_i16m1, 4, 8) +OPENCV_HAL_IMPL_RVV_PACK(v_uint32x4, unsigned, v_uint64x2, uint64, 32, 64, u32, u64, vnclipu_wx_u32m1, vnsrl_wx_u32m1, 2, 4) +OPENCV_HAL_IMPL_RVV_PACK(v_int32x4, int, v_int64x2, int64, 32, 64, i32, i64, vnclip_wx_i32m1, vnsra_wx_i32m1, 2, 4) -#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, width, suffix, rshr, cast) \ +#define OPENCV_HAL_IMPL_RVV_PACK_U(_Tpvec, _Tp, _wTpvec, _wTp, hwidth, width, hsuffix, suffix, rshr, cast, hvl, vl) \ inline _Tpvec v_pack_u(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), 0)); \ + return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl)); \ } \ inline void v_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), 0))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + vse##hwidth##_v_##hsuffix##m1(ptr, rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), 0, vl), hvl); \ } \ template inline \ _Tpvec v_rshr_pack_u(const _wTpvec& a, const _wTpvec& b) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ v_store(arr + _wTpvec::nlanes, b); \ - vsetvlmax_e##width##m2(); \ - return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), n)); \ + return _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl)); \ } \ template inline \ void v_rshr_pack_u_store(_Tp* ptr, const _wTpvec& a) \ { \ - _wTp CV_DECL_ALIGNED(32) arr[_Tpvec::nlanes] = {0}; \ + _wTp arr[_Tpvec::nlanes] = {0}; \ v_store(arr, a); \ - v_store(arr + _wTpvec::nlanes, _wTpvec(vzero_##suffix##m1())); \ - vsetvlmax_e##width##m2(); \ - v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr), 0)), n))); \ + v_store(arr + _wTpvec::nlanes, _wTpvec(vmv_v_x_##suffix##m1(0, hvl))); \ + v_store(ptr, _Tpvec(rshr(cast(vmax_vx_##suffix##m2(vle##width##_v_##suffix##m2(arr, vl), 0, vl)), n, vl))); \ } -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 16, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2) -OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 32, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint8x16, uchar, v_int16x8, short, 8, 16, u8, i16, vnclipu_wx_u8m1, vreinterpret_v_i16m2_u16m2, 8, 16) +OPENCV_HAL_IMPL_RVV_PACK_U(v_uint16x8, ushort, v_int32x4, int, 16, 32, u16, i32, vnclipu_wx_u16m1, vreinterpret_v_i32m2_u32m2, 4, 8) -#define OPENCV_HAL_IMPL_RVV_UNPACKS(_Tpvec, _Tp, width, suffix) \ +#define OPENCV_HAL_IMPL_RVV_UNPACKS(_Tpvec, _Tp, suffix) \ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra0[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptra1[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb0[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb1[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra0[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra1[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb0[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb1[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra0, a0); \ v_store(ptra1, a1); \ int i; \ @@ -2131,16 +2039,16 @@ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_ } \ inline v_##_Tpvec v_combine_low(const v_##_Tpvec& a, const v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes/2] = {0}; \ v_store_low(ptra, a); \ v_store_low(ptrb, b); \ return v_load_halves(ptra, ptrb); \ } \ inline v_##_Tpvec v_combine_high(const v_##_Tpvec& a, const v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes/2] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes/2] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes/2] = {0}; \ v_store_high(ptra, a); \ v_store_high(ptrb, b); \ return v_load_halves(ptra, ptrb); \ @@ -2151,23 +2059,23 @@ inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c, d = v_combine_high(a, b); \ } -OPENCV_HAL_IMPL_RVV_UNPACKS(uint8x16, uchar, 8, u8) -OPENCV_HAL_IMPL_RVV_UNPACKS(int8x16, schar, 8, i8) -OPENCV_HAL_IMPL_RVV_UNPACKS(uint16x8, ushort, 16, u16) -OPENCV_HAL_IMPL_RVV_UNPACKS(int16x8, short, 16, i16) -OPENCV_HAL_IMPL_RVV_UNPACKS(uint32x4, unsigned, 32, u32) -OPENCV_HAL_IMPL_RVV_UNPACKS(int32x4, int, 32, i32) -OPENCV_HAL_IMPL_RVV_UNPACKS(float32x4, float, 32, f32) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint8x16, uchar, u8) +OPENCV_HAL_IMPL_RVV_UNPACKS(int8x16, schar, i8) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint16x8, ushort, u16) +OPENCV_HAL_IMPL_RVV_UNPACKS(int16x8, short, i16) +OPENCV_HAL_IMPL_RVV_UNPACKS(uint32x4, unsigned, u32) +OPENCV_HAL_IMPL_RVV_UNPACKS(int32x4, int, i32) +OPENCV_HAL_IMPL_RVV_UNPACKS(float32x4, float, f32) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_UNPACKS(float64x2, double, 64, f64) +OPENCV_HAL_IMPL_RVV_UNPACKS(float64x2, double, f64) #endif -#define OPENCV_HAL_IMPL_RVV_INTERLEAVED(_Tpvec, _Tp, suffix, width) \ +#define OPENCV_HAL_IMPL_RVV_INTERLEAVED(_Tpvec, _Tp) \ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ int i, i2; \ for( i = i2 = 0; i < v_##_Tpvec::nlanes; i++, i2 += 2 ) \ { \ @@ -2179,9 +2087,9 @@ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b) \ } \ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, v_##_Tpvec& c) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ int i, i3; \ for( i = i3 = 0; i < v_##_Tpvec::nlanes; i++, i3 += 3 ) \ { \ @@ -2196,10 +2104,10 @@ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, v_ inline void v_load_deinterleave(const _Tp* ptr, v_##_Tpvec& a, v_##_Tpvec& b, \ v_##_Tpvec& c, v_##_Tpvec& d) \ { \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrd[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrd[v_##_Tpvec::nlanes] = {0}; \ int i, i4; \ for( i = i4 = 0; i < v_##_Tpvec::nlanes; i++, i4 += 4 ) \ { \ @@ -2217,8 +2125,8 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ { \ int i, i2; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ for( i = i2 = 0; i < v_##_Tpvec::nlanes; i++, i2 += 2 ) \ @@ -2231,9 +2139,9 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& const v_##_Tpvec& c, hal::StoreMode /*mode*/=hal::STORE_UNALIGNED) \ { \ int i, i3; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ v_store(ptrc, c); \ @@ -2249,10 +2157,10 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& hal::StoreMode /*mode*/=hal::STORE_UNALIGNED ) \ { \ int i, i4; \ - _Tp CV_DECL_ALIGNED(32) ptra[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrb[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrc[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrd[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptra[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrb[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrc[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrd[v_##_Tpvec::nlanes] = {0}; \ v_store(ptra, a); \ v_store(ptrb, b); \ v_store(ptrc, c); \ @@ -2267,8 +2175,8 @@ inline void v_store_interleave( _Tp* ptr, const v_##_Tpvec& a, const v_##_Tpvec& } \ inline v_##_Tpvec v_interleave_pairs(const v_##_Tpvec& vec) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptr[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrvec[v_##_Tpvec::nlanes] = {0}; \ v_store(ptrvec, vec); \ for (int i = 0; i < v_##_Tpvec::nlanes/4; i++) \ { \ @@ -2281,8 +2189,8 @@ inline v_##_Tpvec v_interleave_pairs(const v_##_Tpvec& vec) \ } \ inline v_##_Tpvec v_interleave_quads(const v_##_Tpvec& vec) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[v_##_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptr[v_##_Tpvec::nlanes] = {0}; \ + _Tp ptrvec[v_##_Tpvec::nlanes] = {0}; \ v_store(ptrvec, vec); \ for (int i = 0; i < v_##_Tpvec::nlanes/8; i++) \ { \ @@ -2298,17 +2206,17 @@ inline v_##_Tpvec v_interleave_quads(const v_##_Tpvec& vec) \ return v_load(ptr); \ } -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint8x16, uchar, u8, 8) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int8x16, schar, i8, 8) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint16x8, ushort, u16, 16) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int16x8, short, i16, 16) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint32x4, unsigned, u32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int32x4, int, i32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(float32x4, float, f32, 32) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint64x2, uint64, u64, 64) -OPENCV_HAL_IMPL_RVV_INTERLEAVED(int64x2, int64, i64, 64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint8x16, uchar) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int8x16, schar) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint16x8, ushort) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int16x8, short) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint32x4, unsigned) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int32x4, int) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(float32x4, float) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(uint64x2, uint64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(int64x2, int64) #if CV_SIMD128_64F -OPENCV_HAL_IMPL_RVV_INTERLEAVED(float64x2, double, f64, 64) +OPENCV_HAL_IMPL_RVV_INTERLEAVED(float64x2, double) #endif //////////// PopCount //////////// @@ -2336,9 +2244,9 @@ static const unsigned char popCountTable[] = #define OPENCV_HAL_IMPL_RVV_POPCOUNT_OP(_rTpvec, _Tpvec, _rTp, _Tp, suffix) \ inline _rTpvec v_popcount(const _Tpvec& a) \ { \ - uchar CV_DECL_ALIGNED(32) ptra[16] = {0}; \ + uchar ptra[16] = {0}; \ v_store(ptra, v_reinterpret_as_u8(a)); \ - _rTp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ + _rTp ptr[_Tpvec::nlanes] = {0}; \ v_store(ptr, v_setzero_##suffix()); \ for (int i = 0; i < _Tpvec::nlanes*(int)sizeof(_Tp); i++) \ ptr[i/sizeof(_Tp)] += popCountTable[ptra[i]]; \ @@ -2356,21 +2264,20 @@ OPENCV_HAL_IMPL_RVV_POPCOUNT_OP(v_uint64x2, v_int64x2, uint64, int64, u64) //////////// SignMask //////////// -#define OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(_Tpvec, _Tp, suffix, width, shift) \ +#define OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(_Tpvec, _Tp, suffix, vl, shift) \ inline int v_signmask(const _Tpvec& a) \ { \ int mask = 0; \ - vsetvlmax_e##width##m1(); \ - _Tpvec tmp = _Tpvec(vsrl_vx_##suffix##m1(a, shift)); \ + _Tpvec tmp = _Tpvec(vsrl_vx_##suffix##m1(a, shift, vl)); \ for( int i = 0; i < _Tpvec::nlanes; i++ ) \ mask |= (int)(tmp.val[i]) << i; \ return mask; \ } -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint8x16, uchar, u8, 8, 7) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint16x8, ushort, u16, 16, 15) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint32x4, unsigned, u32, 32, 31) -OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint64x2, uint64, u64, 64, 63) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint8x16, uchar, u8, 16, 7) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint16x8, ushort, u16, 8, 15) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint32x4, unsigned, u32, 4, 31) +OPENCV_HAL_IMPL_RVV_SIGNMASK_OP(v_uint64x2, uint64, u64, 2, 63) inline int v_signmask(const v_int8x16& a) { return v_signmask(v_reinterpret_as_u8(a)); } @@ -2393,7 +2300,7 @@ inline int v_signmask(const v_float64x2& a) #define OPENCV_HAL_IMPL_RVV_SCAN_FORWOARD_OP(_Tpvec, _Tp, suffix) \ inline int v_scan_forward(const _Tpvec& a) \ { \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ + _Tp ptr[_Tpvec::nlanes] = {0}; \ v_store(ptr, v_reinterpret_as_##suffix(a)); \ for (int i = 0; i < _Tpvec::nlanes; i++) \ if(int(ptr[i]) < 0) \ @@ -2416,28 +2323,29 @@ OPENCV_HAL_IMPL_RVV_SCAN_FORWOARD_OP(v_float64x2, double, f64) //////////// Pack triplets //////////// -#define OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(_Tpvec, _Tp) \ -inline _Tpvec v_pack_triplets(const _Tpvec& vec) \ -{ \ - _Tp CV_DECL_ALIGNED(32) ptr[_Tpvec::nlanes] = {0}; \ - _Tp CV_DECL_ALIGNED(32) ptrvec[_Tpvec::nlanes] = {0}; \ - v_store(ptrvec, vec); \ - for (int i = 0; i < _Tpvec::nlanes/4; i++) \ - { \ - ptr[3*i ] = ptrvec[4*i ]; \ - ptr[3*i+1] = ptrvec[4*i+2]; \ - ptr[3*i+2] = ptrvec[4*i+2]; \ - } \ - return v_load(ptr); \ +inline v_int8x16 v_pack_triplets(const v_int8x16& vec) +{ + uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; + return v_int8x16((vint8m1_t)vrgather_vv_u8m1((vuint8m1_t)vint8m1_t(vec), (vuint8m1_t)vle64_v_u64m1(ptr, 2), 16)); +} +inline v_uint8x16 v_pack_triplets(const v_uint8x16& vec) +{ + return v_reinterpret_as_u8(v_pack_triplets(v_reinterpret_as_s8(vec))); } -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint8x16, uchar) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int8x16, schar) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint16x8, ushort) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int16x8, short) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_uint32x4, unsigned) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_int32x4, int) -OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_float32x4, float) +inline v_int16x8 v_pack_triplets(const v_int16x8& vec) +{ + uint64 ptr[2] = {0x0908060504020100, 0xFFFFFFFF0E0D0C0A}; + return v_int16x8((vint16m1_t)vrgather_vv_u8m1((vuint8m1_t)vint16m1_t(vec), (vuint8m1_t)vle64_v_u64m1(ptr, 2), 16)); +} +inline v_uint16x8 v_pack_triplets(const v_uint16x8& vec) +{ + return v_reinterpret_as_u16(v_pack_triplets(v_reinterpret_as_s16(vec))); +} + +inline v_int32x4 v_pack_triplets(const v_int32x4& vec) { return vec; } +inline v_uint32x4 v_pack_triplets(const v_uint32x4& vec) { return vec; } +inline v_float32x4 v_pack_triplets(const v_float32x4& vec) { return vec; } ////// FP16 support /////// @@ -2445,12 +2353,12 @@ OPENCV_HAL_IMPL_RVV_PACK_TRIPLETS(v_float32x4, float) #if CV_FP16 inline v_float32x4 v_load_expand(const float16_t* ptr) { - return v_float32x4(vfwcvt_f_f_v_f32m1(vle16_v_f16mf2(ptr))); + return v_float32x4(vfwcvt_f_f_v_f32m1(vle16_v_f16mf2(ptr, 4), 4)); } inline void v_pack_store(float16_t* ptr, const v_float32x4& v) { - vse16_v_f16mf2(ptr, vfncvt_f_f_w_f16mf2(v)); + vse16_v_f16mf2(ptr, vfncvt_f_f_w_f16mf2(v, 4), 4); } #else inline v_float32x4 v_load_expand(const float16_t* ptr) @@ -2474,70 +2382,61 @@ inline void v_pack_store(float16_t* ptr, const v_float32x4& v) inline v_int32x4 v_round(const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(a)); + return v_int32x4(vfcvt_x_f_v_i32m1(a, 4)); } inline v_int32x4 v_floor(const v_float32x4& a) { v_float32x4 ZP5 = v_setall_f32(0.5f); v_float32x4 t = a - ZP5; - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(t)); + return v_int32x4(vfcvt_x_f_v_i32m1(t, 4)); } inline v_int32x4 v_ceil(const v_float32x4& a) { v_float32x4 ZP5 = v_setall_f32(0.5f); v_float32x4 t = a + ZP5; - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_x_f_v_i32m1(t)); + return v_int32x4(vfcvt_x_f_v_i32m1(t, 4)); } inline v_int32x4 v_trunc(const v_float32x4& a) { - vsetvlmax_e32m1(); - return v_int32x4(vfcvt_rtz_x_f_v_i32m1(a)); + return v_int32x4(vfcvt_rtz_x_f_v_i32m1(a, 4)); } #if CV_SIMD128_64F inline v_int32x4 v_round(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_round(const v_float64x2& a, const v_float64x2& b) { double arr[4] = {a.val[0], a.val[1], b.val[0], b.val[1]}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_floor(const v_float64x2& a) { double arr[4] = {a.val[0]-0.5f, a.val[1]-0.5f, 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_ceil(const v_float64x2& a) { double arr[4] = {a.val[0]+0.5f, a.val[1]+0.5f, 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_x_f_w_i32m1(tmp, 4)); } inline v_int32x4 v_trunc(const v_float64x2& a) { double arr[4] = {a.val[0], a.val[1], 0, 0}; - vsetvlmax_e64m2(); - vfloat64m2_t tmp = vle64_v_f64m2(arr); - return v_int32x4(vfncvt_rtz_x_f_w_i32m1(tmp)); + vfloat64m2_t tmp = vle64_v_f64m2(arr, 4); + return v_int32x4(vfncvt_rtz_x_f_w_i32m1(tmp, 4)); } #endif @@ -2547,19 +2446,17 @@ inline v_int32x4 v_trunc(const v_float64x2& a) // 16 >> 32 inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; v_int32x4 t1, t2; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); return t1 + t2; } inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; + int ptr[8] = {0}; v_int32x4 t1, t2; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_load_deinterleave(ptr, t1, t2); return t1 + t2 + c; } @@ -2567,19 +2464,17 @@ inline v_int32x4 v_dotprod(const v_int16x8& a, const v_int16x8& b, const v_int32 // 32 >> 64 inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; v_int64x2 t1, t2; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); return t1 + t2; } inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; + int64 ptr[4] = {0}; v_int64x2 t1, t2; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_load_deinterleave(ptr, t1, t2); return t1 + t2 + c; } @@ -2587,40 +2482,36 @@ inline v_int64x2 v_dotprod(const v_int32x4& a, const v_int32x4& b, const v_int64 // 8 >> 32 inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } inline v_uint32x4 v_dotprod_expand(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; + unsigned ptr[16] = {0}; v_uint32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; + int ptr[16] = {0}; v_int32x4 t1, t2, t3, t4; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2628,39 +2519,35 @@ inline v_int32x4 v_dotprod_expand(const v_int8x16& a, const v_int8x16& b, // 16 >> 64 inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } inline v_uint64x2 v_dotprod_expand(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + uint64 ptr[8] = {0}; v_uint64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4; } inline v_int64x2 v_dotprod_expand(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; + int64 ptr[8] = {0}; v_int64x2 t1, t2, t3, t4; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_load_deinterleave(ptr, t1, t2, t3, t4); return t1 + t2 + t3 + t4 + c; } @@ -2679,18 +2566,16 @@ inline v_float64x2 v_dotprod_expand(const v_int32x4& a, const v_int32x4& b, // 16 >> 32 inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + int ptr[8] = {0}; + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); return t1 + t2; } inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e32m2(); - vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b)); + int ptr[8] = {0}; + vse32_v_i32m2(ptr, vwmul_vv_i32m2(a, b, 8), 8); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); return t1 + t2 + c; @@ -2699,18 +2584,16 @@ inline v_int32x4 v_dotprod_fast(const v_int16x8& a, const v_int16x8& b, const v_ // 32 >> 64 inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + int64 ptr[4] = {0}; + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); return t1 + t2; } inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[4] = {0}; - vsetvlmax_e64m2(); - vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b)); + int64 ptr[4] = {0}; + vse64_v_i64m2(ptr, vwmul_vv_i64m2(a, b, 4), 4); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); return t1 + t2 + c; @@ -2720,9 +2603,8 @@ inline v_int64x2 v_dotprod_fast(const v_int32x4& a, const v_int32x4& b, const v_ // 8 >> 32 inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + unsigned ptr[16] = {0}; + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); v_uint32x4 t3 = v_load(ptr+8); @@ -2731,9 +2613,8 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b } inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b, const v_uint32x4& c) { - unsigned CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_u32m4(ptr, vqmaccu_vv_u32m4(vzero_u32m4(), a, b)); + unsigned ptr[16] = {0}; + vse32_v_u32m4(ptr, vwcvtu_x_x_v_u32m4(vwmulu_vv_u16m2(a, b, 16), 16), 16); v_uint32x4 t1 = v_load(ptr); v_uint32x4 t2 = v_load(ptr+4); v_uint32x4 t3 = v_load(ptr+8); @@ -2742,9 +2623,8 @@ inline v_uint32x4 v_dotprod_expand_fast(const v_uint8x16& a, const v_uint8x16& b } inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + int ptr[16] = {0}; + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); v_int32x4 t3 = v_load(ptr+8); @@ -2753,9 +2633,8 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b) } inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, const v_int32x4& c) { - int CV_DECL_ALIGNED(32) ptr[16] = {0}; - vsetvlmax_e32m4(); - vse32_v_i32m4(ptr, vqmacc_vv_i32m4(vzero_i32m4(), a, b)); + int ptr[16] = {0}; + vse32_v_i32m4(ptr, vwcvt_x_x_v_i32m4(vwmul_vv_i16m2(a, b, 16), 16), 16); v_int32x4 t1 = v_load(ptr); v_int32x4 t2 = v_load(ptr+4); v_int32x4 t3 = v_load(ptr+8); @@ -2766,9 +2645,8 @@ inline v_int32x4 v_dotprod_expand_fast(const v_int8x16& a, const v_int8x16& b, c // 16 >> 64 inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + uint64 ptr[8] = {0}; + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); v_uint64x2 t3 = v_load(ptr+4); @@ -2777,9 +2655,8 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b } inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b, const v_uint64x2& c) { - uint64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_u64m4(ptr, vqmaccu_vv_u64m4(vzero_u64m4(), a, b)); + uint64 ptr[8] = {0}; + vse64_v_u64m4(ptr, vwcvtu_x_x_v_u64m4(vwmulu_vv_u32m2(a, b, 8), 8), 8); v_uint64x2 t1 = v_load(ptr); v_uint64x2 t2 = v_load(ptr+2); v_uint64x2 t3 = v_load(ptr+4); @@ -2788,9 +2665,8 @@ inline v_uint64x2 v_dotprod_expand_fast(const v_uint16x8& a, const v_uint16x8& b } inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + int64 ptr[8] = {0}; + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); v_int64x2 t3 = v_load(ptr+4); @@ -2799,9 +2675,8 @@ inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b) } inline v_int64x2 v_dotprod_expand_fast(const v_int16x8& a, const v_int16x8& b, const v_int64x2& c) { - int64 CV_DECL_ALIGNED(32) ptr[8] = {0}; - vsetvlmax_e64m4(); - vse64_v_i64m4(ptr, vqmacc_vv_i64m4(vzero_i64m4(), a, b)); + int64 ptr[8] = {0}; + vse64_v_i64m4(ptr, vwcvt_x_x_v_i64m4(vwmul_vv_i32m2(a, b, 8), 8), 8); v_int64x2 t1 = v_load(ptr); v_int64x2 t2 = v_load(ptr+2); v_int64x2 t3 = v_load(ptr+4); @@ -2822,11 +2697,10 @@ inline v_float32x4 v_matmul(const v_float32x4& v, const v_float32x4& m0, const v_float32x4& m1, const v_float32x4& m2, const v_float32x4& m3) { - vsetvlmax_e32m1(); - vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v)); - res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1); - res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2); - res = vfmacc_vf_f32m1(res, v_extract_n<3>(v), m3); + vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v), 4); + res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<3>(v), m3, 4); return v_float32x4(res); } @@ -2834,40 +2708,35 @@ inline v_float32x4 v_matmuladd(const v_float32x4& v, const v_float32x4& m0, const v_float32x4& m1, const v_float32x4& m2, const v_float32x4& a) { - vsetvlmax_e32m1(); - vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v)); - res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1); - res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2); + vfloat32m1_t res = vfmul_vf_f32m1(m0, v_extract_n<0>(v), 4); + res = vfmacc_vf_f32m1(res, v_extract_n<1>(v), m1, 4); + res = vfmacc_vf_f32m1(res, v_extract_n<2>(v), m2, 4); return v_float32x4(res) + a; } -#define OPENCV_HAL_IMPL_RVV_MUL_EXPAND(_Tpvec, _Tpwvec, _Tpw, suffix, wmul, width) \ +#define OPENCV_HAL_IMPL_RVV_MUL_EXPAND(_Tpvec, _Tpwvec, _Tpw, suffix, wmul, width, vl, hvl) \ inline void v_mul_expand(const _Tpvec& a, const _Tpvec& b, _Tpwvec& c, _Tpwvec& d) \ { \ - _Tpw CV_DECL_ALIGNED(32) ptr[_Tpwvec::nlanes*2] = {0}; \ - vsetvlmax_e##width##m2(); \ - vse##width##_v_##suffix##m2(ptr, wmul(a, b)); \ - vsetvlmax_e##width##m1(); \ - c = _Tpwvec(vle##width##_v_##suffix##m1(ptr)); \ - d = _Tpwvec(vle##width##_v_##suffix##m1(ptr+_Tpwvec::nlanes)); \ + _Tpw ptr[_Tpwvec::nlanes*2] = {0}; \ + vse##width##_v_##suffix##m2(ptr, wmul(a, b, vl), vl); \ + c = _Tpwvec(vle##width##_v_##suffix##m1(ptr, hvl)); \ + d = _Tpwvec(vle##width##_v_##suffix##m1(ptr+_Tpwvec::nlanes, hvl)); \ } -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint8x16, v_uint16x8, ushort, u16, vwmulu_vv_u16m2, 16) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int8x16, v_int16x8, short, i16, vwmul_vv_i16m2, 16) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint16x8, v_uint32x4, unsigned, u32, vwmulu_vv_u32m2, 32) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int16x8, v_int32x4, int, i32, vwmul_vv_i32m2, 32) -OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint32x4, v_uint64x2, uint64, u64, vwmulu_vv_u64m2, 64) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint8x16, v_uint16x8, ushort, u16, vwmulu_vv_u16m2, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int8x16, v_int16x8, short, i16, vwmul_vv_i16m2, 16, 16, 8) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint16x8, v_uint32x4, unsigned, u32, vwmulu_vv_u32m2, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_int16x8, v_int32x4, int, i32, vwmul_vv_i32m2, 32, 8, 4) +OPENCV_HAL_IMPL_RVV_MUL_EXPAND(v_uint32x4, v_uint64x2, uint64, u64, vwmulu_vv_u64m2, 64, 4, 2) inline v_int16x8 v_mul_hi(const v_int16x8& a, const v_int16x8& b) { - vsetvlmax_e16m1(); - return v_int16x8(vnsra_wx_i16m1(vwmul_vv_i32m2(a, b), 16)); + return v_int16x8(vnsra_wx_i16m1(vwmul_vv_i32m2(a, b, 8), 16, 8)); } inline v_uint16x8 v_mul_hi(const v_uint16x8& a, const v_uint16x8& b) { - vsetvlmax_e16m1(); - return v_uint16x8(vnsrl_wx_u16m1(vwmulu_vv_u32m2(a, b), 16)); + return v_uint16x8(vnsrl_wx_u16m1(vwmulu_vv_u32m2(a, b, 8), 16, 8)); } diff --git a/modules/core/include/opencv2/core/mat.hpp b/modules/core/include/opencv2/core/mat.hpp index 68744f970b..93a2bc1bda 100644 --- a/modules/core/include/opencv2/core/mat.hpp +++ b/modules/core/include/opencv2/core/mat.hpp @@ -1183,14 +1183,14 @@ public: The method creates a square diagonal matrix from specified main diagonal. @param d One-dimensional matrix that represents the main diagonal. */ - static Mat diag(const Mat& d); + CV_NODISCARD_STD static Mat diag(const Mat& d); /** @brief Creates a full copy of the array and the underlying data. The method creates a full copy of the array. The original step[] is not taken into account. So, the array copy is a continuous array occupying total()*elemSize() bytes. */ - Mat clone() const CV_NODISCARD; + CV_NODISCARD_STD Mat clone() const; /** @brief Copies the matrix to another one. @@ -1358,20 +1358,20 @@ public: @param cols Number of columns. @param type Created matrix type. */ - static MatExpr zeros(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr zeros(int rows, int cols, int type); /** @overload @param size Alternative to the matrix size specification Size(cols, rows) . @param type Created matrix type. */ - static MatExpr zeros(Size size, int type); + CV_NODISCARD_STD static MatExpr zeros(Size size, int type); /** @overload @param ndims Array dimensionality. @param sz Array of integers specifying the array shape. @param type Created matrix type. */ - static MatExpr zeros(int ndims, const int* sz, int type); + CV_NODISCARD_STD static MatExpr zeros(int ndims, const int* sz, int type); /** @brief Returns an array of all 1's of the specified size and type. @@ -1389,20 +1389,20 @@ public: @param cols Number of columns. @param type Created matrix type. */ - static MatExpr ones(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr ones(int rows, int cols, int type); /** @overload @param size Alternative to the matrix size specification Size(cols, rows) . @param type Created matrix type. */ - static MatExpr ones(Size size, int type); + CV_NODISCARD_STD static MatExpr ones(Size size, int type); /** @overload @param ndims Array dimensionality. @param sz Array of integers specifying the array shape. @param type Created matrix type. */ - static MatExpr ones(int ndims, const int* sz, int type); + CV_NODISCARD_STD static MatExpr ones(int ndims, const int* sz, int type); /** @brief Returns an identity matrix of the specified size and type. @@ -1418,13 +1418,13 @@ public: @param cols Number of columns. @param type Created matrix type. */ - static MatExpr eye(int rows, int cols, int type); + CV_NODISCARD_STD static MatExpr eye(int rows, int cols, int type); /** @overload @param size Alternative matrix size specification as Size(cols, rows) . @param type Created matrix type. */ - static MatExpr eye(Size size, int type); + CV_NODISCARD_STD static MatExpr eye(Size size, int type); /** @brief Allocates new array data if needed. @@ -2287,7 +2287,7 @@ public: Mat_ row(int y) const; Mat_ col(int x) const; Mat_ diag(int d=0) const; - Mat_ clone() const CV_NODISCARD; + CV_NODISCARD_STD Mat_ clone() const; //! overridden forms of Mat::elemSize() etc. size_t elemSize() const; @@ -2300,14 +2300,14 @@ public: size_t stepT(int i=0) const; //! overridden forms of Mat::zeros() etc. Data type is omitted, of course - static MatExpr zeros(int rows, int cols); - static MatExpr zeros(Size size); - static MatExpr zeros(int _ndims, const int* _sizes); - static MatExpr ones(int rows, int cols); - static MatExpr ones(Size size); - static MatExpr ones(int _ndims, const int* _sizes); - static MatExpr eye(int rows, int cols); - static MatExpr eye(Size size); + CV_NODISCARD_STD static MatExpr zeros(int rows, int cols); + CV_NODISCARD_STD static MatExpr zeros(Size size); + CV_NODISCARD_STD static MatExpr zeros(int _ndims, const int* _sizes); + CV_NODISCARD_STD static MatExpr ones(int rows, int cols); + CV_NODISCARD_STD static MatExpr ones(Size size); + CV_NODISCARD_STD static MatExpr ones(int _ndims, const int* _sizes); + CV_NODISCARD_STD static MatExpr eye(int rows, int cols); + CV_NODISCARD_STD static MatExpr eye(Size size); //! some more overridden methods Mat_& adjustROI( int dtop, int dbottom, int dleft, int dright ); @@ -2450,10 +2450,11 @@ public: //! <0 - a diagonal from the lower half) UMat diag(int d=0) const; //! constructs a square diagonal matrix which main diagonal is vector "d" - static UMat diag(const UMat& d); + CV_NODISCARD_STD static UMat diag(const UMat& d, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat diag(const UMat& d) { return diag(d, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload //! returns deep copy of the matrix, i.e. the data is copied - UMat clone() const CV_NODISCARD; + CV_NODISCARD_STD UMat clone() const; //! copies the matrix content to "m". // It calls m.create(this->size(), this->type()). void copyTo( OutputArray m ) const; @@ -2484,14 +2485,22 @@ public: double dot(InputArray m) const; //! Matlab-style matrix initialization - static UMat zeros(int rows, int cols, int type); - static UMat zeros(Size size, int type); - static UMat zeros(int ndims, const int* sz, int type); - static UMat ones(int rows, int cols, int type); - static UMat ones(Size size, int type); - static UMat ones(int ndims, const int* sz, int type); - static UMat eye(int rows, int cols, int type); - static UMat eye(Size size, int type); + CV_NODISCARD_STD static UMat zeros(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat zeros(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat zeros(int ndims, const int* sz, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat zeros(int rows, int cols, int type) { return zeros(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + CV_NODISCARD_STD static UMat zeros(Size size, int type) { return zeros(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + CV_NODISCARD_STD static UMat zeros(int ndims, const int* sz, int type) { return zeros(ndims, sz, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + CV_NODISCARD_STD static UMat ones(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat ones(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat ones(int ndims, const int* sz, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat ones(int rows, int cols, int type) { return ones(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + CV_NODISCARD_STD static UMat ones(Size size, int type) { return ones(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + CV_NODISCARD_STD static UMat ones(int ndims, const int* sz, int type) { return ones(ndims, sz, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + CV_NODISCARD_STD static UMat eye(int rows, int cols, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat eye(Size size, int type, UMatUsageFlags usageFlags /*= USAGE_DEFAULT*/); + CV_NODISCARD_STD static UMat eye(int rows, int cols, int type) { return eye(rows, cols, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload + CV_NODISCARD_STD static UMat eye(Size size, int type) { return eye(size, type, USAGE_DEFAULT); } // OpenCV 5.0: remove abi compatibility overload //! allocates new matrix data unless the matrix already has specified size and type. // previous data is unreferenced if needed. @@ -2757,7 +2766,7 @@ public: SparseMat& operator = (const Mat& m); //! creates full copy of the matrix - SparseMat clone() const CV_NODISCARD; + CV_NODISCARD_STD SparseMat clone() const; //! copies all the data to the destination matrix. All the previous content of m is erased void copyTo( SparseMat& m ) const; @@ -2994,7 +3003,7 @@ public: SparseMat_& operator = (const Mat& m); //! makes full copy of the matrix. All the elements are duplicated - SparseMat_ clone() const CV_NODISCARD; + CV_NODISCARD_STD SparseMat_ clone() const; //! equivalent to cv::SparseMat::create(dims, _sizes, DataType<_Tp>::type) void create(int dims, const int* _sizes); //! converts sparse matrix to the old-style CvSparseMat. All the elements are copied diff --git a/modules/core/include/opencv2/core/matx.hpp b/modules/core/include/opencv2/core/matx.hpp index 4d7c82f2c7..9189a08a14 100644 --- a/modules/core/include/opencv2/core/matx.hpp +++ b/modules/core/include/opencv2/core/matx.hpp @@ -148,22 +148,22 @@ public: Matx(std::initializer_list<_Tp>); //!< initialize from an initializer list - static Matx all(_Tp alpha); - static Matx zeros(); - static Matx ones(); - static Matx eye(); - static Matx diag(const diag_type& d); + CV_NODISCARD_STD static Matx all(_Tp alpha); + CV_NODISCARD_STD static Matx zeros(); + CV_NODISCARD_STD static Matx ones(); + CV_NODISCARD_STD static Matx eye(); + CV_NODISCARD_STD static Matx diag(const diag_type& d); /** @brief Generates uniformly distributed random numbers @param a Range boundary. @param b The other range boundary (boundaries don't have to be ordered, the lower boundary is inclusive, the upper one is exclusive). */ - static Matx randu(_Tp a, _Tp b); + CV_NODISCARD_STD static Matx randu(_Tp a, _Tp b); /** @brief Generates normally distributed random numbers @param a Mean value. @param b Standard deviation. */ - static Matx randn(_Tp a, _Tp b); + CV_NODISCARD_STD static Matx randn(_Tp a, _Tp b); //! dot product computed with the default precision _Tp dot(const Matx<_Tp, m, n>& v) const; diff --git a/modules/core/include/opencv2/core/ocl.hpp b/modules/core/include/opencv2/core/ocl.hpp index f9cc9e019a..4503fa00dd 100644 --- a/modules/core/include/opencv2/core/ocl.hpp +++ b/modules/core/include/opencv2/core/ocl.hpp @@ -235,7 +235,11 @@ public: /** * @param d OpenCL handle (cl_device_id). clRetainDevice() is called on success. - */ + * + * @note Ownership of the passed device is passed to OpenCV on success. + * The caller should additionally call `clRetainDevice` on it if it intends + * to continue using the device. + */ static Device fromHandle(void* d); struct Impl; @@ -495,8 +499,8 @@ public: template inline Kernel& args(const _Tps&... kernel_args) { set_args_(0, kernel_args...); return *this; } + /** @brief Run the OpenCL kernel (globalsize value may be adjusted) - /** @brief Run the OpenCL kernel. @param dims the work problem dimensions. It is the length of globalsize and localsize. It can be either 1, 2 or 3. @param globalsize work items for each dimension. It is not the final globalsize passed to OpenCL. Each dimension will be adjusted to the nearest integer divisible by the corresponding @@ -505,12 +509,26 @@ public: @param localsize work-group size for each dimension. @param sync specify whether to wait for OpenCL computation to finish before return. @param q command queue + + @note Use run_() if your kernel code doesn't support adjusted globalsize. */ bool run(int dims, size_t globalsize[], size_t localsize[], bool sync, const Queue& q=Queue()); + + /** @brief Run the OpenCL kernel + * + * @param dims the work problem dimensions. It is the length of globalsize and localsize. It can be either 1, 2 or 3. + * @param globalsize work items for each dimension. This value is passed to OpenCL without changes. + * @param localsize work-group size for each dimension. + * @param sync specify whether to wait for OpenCL computation to finish before return. + * @param q command queue + */ + bool run_(int dims, size_t globalsize[], size_t localsize[], bool sync, const Queue& q=Queue()); + bool runTask(bool sync, const Queue& q=Queue()); - /** @brief Similar to synchronized run() call with returning of kernel execution time + /** @brief Similar to synchronized run_() call with returning of kernel execution time + * * Separate OpenCL command queue may be used (with CL_QUEUE_PROFILING_ENABLE) * @return Execution time in nanoseconds or negative number on error */ @@ -826,11 +844,13 @@ public: OpenCLExecutionContext cloneWithNewQueue() const; /** @brief Creates OpenCL execution context - * OpenCV will check if available OpenCL platform has platformName name, then assign context to - * OpenCV and call `clRetainContext` function. The deviceID device will be used as target device and - * new command queue will be created. + * OpenCV will check if available OpenCL platform has platformName name, + * then assign context to OpenCV. + * The deviceID device will be used as target device and a new command queue will be created. * - * @note Lifetime of passed handles is transferred to OpenCV wrappers on success + * @note On success, ownership of one reference of the context and device is taken. + * The caller should additionally call `clRetainContext` and/or `clRetainDevice` + * to increase the reference count if it wishes to continue using them. * * @param platformName name of OpenCL platform to attach, this string is used to check if platform is available to OpenCV at runtime * @param platformID ID of platform attached context was created for (cl_platform_id) diff --git a/modules/core/include/opencv2/core/opencl/opencl_info.hpp b/modules/core/include/opencv2/core/opencl/opencl_info.hpp index 5e5c846ad0..3ead76e5c4 100644 --- a/modules/core/include/opencv2/core/opencl/opencl_info.hpp +++ b/modules/core/include/opencv2/core/opencl/opencl_info.hpp @@ -144,6 +144,10 @@ static void dumpOpenCLInformation() DUMP_MESSAGE_STDOUT(" Double support = " << doubleSupportStr); DUMP_CONFIG_PROPERTY("cv_ocl_current_haveDoubleSupport", device.doubleFPConfig() > 0); + const char* halfSupportStr = device.halfFPConfig() > 0 ? "Yes" : "No"; + DUMP_MESSAGE_STDOUT(" Half support = " << halfSupportStr); + DUMP_CONFIG_PROPERTY("cv_ocl_current_haveHalfSupport", device.halfFPConfig() > 0); + const char* isUnifiedMemoryStr = device.hostUnifiedMemory() ? "Yes" : "No"; DUMP_MESSAGE_STDOUT(" Host unified memory = " << isUnifiedMemoryStr); DUMP_CONFIG_PROPERTY("cv_ocl_current_hostUnifiedMemory", device.hostUnifiedMemory()); @@ -191,6 +195,9 @@ static void dumpOpenCLInformation() DUMP_MESSAGE_STDOUT(" Preferred vector width double = " << device.preferredVectorWidthDouble()); DUMP_CONFIG_PROPERTY("cv_ocl_current_preferredVectorWidthDouble", device.preferredVectorWidthDouble()); + + DUMP_MESSAGE_STDOUT(" Preferred vector width half = " << device.preferredVectorWidthHalf()); + DUMP_CONFIG_PROPERTY("cv_ocl_current_preferredVectorWidthHalf", device.preferredVectorWidthHalf()); } catch (...) { diff --git a/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp b/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp index 264def5f50..04b0c4c6cb 100644 --- a/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp +++ b/modules/core/include/opencv2/core/parallel/backend/parallel_for.tbb.hpp @@ -38,7 +38,7 @@ static tbb::task_scheduler_init& getScheduler() } #endif -/** OpenMP parallel_for API implementation +/** TBB parallel_for API implementation * * @sa setParallelForBackend * @ingroup core_parallel_backend diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 7cbe0ea9fb..2c61872ba1 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -162,13 +162,23 @@ public: //! default constructor Point_(); Point_(_Tp _x, _Tp _y); +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 Point_(const Point_& pt); - Point_(Point_&& pt) CV_NOEXCEPT; + Point_(Point_&& pt) CV_NOEXCEPT = default; +#elif OPENCV_ABI_COMPATIBILITY < 500 + Point_(const Point_& pt) = default; + Point_(Point_&& pt) CV_NOEXCEPT = default; +#endif Point_(const Size_<_Tp>& sz); Point_(const Vec<_Tp, 2>& v); +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 Point_& operator = (const Point_& pt); - Point_& operator = (Point_&& pt) CV_NOEXCEPT; + Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; +#elif OPENCV_ABI_COMPATIBILITY < 500 + Point_& operator = (const Point_& pt) = default; + Point_& operator = (Point_&& pt) CV_NOEXCEPT = default; +#endif //! conversion to another data type template operator Point_<_Tp2>() const; @@ -244,13 +254,17 @@ public: //! default constructor Point3_(); Point3_(_Tp _x, _Tp _y, _Tp _z); - Point3_(const Point3_& pt); - Point3_(Point3_&& pt) CV_NOEXCEPT; +#if OPENCV_ABI_COMPATIBILITY < 500 + Point3_(const Point3_& pt) = default; + Point3_(Point3_&& pt) CV_NOEXCEPT = default; +#endif explicit Point3_(const Point_<_Tp>& pt); Point3_(const Vec<_Tp, 3>& v); - Point3_& operator = (const Point3_& pt); - Point3_& operator = (Point3_&& pt) CV_NOEXCEPT; +#if OPENCV_ABI_COMPATIBILITY < 500 + Point3_& operator = (const Point3_& pt) = default; + Point3_& operator = (Point3_&& pt) CV_NOEXCEPT = default; +#endif //! conversion to another data type template operator Point3_<_Tp2>() const; //! conversion to cv::Vec<> @@ -320,12 +334,16 @@ public: //! default constructor Size_(); Size_(_Tp _width, _Tp _height); - Size_(const Size_& sz); - Size_(Size_&& sz) CV_NOEXCEPT; +#if OPENCV_ABI_COMPATIBILITY < 500 + Size_(const Size_& sz) = default; + Size_(Size_&& sz) CV_NOEXCEPT = default; +#endif Size_(const Point_<_Tp>& pt); - Size_& operator = (const Size_& sz); - Size_& operator = (Size_&& sz) CV_NOEXCEPT; +#if OPENCV_ABI_COMPATIBILITY < 500 + Size_& operator = (const Size_& sz) = default; + Size_& operator = (Size_&& sz) CV_NOEXCEPT = default; +#endif //! the area (width*height) _Tp area() const; //! aspect ratio (width/height) @@ -425,13 +443,17 @@ public: //! default constructor Rect_(); Rect_(_Tp _x, _Tp _y, _Tp _width, _Tp _height); - Rect_(const Rect_& r); - Rect_(Rect_&& r) CV_NOEXCEPT; +#if OPENCV_ABI_COMPATIBILITY < 500 + Rect_(const Rect_& r) = default; + Rect_(Rect_&& r) CV_NOEXCEPT = default; +#endif Rect_(const Point_<_Tp>& org, const Size_<_Tp>& sz); Rect_(const Point_<_Tp>& pt1, const Point_<_Tp>& pt2); - Rect_& operator = ( const Rect_& r ); - Rect_& operator = ( Rect_&& r ) CV_NOEXCEPT; +#if OPENCV_ABI_COMPATIBILITY < 500 + Rect_& operator = (const Rect_& r) = default; + Rect_& operator = (Rect_&& r) CV_NOEXCEPT = default; +#endif //! the top-left corner Point_<_Tp> tl() const; //! the bottom-right corner @@ -1165,13 +1187,11 @@ template inline Point_<_Tp>::Point_(_Tp _x, _Tp _y) : x(_x), y(_y) {} +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 template inline Point_<_Tp>::Point_(const Point_& pt) : x(pt.x), y(pt.y) {} - -template inline -Point_<_Tp>::Point_(Point_&& pt) CV_NOEXCEPT - : x(std::move(pt.x)), y(std::move(pt.y)) {} +#endif template inline Point_<_Tp>::Point_(const Size_<_Tp>& sz) @@ -1181,19 +1201,14 @@ template inline Point_<_Tp>::Point_(const Vec<_Tp,2>& v) : x(v[0]), y(v[1]) {} +#if (defined(__GNUC__) && __GNUC__ < 5) // GCC 4.x bug. Details: https://github.com/opencv/opencv/pull/20837 template inline Point_<_Tp>& Point_<_Tp>::operator = (const Point_& pt) { x = pt.x; y = pt.y; return *this; } - -template inline -Point_<_Tp>& Point_<_Tp>::operator = (Point_&& pt) CV_NOEXCEPT -{ - x = std::move(pt.x); y = std::move(pt.y); - return *this; -} +#endif template template inline Point_<_Tp>::operator Point_<_Tp2>() const @@ -1432,14 +1447,6 @@ template inline Point3_<_Tp>::Point3_(_Tp _x, _Tp _y, _Tp _z) : x(_x), y(_y), z(_z) {} -template inline -Point3_<_Tp>::Point3_(const Point3_& pt) - : x(pt.x), y(pt.y), z(pt.z) {} - -template inline -Point3_<_Tp>::Point3_(Point3_&& pt) CV_NOEXCEPT - : x(std::move(pt.x)), y(std::move(pt.y)), z(std::move(pt.z)) {} - template inline Point3_<_Tp>::Point3_(const Point_<_Tp>& pt) : x(pt.x), y(pt.y), z(_Tp()) {} @@ -1460,20 +1467,6 @@ Point3_<_Tp>::operator Vec<_Tp, 3>() const return Vec<_Tp, 3>(x, y, z); } -template inline -Point3_<_Tp>& Point3_<_Tp>::operator = (const Point3_& pt) -{ - x = pt.x; y = pt.y; z = pt.z; - return *this; -} - -template inline -Point3_<_Tp>& Point3_<_Tp>::operator = (Point3_&& pt) CV_NOEXCEPT -{ - x = std::move(pt.x); y = std::move(pt.y); z = std::move(pt.z); - return *this; -} - template inline _Tp Point3_<_Tp>::dot(const Point3_& pt) const { @@ -1686,14 +1679,6 @@ template inline Size_<_Tp>::Size_(_Tp _width, _Tp _height) : width(_width), height(_height) {} -template inline -Size_<_Tp>::Size_(const Size_& sz) - : width(sz.width), height(sz.height) {} - -template inline -Size_<_Tp>::Size_(Size_&& sz) CV_NOEXCEPT - : width(std::move(sz.width)), height(std::move(sz.height)) {} - template inline Size_<_Tp>::Size_(const Point_<_Tp>& pt) : width(pt.x), height(pt.y) {} @@ -1704,20 +1689,6 @@ Size_<_Tp>::operator Size_<_Tp2>() const return Size_<_Tp2>(saturate_cast<_Tp2>(width), saturate_cast<_Tp2>(height)); } -template inline -Size_<_Tp>& Size_<_Tp>::operator = (const Size_<_Tp>& sz) -{ - width = sz.width; height = sz.height; - return *this; -} - -template inline -Size_<_Tp>& Size_<_Tp>::operator = (Size_<_Tp>&& sz) CV_NOEXCEPT -{ - width = std::move(sz.width); height = std::move(sz.height); - return *this; -} - template inline _Tp Size_<_Tp>::area() const { @@ -1828,14 +1799,6 @@ template inline Rect_<_Tp>::Rect_(_Tp _x, _Tp _y, _Tp _width, _Tp _height) : x(_x), y(_y), width(_width), height(_height) {} -template inline -Rect_<_Tp>::Rect_(const Rect_<_Tp>& r) - : x(r.x), y(r.y), width(r.width), height(r.height) {} - -template inline -Rect_<_Tp>::Rect_(Rect_<_Tp>&& r) CV_NOEXCEPT - : x(std::move(r.x)), y(std::move(r.y)), width(std::move(r.width)), height(std::move(r.height)) {} - template inline Rect_<_Tp>::Rect_(const Point_<_Tp>& org, const Size_<_Tp>& sz) : x(org.x), y(org.y), width(sz.width), height(sz.height) {} @@ -1849,26 +1812,6 @@ Rect_<_Tp>::Rect_(const Point_<_Tp>& pt1, const Point_<_Tp>& pt2) height = std::max(pt1.y, pt2.y) - y; } -template inline -Rect_<_Tp>& Rect_<_Tp>::operator = ( const Rect_<_Tp>& r ) -{ - x = r.x; - y = r.y; - width = r.width; - height = r.height; - return *this; -} - -template inline -Rect_<_Tp>& Rect_<_Tp>::operator = ( Rect_<_Tp>&& r ) CV_NOEXCEPT -{ - x = std::move(r.x); - y = std::move(r.y); - width = std::move(r.width); - height = std::move(r.height); - return *this; -} - template inline Point_<_Tp> Rect_<_Tp>::tl() const { diff --git a/modules/core/include/opencv2/core/utility.hpp b/modules/core/include/opencv2/core/utility.hpp index f0368027aa..108c0d93e7 100644 --- a/modules/core/include/opencv2/core/utility.hpp +++ b/modules/core/include/opencv2/core/utility.hpp @@ -714,9 +714,27 @@ void Mat::forEach_impl(const Functor& operation) { /////////////////////////// Synchronization Primitives /////////////////////////////// #if !defined(_M_CEE) +#ifndef OPENCV_DISABLE_THREAD_SUPPORT typedef std::recursive_mutex Mutex; typedef std::lock_guard AutoLock; -#endif +#else // OPENCV_DISABLE_THREAD_SUPPORT +// Custom (failing) implementation of `std::recursive_mutex`. +struct Mutex { + void lock(){ + CV_Error(cv::Error::StsNotImplemented, + "cv::Mutex is disabled by OPENCV_DISABLE_THREAD_SUPPORT=ON"); + } + void unlock(){ + CV_Error(cv::Error::StsNotImplemented, + "cv::Mutex is disabled by OPENCV_DISABLE_THREAD_SUPPORT=ON"); + } +}; +// Stub for cv::AutoLock when threads are disabled. +struct AutoLock { + AutoLock(Mutex &) { } +}; +#endif // OPENCV_DISABLE_THREAD_SUPPORT +#endif // !defined(_M_CEE) /** @brief Designed for command line parsing diff --git a/modules/core/include/opencv2/core/utils/filesystem.private.hpp b/modules/core/include/opencv2/core/utils/filesystem.private.hpp index ea2591c9de..72b2bb9479 100644 --- a/modules/core/include/opencv2/core/utils/filesystem.private.hpp +++ b/modules/core/include/opencv2/core/utils/filesystem.private.hpp @@ -16,8 +16,8 @@ # define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 # elif defined(__APPLE__) # include -# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX) || (!defined(TARGET_OS_OSX) && !TARGET_OS_IPHONE) -# define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 // OSX only +# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX) || (defined(TARGET_OS_IOS) && TARGET_OS_IOS) +# define OPENCV_HAVE_FILESYSTEM_SUPPORT 1 // OSX, iOS only # endif # else /* unknown */ diff --git a/modules/core/misc/java/src/java/core+Mat.java b/modules/core/misc/java/src/java/core+Mat.java index 5fcc727738..2d9ee314a3 100644 --- a/modules/core/misc/java/src/java/core+Mat.java +++ b/modules/core/misc/java/src/java/core+Mat.java @@ -466,16 +466,32 @@ public class Mat { // C++: Mat Mat::mul(Mat m, double scale = 1) // - // javadoc: Mat::mul(m, scale) + /** + * Element-wise multiplication with scale factor + * @param m operand with with which to perform element-wise multiplication + * @param scale scale factor + */ public Mat mul(Mat m, double scale) { return new Mat(n_mul(nativeObj, m.nativeObj, scale)); } - // javadoc: Mat::mul(m) + /** + * Element-wise multiplication + * @param m operand with with which to perform element-wise multiplication + */ public Mat mul(Mat m) { return new Mat(n_mul(nativeObj, m.nativeObj)); } + /** + * Matrix multiplication + * @param m operand with with which to perform matrix multiplication + * @see Core#gemm(Mat, Mat, double, Mat, double, Mat, int) + */ + public Mat matMul(Mat m) { + return new Mat(n_matMul(nativeObj, m.nativeObj)); + } + // // C++: static Mat Mat::ones(int rows, int cols, int type) // @@ -1732,6 +1748,8 @@ public class Mat { private static native long n_mul(long nativeObj, long m_nativeObj); + private static native long n_matMul(long nativeObj, long m_nativeObj); + // C++: static Mat Mat::ones(int rows, int cols, int type) private static native long n_ones(int rows, int cols, int type); diff --git a/modules/core/misc/java/src/java/core+MatAt.kt b/modules/core/misc/java/src/java/core+MatAt.kt index f48a3deaed..c81e21057f 100644 --- a/modules/core/misc/java/src/java/core+MatAt.kt +++ b/modules/core/misc/java/src/java/core+MatAt.kt @@ -3,6 +3,16 @@ package org.opencv.core import org.opencv.core.Mat.* import java.lang.RuntimeException +fun Mat.get(row: Int, col: Int, data: UByteArray) = this.get(row, col, data.asByteArray()) +fun Mat.get(indices: IntArray, data: UByteArray) = this.get(indices, data.asByteArray()) +fun Mat.put(row: Int, col: Int, data: UByteArray) = this.put(row, col, data.asByteArray()) +fun Mat.put(indices: IntArray, data: UByteArray) = this.put(indices, data.asByteArray()) + +fun Mat.get(row: Int, col: Int, data: UShortArray) = this.get(row, col, data.asShortArray()) +fun Mat.get(indices: IntArray, data: UShortArray) = this.get(indices, data.asShortArray()) +fun Mat.put(row: Int, col: Int, data: UShortArray) = this.put(row, col, data.asShortArray()) +fun Mat.put(indices: IntArray, data: UShortArray) = this.put(indices, data.asShortArray()) + /*** * Example use: * @@ -19,6 +29,7 @@ inline fun Mat.at(row: Int, col: Int) : Atable = col ) UByte::class -> AtableUByte(this, row, col) as Atable + UShort::class -> AtableUShort(this, row, col) as Atable else -> throw RuntimeException("Unsupported class type") } @@ -30,6 +41,7 @@ inline fun Mat.at(idx: IntArray) : Atable = idx ) UByte::class -> AtableUByte(this, idx) as Atable + UShort::class -> AtableUShort(this, idx) as Atable else -> throw RuntimeException("Unsupported class type") } @@ -38,46 +50,95 @@ class AtableUByte(val mat: Mat, val indices: IntArray): Atable { constructor(mat: Mat, row: Int, col: Int) : this(mat, intArrayOf(row, col)) override fun getV(): UByte { - val data = ByteArray(1) - mat[indices, data] - return data[0].toUByte() + val data = UByteArray(1) + mat.get(indices, data) + return data[0] } override fun setV(v: UByte) { - val data = byteArrayOf(v.toByte()) + val data = ubyteArrayOf(v) mat.put(indices, data) } override fun getV2c(): Tuple2 { - val data = ByteArray(2) - mat[indices, data] - return Tuple2(data[0].toUByte(), data[1].toUByte()) + val data = UByteArray(2) + mat.get(indices, data) + return Tuple2(data[0], data[1]) } override fun setV2c(v: Tuple2) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte()) + val data = ubyteArrayOf(v._0, v._1) mat.put(indices, data) } override fun getV3c(): Tuple3 { - val data = ByteArray(3) - mat[indices, data] - return Tuple3(data[0].toUByte(), data[1].toUByte(), data[2].toUByte()) + val data = UByteArray(3) + mat.get(indices, data) + return Tuple3(data[0], data[1], data[2]) } override fun setV3c(v: Tuple3) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte(), v._2.toByte()) + val data = ubyteArrayOf(v._0, v._1, v._2) mat.put(indices, data) } override fun getV4c(): Tuple4 { - val data = ByteArray(4) - mat[indices, data] - return Tuple4(data[0].toUByte(), data[1].toUByte(), data[2].toUByte(), data[3].toUByte()) + val data = UByteArray(4) + mat.get(indices, data) + return Tuple4(data[0], data[1], data[2], data[3]) } override fun setV4c(v: Tuple4) { - val data = byteArrayOf(v._0.toByte(), v._1.toByte(), v._2.toByte(), v._3.toByte()) + val data = ubyteArrayOf(v._0, v._1, v._2, v._3) + mat.put(indices, data) + } +} + +class AtableUShort(val mat: Mat, val indices: IntArray): Atable { + + constructor(mat: Mat, row: Int, col: Int) : this(mat, intArrayOf(row, col)) + + override fun getV(): UShort { + val data = UShortArray(1) + mat.get(indices, data) + return data[0] + } + + override fun setV(v: UShort) { + val data = ushortArrayOf(v) + mat.put(indices, data) + } + + override fun getV2c(): Tuple2 { + val data = UShortArray(2) + mat.get(indices, data) + return Tuple2(data[0], data[1]) + } + + override fun setV2c(v: Tuple2) { + val data = ushortArrayOf(v._0, v._1) + mat.put(indices, data) + } + + override fun getV3c(): Tuple3 { + val data = UShortArray(3) + mat.get(indices, data) + return Tuple3(data[0], data[1], data[2]) + } + + override fun setV3c(v: Tuple3) { + val data = ushortArrayOf(v._0, v._1, v._2) + mat.put(indices, data) + } + + override fun getV4c(): Tuple4 { + val data = UShortArray(4) + mat.get(indices, data) + return Tuple4(data[0], data[1], data[2], data[3]) + } + + override fun setV4c(v: Tuple4) { + val data = ushortArrayOf(v._0, v._1, v._2, v._3) mat.put(indices, data) } } diff --git a/modules/core/misc/java/src/java/core+MatMatMul.kt b/modules/core/misc/java/src/java/core+MatMatMul.kt new file mode 100644 index 0000000000..3dcb6bde79 --- /dev/null +++ b/modules/core/misc/java/src/java/core+MatMatMul.kt @@ -0,0 +1,3 @@ +package org.opencv.core + +operator fun Mat.times(other: Mat): Mat = this.matMul(other) diff --git a/modules/core/misc/java/test/MatTest.java b/modules/core/misc/java/test/MatTest.java index 3075dba16b..323440b2eb 100644 --- a/modules/core/misc/java/test/MatTest.java +++ b/modules/core/misc/java/test/MatTest.java @@ -686,6 +686,16 @@ public class MatTest extends OpenCVTestCase { assertMatEqual(truth, dst, EPS); } + public void testMatMulMat() { + Mat m1 = new Mat(2, 2, CvType.CV_32F, new Scalar(2)); + Mat m2 = new Mat(2, 2, CvType.CV_32F, new Scalar(3)); + + dst = m1.matMul(m2); + + truth = new Mat(2, 2, CvType.CV_32F, new Scalar(12)); + assertMatEqual(truth, dst, EPS); + } + public void testOnesIntIntInt() { dst = Mat.ones(matSize, matSize, CvType.CV_32F); diff --git a/modules/core/misc/objc/common/Mat.h b/modules/core/misc/objc/common/Mat.h index fd1dce27ba..b5e868306c 100644 --- a/modules/core/misc/objc/common/Mat.h +++ b/modules/core/misc/objc/common/Mat.h @@ -114,7 +114,17 @@ CV_EXPORTS @interface Mat : NSObject - (BOOL)isSubmatrix; - (void)locateROI:(Size2i*)wholeSize ofs:(Point2i*)offset NS_SWIFT_NAME(locateROI(wholeSize:offset:)); - (Mat*)mul:(Mat*)mat scale:(double)scale; +/** + Performs element-wise multiplication + @param mat operand with with which to perform element-wise multiplication +*/ - (Mat*)mul:(Mat*)mat; +/** + Performs matrix multiplication + @param mat operand with with which to perform matrix multiplication + @see `Core.gemm(...)` +*/ +- (Mat*)matMul:(Mat*)mat; + (Mat*)ones:(int)rows cols:(int)cols type:(int)type NS_SWIFT_NAME(ones(rows:cols:type:)); + (Mat*)ones:(Size2i*)size type:(int)type NS_SWIFT_NAME(ones(size:type:)); + (Mat*)onesEx:(NSArray*)sizes type:(int)type NS_SWIFT_NAME(ones(sizes:type:)); diff --git a/modules/core/misc/objc/common/Mat.mm b/modules/core/misc/objc/common/Mat.mm index 5d41a3622e..ab5e1de6f9 100644 --- a/modules/core/misc/objc/common/Mat.mm +++ b/modules/core/misc/objc/common/Mat.mm @@ -13,25 +13,28 @@ #import "CvType.h" #import "CVObjcUtil.h" -// return true if we have reached the final index -static bool incIdx(cv::Mat* mat, std::vector& indices) { - for (int dim = mat->dims-1; dim>=0; dim--) { - indices[dim] = (indices[dim] + 1) % mat->size[dim]; - if (indices[dim] != 0) { - return false; - } +static int idx2Offset(cv::Mat* mat, std::vector& indices) { + int offset = indices[0]; + for (int dim=1; dim < mat->dims; dim++) { + offset = offset*mat->size[dim] + indices[dim]; + } + return offset; +} + +static void offset2Idx(cv::Mat* mat, size_t offset, std::vector& indices) { + for (int dim=mat->dims-1; dim>=0; dim--) { + indices[dim] = offset % mat->size[dim]; + offset = (offset - indices[dim]) / mat->size[dim]; } - return true; } // returns true if final index was reached -static bool updateIdx(cv::Mat* mat, std::vector& indices, int inc) { - for (int index = 0; index < inc; index++) { - if (incIdx(mat, indices)) { - return true; - } - } - return false; +static bool updateIdx(cv::Mat* mat, std::vector& indices, size_t inc) { + size_t currentOffset = idx2Offset(mat, indices); + size_t newOffset = currentOffset + inc; + bool reachedEnd = newOffset>=(size_t)mat->total(); + offset2Idx(mat, reachedEnd?0:newOffset, indices); + return reachedEnd; } @implementation Mat { @@ -369,6 +372,11 @@ static bool updateIdx(cv::Mat* mat, std::vector& indices, int inc) { return [[Mat alloc] initWithNativeMat:new cv::Mat(_nativePtr->mul(*(cv::Mat*)mat.nativePtr))]; } +- (Mat*)matMul:(Mat*)mat { + cv::Mat temp = self.nativeRef * mat.nativeRef; + return [Mat fromNative:temp]; +} + + (Mat*)ones:(int)rows cols:(int)cols type:(int)type { return [[Mat alloc] initWithNativeMat:new cv::Mat(cv::Mat::ones(rows, cols, type))]; } @@ -548,7 +556,7 @@ template void putData(uchar* dataDest, int count, T (^readData)(int) if (depth == CV_8U) { putData(dest, count, ^uchar (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_8S) { - putData(dest, count, ^char (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); + putData(dest, count, ^schar (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_16U) { putData(dest, count, ^ushort (int index) { return cv::saturate_cast(data[offset + index].doubleValue);} ); } else if (depth == CV_16S) { @@ -724,22 +732,31 @@ template int getData(NSArray* indices, cv::Mat* mat, int } int arrayAvailable = count; + size_t countBytes = count * sizeof(T); + size_t remainingBytes = (size_t)(mat->total() - idx2Offset(mat, tempIndices))*mat->elemSize(); + countBytes = (countBytes>remainingBytes)?remainingBytes:countBytes; + int result = (int)countBytes; int matAvailable = getMatAvailable(mat, tempIndices); int available = MIN(arrayAvailable, matAvailable); - int result = (int)(available * mat->elemSize() / mat->channels()); if (mat->isContinuous()) { memcpy(tBuffer, mat->ptr(tempIndices.data()), available * sizeof(T)); } else { - int copyOffset = 0; - int copyCount = MIN((mat->size[mat->dims - 1] - tempIndices[mat->dims - 1]) * mat->channels(), available); - while (available > 0) { - memcpy(tBuffer + copyOffset, mat->ptr(tempIndices.data()), copyCount * sizeof(T)); - if (updateIdx(mat, tempIndices, copyCount / mat->channels())) { - break; - } - available -= copyCount; - copyOffset += copyCount * sizeof(T); - copyCount = MIN(mat->size[mat->dims-1] * mat->channels(), available); + char* buff = (char*)tBuffer; + size_t blockSize = mat->size[mat->dims-1] * mat->elemSize(); + size_t firstPartialBlockSize = (mat->size[mat->dims-1] - tempIndices[mat->dims-1]) * mat->step[mat->dims-1]; + for (int dim=mat->dims-2; dim>=0 && blockSize == mat->step[dim]; dim--) { + blockSize *= mat->size[dim]; + firstPartialBlockSize += (mat->size[dim] - (tempIndices[dim]+1)) * mat->step[dim]; + } + size_t copyCount = (countBytesptr(tempIndices.data()); + while(countBytes>0) { + memcpy(buff, data, copyCount); + updateIdx(mat, tempIndices, copyCount / mat->elemSize()); + countBytes -= copyCount; + buff += copyCount; + copyCount = countBytesptr(tempIndices.data()); } } return result; @@ -817,22 +834,31 @@ template int putData(NSArray* indices, cv::Mat* mat, int } int arrayAvailable = count; + size_t countBytes = count * sizeof(T); + size_t remainingBytes = (size_t)(mat->total() - idx2Offset(mat, tempIndices))*mat->elemSize(); + countBytes = (countBytes>remainingBytes)?remainingBytes:countBytes; + int result = (int)countBytes; int matAvailable = getMatAvailable(mat, tempIndices); int available = MIN(arrayAvailable, matAvailable); - int result = (int)(available * mat->elemSize() / mat->channels()); if (mat->isContinuous()) { memcpy(mat->ptr(tempIndices.data()), tBuffer, available * sizeof(T)); } else { - int copyOffset = 0; - int copyCount = MIN((mat->size[mat->dims - 1] - tempIndices[mat->dims - 1]) * mat->channels(), available); - while (available > 0) { - memcpy(mat->ptr(tempIndices.data()), tBuffer + copyOffset, copyCount * sizeof(T)); - if (updateIdx(mat, tempIndices, copyCount / mat->channels())) { - break; - } - available -= copyCount; - copyOffset += copyCount * sizeof(T); - copyCount = MIN(mat->size[mat->dims-1] * (int)mat->channels(), available); + char* buff = (char*)tBuffer; + size_t blockSize = mat->size[mat->dims-1] * mat->elemSize(); + size_t firstPartialBlockSize = (mat->size[mat->dims-1] - tempIndices[mat->dims-1]) * mat->step[mat->dims-1]; + for (int dim=mat->dims-2; dim>=0 && blockSize == mat->step[dim]; dim--) { + blockSize *= mat->size[dim]; + firstPartialBlockSize += (mat->size[dim] - (tempIndices[dim]+1)) * mat->step[dim]; + } + size_t copyCount = (countBytesptr(tempIndices.data()); + while(countBytes>0){ + memcpy(data, buff, copyCount); + updateIdx(mat, tempIndices, copyCount / mat->elemSize()); + countBytes -= copyCount; + buff += copyCount; + copyCount = countBytesptr(tempIndices.data()); } } return result; diff --git a/modules/core/misc/objc/common/MatExt.swift b/modules/core/misc/objc/common/MatExt.swift index 5ce3a5e6fb..f0f4446682 100644 --- a/modules/core/misc/objc/common/MatExt.swift +++ b/modules/core/misc/objc/common/MatExt.swift @@ -62,6 +62,21 @@ public extension Mat { } } + @discardableResult func get(indices:[Int32], data:inout [UInt8]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_8U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeMutableBufferPointer { body in + body.withMemoryRebound(to: Int8.self) { reboundBody in + return __get(indices as [NSNumber], count: count, byteBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func get(indices:[Int32], data:inout [Double]) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -114,10 +129,29 @@ public extension Mat { } } + @discardableResult func get(indices:[Int32], data:inout [UInt16]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_16U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeMutableBufferPointer { body in + body.withMemoryRebound(to: Int16.self) { reboundBody in + return __get(indices as [NSNumber], count: count, shortBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func get(row: Int32, col: Int32, data:inout [Int8]) throws -> Int32 { return try get(indices: [row, col], data: &data) } + @discardableResult func get(row: Int32, col: Int32, data:inout [UInt8]) throws -> Int32 { + return try get(indices: [row, col], data: &data) + } + @discardableResult func get(row: Int32, col: Int32, data:inout [Double]) throws -> Int32 { return try get(indices: [row, col], data: &data) } @@ -134,6 +168,10 @@ public extension Mat { return try get(indices: [row, col], data: &data) } + @discardableResult func get(row: Int32, col: Int32, data:inout [UInt16]) throws -> Int32 { + return try get(indices: [row, col], data: &data) + } + @discardableResult func put(indices:[Int32], data:[Int8]) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -147,6 +185,21 @@ public extension Mat { } } + @discardableResult func put(indices:[Int32], data:[UInt8]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_8U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeBufferPointer { body in + body.withMemoryRebound(to: Int8.self) { reboundBody in + return __put(indices as [NSNumber], count: count, byteBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func put(indices:[Int32], data:[Int8], offset: Int, length: Int32) throws -> Int32 { let channels = CvType.channels(Int32(type())) if Int32(data.count) % channels != 0 { @@ -214,10 +267,29 @@ public extension Mat { } } + @discardableResult func put(indices:[Int32], data:[UInt16]) throws -> Int32 { + let channels = CvType.channels(Int32(type())) + if Int32(data.count) % channels != 0 { + try throwIncompatibleBufferSize(count: data.count, channels: channels) + } else if depth() != CvType.CV_16U { + try throwIncompatibleDataType(typeName: CvType.type(toString: type())) + } + let count = Int32(data.count) + return data.withUnsafeBufferPointer { body in + body.withMemoryRebound(to: Int16.self) { reboundBody in + return __put(indices as [NSNumber], count: count, shortBuffer: reboundBody.baseAddress!) + } + } + } + @discardableResult func put(row: Int32, col: Int32, data:[Int8]) throws -> Int32 { return try put(indices: [row, col], data: data) } + @discardableResult func put(row: Int32, col: Int32, data:[UInt8]) throws -> Int32 { + return try put(indices: [row, col], data: data) + } + @discardableResult func put(row: Int32, col: Int32, data: [Int8], offset: Int, length: Int32) throws -> Int32 { return try put(indices: [row, col], data: data, offset: offset, length: length) } @@ -238,6 +310,10 @@ public extension Mat { return try put(indices: [row, col], data: data) } + @discardableResult func put(row: Int32, col: Int32, data: [UInt16]) throws -> Int32 { + return try put(indices: [row, col], data: data) + } + @discardableResult func get(row: Int32, col: Int32) -> [Double] { return get(indices: [row, col]) } @@ -303,46 +379,46 @@ public class MatAt { extension UInt8: Atable { public static func getAt(m: Mat, indices:[Int32]) -> UInt8 { - var tmp = [Int8](repeating: 0, count: 1) + var tmp = [UInt8](repeating: 0, count: 1) try! m.get(indices: indices, data: &tmp) - return UInt8(bitPattern: tmp[0]) + return tmp[0] } public static func putAt(m: Mat, indices: [Int32], v: UInt8) { - let tmp = [Int8(bitPattern: v)] + let tmp = [v] try! m.put(indices: indices, data: tmp) } public static func getAt2c(m: Mat, indices:[Int32]) -> (UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 2) + var tmp = [UInt8](repeating: 0, count: 2) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1])) + return (tmp[0], tmp[1]) } public static func putAt2c(m: Mat, indices: [Int32], v: (UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1)] + let tmp = [v.0, v.1] try! m.put(indices: indices, data: tmp) } public static func getAt3c(m: Mat, indices:[Int32]) -> (UInt8, UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 3) + var tmp = [UInt8](repeating: 0, count: 3) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1]), UInt8(bitPattern: tmp[2])) + return (tmp[0], tmp[1], tmp[2]) } public static func putAt3c(m: Mat, indices: [Int32], v: (UInt8, UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1), Int8(bitPattern: v.2)] + let tmp = [v.0, v.1, v.2] try! m.put(indices: indices, data: tmp) } public static func getAt4c(m: Mat, indices:[Int32]) -> (UInt8, UInt8, UInt8, UInt8) { - var tmp = [Int8](repeating: 0, count: 4) + var tmp = [UInt8](repeating: 0, count: 4) try! m.get(indices: indices, data: &tmp) - return (UInt8(bitPattern: tmp[0]), UInt8(bitPattern: tmp[1]), UInt8(bitPattern: tmp[2]), UInt8(bitPattern: tmp[3])) + return (tmp[0], tmp[1], tmp[2], tmp[3]) } public static func putAt4c(m: Mat, indices: [Int32], v: (UInt8, UInt8, UInt8, UInt8)) { - let tmp = [Int8(bitPattern: v.0), Int8(bitPattern: v.1), Int8(bitPattern: v.2), Int8(bitPattern: v.3)] + let tmp = [v.0, v.1, v.2, v.3] try! m.put(indices: indices, data: tmp) } } @@ -531,6 +607,52 @@ extension Int32: Atable { } } +extension UInt16: Atable { + public static func getAt(m: Mat, indices:[Int32]) -> UInt16 { + var tmp = [UInt16](repeating: 0, count: 1) + try! m.get(indices: indices, data: &tmp) + return tmp[0] + } + + public static func putAt(m: Mat, indices: [Int32], v: UInt16) { + let tmp = [v] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt2c(m: Mat, indices:[Int32]) -> (UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 2) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1]) + } + + public static func putAt2c(m: Mat, indices: [Int32], v: (UInt16, UInt16)) { + let tmp = [v.0, v.1] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt3c(m: Mat, indices:[Int32]) -> (UInt16, UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 3) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1], tmp[2]) + } + + public static func putAt3c(m: Mat, indices: [Int32], v: (UInt16, UInt16, UInt16)) { + let tmp = [v.0, v.1, v.2] + try! m.put(indices: indices, data: tmp) + } + + public static func getAt4c(m: Mat, indices:[Int32]) -> (UInt16, UInt16, UInt16, UInt16) { + var tmp = [UInt16](repeating: 0, count: 4) + try! m.get(indices: indices, data: &tmp) + return (tmp[0], tmp[1], tmp[2], tmp[3]) + } + + public static func putAt4c(m: Mat, indices: [Int32], v: (UInt16, UInt16, UInt16, UInt16)) { + let tmp = [v.0, v.1, v.2, v.3] + try! m.put(indices: indices, data: tmp) + } +} + extension Int16: Atable { public static func getAt(m: Mat, indices:[Int32]) -> Int16 { var tmp = [Int16](repeating: 0, count: 1) @@ -593,3 +715,9 @@ public extension Mat { return MatAt(mat: self, indices: indices) } } + +public extension Mat { + static func *(lhs:Mat, rhs: Mat) -> Mat { + return lhs.matMul(rhs) + } +} diff --git a/modules/core/misc/objc/common/Rect2d.mm b/modules/core/misc/objc/common/Rect2d.mm index e0aa8b2844..977294d66c 100644 --- a/modules/core/misc/objc/common/Rect2d.mm +++ b/modules/core/misc/objc/common/Rect2d.mm @@ -64,10 +64,10 @@ } - (instancetype)initWithPoint:(Point2d*)point1 point:(Point2d*)point2 { - int x = (point1.x < point2.x ? point1.x : point2.x); - int y = (point1.y < point2.y ? point1.y : point2.y); - int width = (point1.x > point2.x ? point1.x : point2.x) - x; - int height = (point1.y > point2.y ? point1.y : point2.y) - y; + double x = (point1.x < point2.x ? point1.x : point2.x); + double y = (point1.y < point2.y ? point1.y : point2.y); + double width = (point1.x > point2.x ? point1.x : point2.x) - x; + double height = (point1.y > point2.y ? point1.y : point2.y) - y; return [self initWithX:x y:y width:width height:height]; } diff --git a/modules/core/misc/objc/common/Rect2f.mm b/modules/core/misc/objc/common/Rect2f.mm index 243154fe1f..2d6c326cc6 100644 --- a/modules/core/misc/objc/common/Rect2f.mm +++ b/modules/core/misc/objc/common/Rect2f.mm @@ -64,10 +64,10 @@ } - (instancetype)initWithPoint:(Point2f*)point1 point:(Point2f*)point2 { - int x = (point1.x < point2.x ? point1.x : point2.x); - int y = (point1.y < point2.y ? point1.y : point2.y); - int width = (point1.x > point2.x ? point1.x : point2.x) - x; - int height = (point1.y > point2.y ? point1.y : point2.y) - y; + float x = (point1.x < point2.x ? point1.x : point2.x); + float y = (point1.y < point2.y ? point1.y : point2.y); + float width = (point1.x > point2.x ? point1.x : point2.x) - x; + float height = (point1.y > point2.y ? point1.y : point2.y) - y; return [self initWithX:x y:y width:width height:height]; } diff --git a/modules/core/misc/objc/test/MatTest.swift b/modules/core/misc/objc/test/MatTest.swift index 14c440b5eb..87f99fb84a 100644 --- a/modules/core/misc/objc/test/MatTest.swift +++ b/modules/core/misc/objc/test/MatTest.swift @@ -308,15 +308,15 @@ class MatTests: OpenCVTestCase { XCTAssert([340] == sm.get(row: 1, col: 1)) } - func testGetIntIntByteArray() throws { - let m = try getTestMat(size: 5, type: CvType.CV_8UC3) + func testGetIntIntInt8Array() throws { + let m = try getTestMat(size: 5, type: CvType.CV_8SC3) var goodData = [Int8](repeating: 0, count: 9) // whole Mat var bytesNum = try m.get(row: 1, col: 1, data: &goodData) XCTAssertEqual(9, bytesNum) - XCTAssert([110, 111, 112, 120, 121, 122, -126, -125, -124] == goodData) + XCTAssert([110, 111, 112, 120, 121, 122, 127, 127, 127] == goodData) var badData = [Int8](repeating: 0, count: 7) XCTAssertThrowsError(bytesNum = try m.get(row: 0, col: 0, data: &badData)) @@ -326,11 +326,36 @@ class MatTests: OpenCVTestCase { var buff00 = [Int8](repeating: 0, count: 3) bytesNum = try sm.get(row: 0, col: 0, data: &buff00) XCTAssertEqual(3, bytesNum) - XCTAssert(buff00 == [-26, -25, -24]) + XCTAssert(buff00 == [127, 127, 127]) var buff11 = [Int8](repeating: 0, count: 3) bytesNum = try sm.get(row: 1, col: 1, data: &buff11) XCTAssertEqual(3, bytesNum) - XCTAssert(buff11 == [-1, -1, -1]) + XCTAssert(buff11 == [127, 127, 127]) + } + + func testGetIntIntUInt8Array() throws { + let m = try getTestMat(size: 5, type: CvType.CV_8UC3) + var goodData = [UInt8](repeating: 0, count: 9) + + // whole Mat + var bytesNum = try m.get(row: 1, col: 1, data: &goodData) + + XCTAssertEqual(9, bytesNum) + XCTAssert([110, 111, 112, 120, 121, 122, 130, 131, 132] == goodData) + + var badData = [UInt8](repeating: 0, count: 7) + XCTAssertThrowsError(bytesNum = try m.get(row: 0, col: 0, data: &badData)) + + // sub-Mat + let sm = m.submat(rowStart: 2, rowEnd: 4, colStart: 3, colEnd: 5) + var buff00 = [UInt8](repeating: 0, count: 3) + bytesNum = try sm.get(row: 0, col: 0, data: &buff00) + XCTAssertEqual(3, bytesNum) + XCTAssert(buff00 == [230, 231, 232]) + var buff11 = [UInt8](repeating: 0, count: 3) + bytesNum = try sm.get(row: 1, col: 1, data: &buff11) + XCTAssertEqual(3, bytesNum) + XCTAssert(buff11 == [255, 255, 255]) } func testGetIntIntDoubleArray() throws { @@ -399,14 +424,14 @@ class MatTests: OpenCVTestCase { XCTAssert(buff11 == [340, 341, 0, 0]) } - func testGetIntIntShortArray() throws { + func testGetIntIntInt16Array() throws { let m = try getTestMat(size: 5, type: CvType.CV_16SC2) var buff = [Int16](repeating: 0, count: 6) // whole Mat var bytesNum = try m.get(row: 1, col: 1, data: &buff) - XCTAssertEqual(12, bytesNum); + XCTAssertEqual(12, bytesNum) XCTAssert(buff == [110, 111, 120, 121, 130, 131]) // sub-Mat @@ -417,6 +442,46 @@ class MatTests: OpenCVTestCase { XCTAssert(buff00 == [230, 231, 240, 241]) var buff11 = [Int16](repeating: 0, count: 4) bytesNum = try sm.get(row: 1, col: 1, data: &buff11) + XCTAssertEqual(4, bytesNum) + XCTAssert(buff11 == [340, 341, 0, 0]) + + let m2 = Mat(sizes: [5, 6, 8], type: CvType.CV_16S) + let data:[Int16] = (0.. 0: + raise TypeError('Unknown parameters: {}'.format(repr(kwargs))) + + def __array_finalize__(self, obj): + if obj is None: + return + self.wrap_channels = getattr(obj, 'wrap_channels', None) + + +Mat.__module__ = cv.__name__ +cv.Mat = Mat +cv._registerMatType(Mat) diff --git a/modules/core/misc/python/package/utils/__init__.py b/modules/core/misc/python/package/utils/__init__.py new file mode 100644 index 0000000000..49cd40ba2d --- /dev/null +++ b/modules/core/misc/python/package/utils/__init__.py @@ -0,0 +1,14 @@ +from collections import namedtuple + +import cv2 + + +NativeMethodPatchedResult = namedtuple("NativeMethodPatchedResult", + ("py", "native")) + + +def testOverwriteNativeMethod(arg): + return NativeMethodPatchedResult( + arg + 1, + cv2.utils._native.testOverwriteNativeMethod(arg) + ) diff --git a/modules/core/src/async.cpp b/modules/core/src/async.cpp index a2f4612365..78c0a1ee81 100644 --- a/modules/core/src/async.cpp +++ b/modules/core/src/async.cpp @@ -14,6 +14,7 @@ #define CV_LOG_STRIP_LEVEL CV_LOG_LEVEL_DEBUG + 1 #include +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #ifdef CV_CXX11 #include @@ -236,6 +237,171 @@ struct AsyncArray::Impl } }; +} // namespace + +#else // OPENCV_DISABLE_THREAD_SUPPORT + +namespace cv { + +// no threading +struct AsyncArray::Impl +{ + int refcount; + void addrefFuture() CV_NOEXCEPT { refcount_future++; refcount++; } + void releaseFuture() CV_NOEXCEPT { refcount_future--; if (0 == --refcount) delete this; } + int refcount_future; + void addrefPromise() CV_NOEXCEPT { refcount_promise++; refcount++; } \ + void releasePromise() CV_NOEXCEPT { refcount_promise--; if (0 == --refcount) delete this; } + int refcount_promise; + + mutable bool has_result; // Mat, UMat or exception + + mutable cv::Ptr result_mat; + mutable cv::Ptr result_umat; + + + bool has_exception; +#if CV__EXCEPTION_PTR + std::exception_ptr exception; +#endif + cv::Exception cv_exception; + + mutable bool result_is_fetched; + + bool future_is_returned; + + Impl() + : refcount(1), refcount_future(0), refcount_promise(1) + , has_result(false) + , has_exception(false) + , result_is_fetched(false) + , future_is_returned(false) + { + // nothing + } + + ~Impl() + { + if (has_result && !result_is_fetched) + { + CV_LOG_INFO(NULL, "Asynchronous result has not been fetched"); + } + } + + bool get(OutputArray dst, int64 timeoutNs) const + { + CV_Assert(!result_is_fetched); + if (!has_result) + { + CV_UNUSED(timeoutNs); + CV_Error(Error::StsError, "Result is not produced (unable to wait for result in OPENCV_DISABLE_THREAD_SUPPORT mode)"); + } + if (!result_mat.empty()) + { + dst.move(*result_mat.get()); + result_mat.release(); + result_is_fetched = true; + return true; + } + if (!result_umat.empty()) + { + dst.move(*result_umat.get()); + result_umat.release(); + result_is_fetched = true; + return true; + } +#if CV__EXCEPTION_PTR + if (has_exception && exception) + { + result_is_fetched = true; + std::rethrow_exception(exception); + } +#endif + if (has_exception) + { + result_is_fetched = true; + throw cv_exception; + } + CV_Error(Error::StsInternal, "AsyncArray: invalid state of 'has_result = true'"); + return false; + } + + bool valid() const CV_NOEXCEPT + { + if (result_is_fetched) + return false; + if (refcount_promise == 0 && !has_result) + return false; + return true; + } + + bool wait_for(int64 timeoutNs) const + { + CV_Assert(valid()); + if (has_result) + return has_result; + if (timeoutNs == 0) + return has_result; + CV_Error(Error::StsError, "Unable to wait in OPENCV_DISABLE_THREAD_SUPPORT mode"); + } + + AsyncArray getArrayResult() + { + CV_Assert(refcount_future == 0); + AsyncArray result; + addrefFuture(); + result.p = this; + future_is_returned = true; + return result; + } + + void setValue(InputArray value) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + int k = value.kind(); + if (k == _InputArray::UMAT) + { + result_umat = makePtr(); + value.copyTo(*result_umat.get()); + } + else + { + result_mat = makePtr(); + value.copyTo(*result_mat.get()); + } + has_result = true; + } + +#if CV__EXCEPTION_PTR + void setException(std::exception_ptr e) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + has_exception = true; + exception = e; + has_result = true; + } +#endif + + void setException(const cv::Exception e) + { + if (future_is_returned && refcount_future == 0) + CV_Error(Error::StsError, "Associated AsyncArray has been destroyed"); + CV_Assert(!has_result); + has_exception = true; + cv_exception = e; + has_result = true; + } +}; + +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + +namespace cv { AsyncArray::AsyncArray() CV_NOEXCEPT : p(NULL) diff --git a/modules/core/src/bindings_utils.cpp b/modules/core/src/bindings_utils.cpp index 78716c21f6..78093002f7 100644 --- a/modules/core/src/bindings_utils.cpp +++ b/modules/core/src/bindings_utils.cpp @@ -5,6 +5,7 @@ #include "precomp.hpp" #include "opencv2/core/bindings_utils.hpp" #include +#include #include #include @@ -210,6 +211,53 @@ CV_EXPORTS_W String dumpInputOutputArrayOfArrays(InputOutputArrayOfArrays argume return ss.str(); } +static inline std::ostream& operator<<(std::ostream& os, const cv::Rect& rect) +{ + return os << "[x=" << rect.x << ", y=" << rect.y << ", w=" << rect.width << ", h=" << rect.height << ']'; +} + +template +static inline String dumpVector(const std::vector& vec, Formatter format) +{ + std::ostringstream oss("[", std::ios::ate); + if (!vec.empty()) + { + oss << format << vec[0]; + for (std::size_t i = 1; i < vec.size(); ++i) + { + oss << ", " << format << vec[i]; + } + } + oss << "]"; + return oss.str(); +} + +static inline std::ostream& noFormat(std::ostream& os) +{ + return os; +} + +static inline std::ostream& floatFormat(std::ostream& os) +{ + return os << std::fixed << std::setprecision(2); +} + +String dumpVectorOfInt(const std::vector& vec) +{ + return dumpVector(vec, &noFormat); +} + +String dumpVectorOfDouble(const std::vector& vec) +{ + return dumpVector(vec, &floatFormat); +} + +String dumpVectorOfRect(const std::vector& vec) +{ + return dumpVector(vec, &noFormat); +} + + namespace fs { cv::String getCacheDirectoryForDownloads() { diff --git a/modules/core/src/intel_gpu_gemm.inl.hpp b/modules/core/src/intel_gpu_gemm.inl.hpp index a73e71a67d..28cc4ab9b9 100644 --- a/modules/core/src/intel_gpu_gemm.inl.hpp +++ b/modules/core/src/intel_gpu_gemm.inl.hpp @@ -24,11 +24,6 @@ #ifdef HAVE_OPENCL -#include -#include "opencl_kernels_core.hpp" -#include "opencv2/core/opencl/runtime/opencl_clblas.hpp" -#include "opencv2/core/opencl/runtime/opencl_core.hpp" - namespace cv { @@ -37,52 +32,75 @@ static bool intel_gpu_gemm( UMat B, Size sizeB, UMat D, Size sizeD, double alpha, double beta, - bool atrans, bool btrans) + bool atrans, bool btrans, + bool& isPropagatedC2D +) { CV_UNUSED(sizeB); int M = sizeD.height, N = sizeD.width, K = ((atrans)? sizeA.height : sizeA.width); - std::string kernelName; - bool ret = true; + if (M < 4 || N < 4 || K < 4) // vload4 + return false; - size_t lx = 8, ly = 4; - size_t dx = 4, dy = 8; + CV_LOG_VERBOSE(NULL, 0, "M=" << M << " N=" << N << " K=" << K); + + std::string kernelName; + + unsigned int lx = 8, ly = 4; + unsigned int dx = 4, dy = 8; if(!atrans && !btrans) { - if (M % 32 == 0 && N % 32 == 0 && K % 16 == 0) { kernelName = "intelblas_gemm_buffer_NN_sp"; } else { + if (M % 2 != 0) + return false; + // vload4(0, dst_write0) - 4 cols + // multiply by lx: 8 + if (N % (4*8) != 0) + return false; kernelName = "intelblas_gemm_buffer_NN"; } } else if(atrans && !btrans) { + if (M % 32 != 0) + return false; + if (N % 32 != 0) + return false; kernelName = "intelblas_gemm_buffer_TN"; } else if(!atrans && btrans) { + if (K % 4 != 0) + return false; kernelName = "intelblas_gemm_buffer_NT"; ly = 16; dx = 1; } else { + if (M % 32 != 0) + return false; + if (N % 32 != 0) + return false; + if (K % 16 != 0) + return false; kernelName = "intelblas_gemm_buffer_TT"; } - const size_t gx = (size_t)(N + dx - 1) / dx; - const size_t gy = (size_t)(M + dy - 1) / dy; + CV_LOG_DEBUG(NULL, "kernel: " << kernelName << " (M=" << M << " N=" << N << " K=" << K << ")"); + + const size_t gx = divUp((size_t)N, dx); + const size_t gy = divUp((size_t)M, dy); size_t local[] = {lx, ly, 1}; - size_t global[] = {(gx + lx - 1) / lx * lx, (gy + ly - 1) / ly * ly, 1}; - - int stride = (M * N < 1024 * 1024) ? 10000000 : 256; + size_t global[] = {roundUp(gx, lx), roundUp(gy, ly), 1}; ocl::Queue q; String errmsg; @@ -110,10 +128,13 @@ static bool intel_gpu_gemm( (int)(D.step / sizeof(float)) ); - ret = k.run(2, global, local, false, q); + bool ret = k.run(2, global, local, false, q); + return ret; } else { + int stride = (M * N < 1024 * 1024) ? 10000000 : 256; + for(int start_index = 0; start_index < K; start_index += stride) { ocl::Kernel k(kernelName.c_str(), program); @@ -132,12 +153,16 @@ static bool intel_gpu_gemm( (int) start_index, // 14 start_index stride); - ret = k.run(2, global, local, false, q); - if (!ret) return ret; + bool ret = k.run(2, global, local, false, q); + if (!ret) + { + if (start_index != 0) + isPropagatedC2D = false; // D array content is changed, need to rewrite + return false; + } } + return true; } - - return ret; } } // namespace cv diff --git a/modules/core/src/matmul.dispatch.cpp b/modules/core/src/matmul.dispatch.cpp index 651c454490..92e44f45c9 100644 --- a/modules/core/src/matmul.dispatch.cpp +++ b/modules/core/src/matmul.dispatch.cpp @@ -42,6 +42,8 @@ //M*/ #include "precomp.hpp" +#include + #include "opencl_kernels_core.hpp" #include "opencv2/core/opencl/runtime/opencl_clblas.hpp" #include "opencv2/core/opencl/runtime/opencl_core.hpp" @@ -155,10 +157,12 @@ static bool ocl_gemm_amdblas( InputArray matA, InputArray matB, double alpha, static bool ocl_gemm( InputArray matA, InputArray matB, double alpha, InputArray matC, double beta, OutputArray matD, int flags ) { - int depth = matA.depth(), cn = matA.channels(); - int type = CV_MAKETYPE(depth, cn); + int type = matA.type(); + int depth = CV_MAT_DEPTH(type); + int cn = CV_MAT_CN(type); - CV_Assert_N( type == matB.type(), (type == CV_32FC1 || type == CV_64FC1 || type == CV_32FC2 || type == CV_64FC2) ); + CV_CheckTypeEQ(type, matB.type(), ""); + CV_CheckType(type, type == CV_32FC1 || type == CV_64FC1 || type == CV_32FC2 || type == CV_64FC2, ""); const ocl::Device & dev = ocl::Device::getDefault(); bool doubleSupport = dev.doubleFPConfig() > 0; @@ -170,88 +174,103 @@ static bool ocl_gemm( InputArray matA, InputArray matB, double alpha, Size sizeA = matA.size(), sizeB = matB.size(), sizeC = haveC ? matC.size() : Size(0, 0); bool atrans = (flags & GEMM_1_T) != 0, btrans = (flags & GEMM_2_T) != 0, ctrans = (flags & GEMM_3_T) != 0; - CV_Assert( !haveC || matC.type() == type ); + if (haveC) + CV_CheckTypeEQ(type, matC.type(), ""); + + Size sizeD(((btrans) ? sizeB.height : sizeB.width), + ((atrans) ? sizeA.width : sizeA.height)); + + if (atrans) + sizeA = Size(sizeA.height, sizeA.width); + if (btrans) + sizeB = Size(sizeB.height, sizeB.width); + if (haveC && ctrans) + sizeC = Size(sizeC.height, sizeC.width); + + CV_CheckEQ(sizeA.width, sizeB.height, ""); + if (haveC) + CV_CheckEQ(sizeC, sizeD, ""); + + UMat A = matA.getUMat(); + UMat B = matB.getUMat(); - Size sizeD(((btrans)? sizeB.height : sizeB.width), - ((atrans)? sizeA.width : sizeA.height)); matD.create(sizeD, type); + UMat D = matD.getUMat(); - UMat A = matA.getUMat(), B = matB.getUMat(), D = matD.getUMat(); + bool isPropagatedC2D = false; // D content is updated with C / C.t() - - if (!dev.intelSubgroupsSupport() || (depth == CV_64F) || cn != 1) - { - String opts; - - if (atrans) - sizeA = Size(sizeA.height, sizeA.width); - if (btrans) - sizeB = Size(sizeB.height, sizeB.width); - if (haveC && ctrans) - sizeC = Size(sizeC.height, sizeC.width); - - CV_Assert( sizeA.width == sizeB.height && (!haveC || sizeC == sizeD) ); - - int max_wg_size = (int)dev.maxWorkGroupSize(); - int block_size = (max_wg_size / (32*cn) < 32) ? (max_wg_size / (16*cn) < 16) ? (max_wg_size / (8*cn) < 8) ? 1 : 8 : 16 : 32; - - if (atrans) - A = A.t(); - - if (btrans) - B = B.t(); - - if (haveC) - ctrans ? transpose(matC, D) : matC.copyTo(D); - - int vectorWidths[] = { 4, 4, 2, 2, 1, 4, cn, -1 }; - int kercn = ocl::checkOptimalVectorWidth(vectorWidths, B, D); - - opts += format(" -D T=%s -D T1=%s -D WT=%s -D cn=%d -D kercn=%d -D LOCAL_SIZE=%d%s%s%s", - ocl::typeToStr(type), ocl::typeToStr(depth), ocl::typeToStr(CV_MAKETYPE(depth, kercn)), - cn, kercn, block_size, - (sizeA.width % block_size !=0) ? " -D NO_MULT" : "", - haveC ? " -D HAVE_C" : "", - doubleSupport ? " -D DOUBLE_SUPPORT" : ""); - - ocl::Kernel k("gemm", cv::ocl::core::gemm_oclsrc, opts); - if (k.empty()) - return false; - - if (depth == CV_64F) - k.args(ocl::KernelArg::ReadOnlyNoSize(A), - ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), - ocl::KernelArg::ReadWrite(D, cn, kercn), - sizeA.width, alpha, beta); - else - k.args(ocl::KernelArg::ReadOnlyNoSize(A), - ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), - ocl::KernelArg::ReadWrite(D, cn, kercn), - sizeA.width, (float)alpha, (float)beta); - - size_t globalsize[2] = { (size_t)sizeD.width * cn / kercn, (size_t)sizeD.height}; - size_t localsize[2] = { (size_t)block_size, (size_t)block_size}; - - return k.run(2, globalsize, block_size!=1 ? localsize : NULL, false); - } - else + if (dev.intelSubgroupsSupport() && (depth == CV_32F) && cn == 1) { if (haveC && beta != 0.0) { ctrans ? transpose(matC, D) : matC.copyTo(D); + isPropagatedC2D = true; } else { beta = 0.0; } - return intel_gpu_gemm(A, sizeA, - B, sizeB, - D, sizeD, - alpha, - beta, - atrans, btrans); + bool res = intel_gpu_gemm(A, matA.size(), + B, matB.size(), + D, sizeD, + alpha, + beta, + atrans, btrans, + isPropagatedC2D); + if (res) + return true; + // fallback on generic OpenCL code } + + if (sizeD.width < 8 || sizeD.height < 8) + return false; + + String opts; + + int wg_size = (int)dev.maxWorkGroupSize(); + int sizeDmin = std::min(sizeD.width, sizeD.height); + wg_size = std::min(wg_size, sizeDmin * sizeDmin); + int block_size = (wg_size / (32*cn) < 32) ? (wg_size / (16*cn) < 16) ? (wg_size / (8*cn) < 8) ? 1 : 8 : 16 : 32; + + if (atrans) + A = A.t(); + + if (btrans) + B = B.t(); + + if (haveC && !isPropagatedC2D) + ctrans ? transpose(matC, D) : matC.copyTo(D); + + int vectorWidths[] = { 4, 4, 2, 2, 1, 4, cn, -1 }; + int kercn = ocl::checkOptimalVectorWidth(vectorWidths, B, D); + + opts += format(" -D T=%s -D T1=%s -D WT=%s -D cn=%d -D kercn=%d -D LOCAL_SIZE=%d%s%s%s", + ocl::typeToStr(type), ocl::typeToStr(depth), ocl::typeToStr(CV_MAKETYPE(depth, kercn)), + cn, kercn, block_size, + (sizeA.width % block_size !=0) ? " -D NO_MULT" : "", + haveC ? " -D HAVE_C" : "", + doubleSupport ? " -D DOUBLE_SUPPORT" : ""); + + ocl::Kernel k("gemm", cv::ocl::core::gemm_oclsrc, opts); + if (k.empty()) + return false; + + if (depth == CV_64F) + k.args(ocl::KernelArg::ReadOnlyNoSize(A), + ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), + ocl::KernelArg::ReadWrite(D, cn, kercn), + sizeA.width, alpha, beta); + else + k.args(ocl::KernelArg::ReadOnlyNoSize(A), + ocl::KernelArg::ReadOnlyNoSize(B, cn, kercn), + ocl::KernelArg::ReadWrite(D, cn, kercn), + sizeA.width, (float)alpha, (float)beta); + + size_t globalsize[2] = { (size_t)sizeD.width * cn / kercn, (size_t)sizeD.height}; + size_t localsize[2] = { (size_t)block_size, (size_t)block_size}; + + return k.run(2, globalsize, block_size !=1 ? localsize : NULL, false); } #endif diff --git a/modules/core/src/matrix.cpp b/modules/core/src/matrix.cpp index 61abc2ba8f..4642d3b623 100644 --- a/modules/core/src/matrix.cpp +++ b/modules/core/src/matrix.cpp @@ -809,18 +809,17 @@ Mat::Mat(const Mat& m, const Rect& roi) data += roi.x*esz; CV_Assert( 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= m.cols && 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= m.rows ); - if( u ) - CV_XADD(&u->refcount, 1); if( roi.width < m.cols || roi.height < m.rows ) flags |= SUBMATRIX_FLAG; step[0] = m.step[0]; step[1] = esz; updateContinuityFlag(); + addref(); if( rows <= 0 || cols <= 0 ) { - release(); rows = cols = 0; + release(); } } diff --git a/modules/core/src/matrix_operations.cpp b/modules/core/src/matrix_operations.cpp index 83c8aaeb57..227c7aaef7 100644 --- a/modules/core/src/matrix_operations.cpp +++ b/modules/core/src/matrix_operations.cpp @@ -229,14 +229,14 @@ void cv::setIdentity( InputOutputArray _m, const Scalar& s ) namespace cv { -UMat UMat::eye(int rows, int cols, int type) +UMat UMat::eye(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat::eye(Size(cols, rows), type); + return UMat::eye(Size(cols, rows), type, usageFlags); } -UMat UMat::eye(Size size, int type) +UMat UMat::eye(Size size, int type, UMatUsageFlags usageFlags) { - UMat m(size, type); + UMat m(size, type, usageFlags); setIdentity(m); return m; } diff --git a/modules/core/src/norm.cpp b/modules/core/src/norm.cpp index bbefefc95d..4df25f4957 100644 --- a/modules/core/src/norm.cpp +++ b/modules/core/src/norm.cpp @@ -1194,7 +1194,7 @@ double norm( InputArray _src1, InputArray _src2, int normType, InputArray _mask // special case to handle "integer" overflow in accumulator const size_t esz = src1.elemSize(); const int total = (int)it.size; - const int intSumBlockSize = normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15); + const int intSumBlockSize = (normType == NORM_L1 && depth <= CV_8S ? (1 << 23) : (1 << 15))/cn; const int blockSize = std::min(total, intSumBlockSize); int isum = 0; int count = 0; diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index 0e97cf52fe..8e545f6293 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -76,8 +76,11 @@ #undef CV__ALLOCATOR_STATS_LOG #define CV_OPENCL_ALWAYS_SHOW_BUILD_LOG 0 +#define CV_OPENCL_SHOW_BUILD_OPTIONS 0 +#define CV_OPENCL_SHOW_BUILD_KERNELS 0 #define CV_OPENCL_SHOW_RUN_KERNELS 0 +#define CV_OPENCL_SYNC_RUN_KERNELS 0 #define CV_OPENCL_TRACE_CHECK 0 #define CV_OPENCL_VALIDATE_BINARY_PROGRAMS 1 @@ -1566,6 +1569,7 @@ struct Device::Impl version_ = getStrProp(CL_DEVICE_VERSION); extensions_ = getStrProp(CL_DEVICE_EXTENSIONS); doubleFPConfig_ = getProp(CL_DEVICE_DOUBLE_FP_CONFIG); + halfFPConfig_ = getProp(CL_DEVICE_HALF_FP_CONFIG); hostUnifiedMemory_ = getBoolProp(CL_DEVICE_HOST_UNIFIED_MEMORY); maxComputeUnits_ = getProp(CL_DEVICE_MAX_COMPUTE_UNITS); maxWorkGroupSize_ = getProp(CL_DEVICE_MAX_WORK_GROUP_SIZE); @@ -1678,6 +1682,7 @@ struct Device::Impl String version_; std::string extensions_; int doubleFPConfig_; + int halfFPConfig_; bool hostUnifiedMemory_; int maxComputeUnits_; size_t maxWorkGroupSize_; @@ -1827,11 +1832,7 @@ int Device::singleFPConfig() const { return p ? p->getProp(CL_DEVICE_SINGLE_FP_CONFIG) : 0; } int Device::halfFPConfig() const -#ifdef CL_VERSION_1_2 -{ return p ? p->getProp(CL_DEVICE_HALF_FP_CONFIG) : 0; } -#else -{ CV_REQUIRE_OPENCL_1_2_ERROR; } -#endif +{ return p ? p->halfFPConfig_ : 0; } bool Device::endianLittle() const { return p ? p->getBoolProp(CL_DEVICE_ENDIAN_LITTLE) : false; } @@ -2157,20 +2158,22 @@ static cl_device_id selectOpenCLDevice(const char* configuration = NULL) platforms.resize(numPlatforms); } - int selectedPlatform = -1; if (platform.length() > 0) { - for (size_t i = 0; i < platforms.size(); i++) + for (std::vector::iterator currentPlatform = platforms.begin(); currentPlatform != platforms.end();) { std::string name; - CV_OCL_DBG_CHECK(getStringInfo(clGetPlatformInfo, platforms[i], CL_PLATFORM_NAME, name)); + CV_OCL_DBG_CHECK(getStringInfo(clGetPlatformInfo, *currentPlatform, CL_PLATFORM_NAME, name)); if (name.find(platform) != std::string::npos) { - selectedPlatform = (int)i; - break; + ++currentPlatform; + } + else + { + currentPlatform = platforms.erase(currentPlatform); } } - if (selectedPlatform == -1) + if (platforms.size() == 0) { CV_LOG_ERROR(NULL, "OpenCL: Can't find OpenCL platform by name: " << platform); goto not_found; @@ -2207,13 +2210,11 @@ static cl_device_id selectOpenCLDevice(const char* configuration = NULL) goto not_found; } - std::vector devices; // TODO Use clReleaseDevice to cleanup - for (int i = selectedPlatform >= 0 ? selectedPlatform : 0; - (selectedPlatform >= 0 ? i == selectedPlatform : true) && (i < (int)platforms.size()); - i++) + std::vector devices; + for (std::vector::iterator currentPlatform = platforms.begin(); currentPlatform != platforms.end(); ++currentPlatform) { cl_uint count = 0; - cl_int status = clGetDeviceIDs(platforms[i], deviceType, 0, NULL, &count); + cl_int status = clGetDeviceIDs(*currentPlatform, deviceType, 0, NULL, &count); if (!(status == CL_SUCCESS || status == CL_DEVICE_NOT_FOUND)) { CV_OCL_DBG_CHECK_RESULT(status, "clGetDeviceIDs get count"); @@ -2222,7 +2223,7 @@ static cl_device_id selectOpenCLDevice(const char* configuration = NULL) continue; size_t base = devices.size(); devices.resize(base + count); - status = clGetDeviceIDs(platforms[i], deviceType, count, &devices[base], &count); + status = clGetDeviceIDs(*currentPlatform, deviceType, count, &devices[base], &count); if (!(status == CL_SUCCESS || status == CL_DEVICE_NOT_FOUND)) { CV_OCL_DBG_CHECK_RESULT(status, "clGetDeviceIDs get IDs"); @@ -3455,19 +3456,33 @@ struct Kernel::Impl void cleanupUMats() { + bool exceptionOccurred = false; for( int i = 0; i < MAX_ARRS; i++ ) + { if( u[i] ) { if( CV_XADD(&u[i]->urefcount, -1) == 1 ) { u[i]->flags |= UMatData::ASYNC_CLEANUP; - u[i]->currAllocator->deallocate(u[i]); + try + { + u[i]->currAllocator->deallocate(u[i]); + } + catch(const std::exception& exc) + { + // limited by legacy before C++11, therefore log and + // remember some exception occurred to throw below + CV_LOG_ERROR(NULL, "OCL: Unexpected C++ exception in OpenCL Kernel::Impl::cleanupUMats(): " << exc.what()); + exceptionOccurred = true; + } } u[i] = 0; } + } nu = 0; haveTempDstUMats = false; haveTempSrcUMats = false; + CV_Assert(!exceptionOccurred); } void addUMat(const UMat& m, bool dst) @@ -3498,8 +3513,16 @@ struct Kernel::Impl void finit(cl_event e) { CV_UNUSED(e); - cleanupUMats(); isInProgress = false; + try + { + cleanupUMats(); + } + catch(...) + { + release(); + throw; + } release(); } @@ -3657,6 +3680,10 @@ bool Kernel::empty() const static cv::String dumpValue(size_t sz, const void* p) { + if (!p) + return "NULL"; + if (sz == 2) + return cv::format("%d / %uu / 0x%04x", *(short*)p, *(unsigned short*)p, *(short*)p); if (sz == 4) return cv::format("%d / %uu / 0x%08x / %g", *(int*)p, *(int*)p, *(int*)p, *(float*)p); if (sz == 8) @@ -3829,6 +3856,14 @@ bool Kernel::run(int dims, size_t _globalsize[], size_t _localsize[], } +bool Kernel::run_(int dims, size_t _globalsize[], size_t _localsize[], + bool sync, const Queue& q) +{ + CV_Assert(p); + return p->run(dims, _globalsize, _localsize, sync, NULL, q); +} + + static bool isRaiseErrorOnReuseAsyncKernel() { static bool initialized = false; @@ -3869,6 +3904,10 @@ bool Kernel::Impl::run(int dims, size_t globalsize[], size_t localsize[], return false; // OpenCV 5.0: raise error } +#if CV_OPENCL_SYNC_RUN_KERNELS + sync = true; +#endif + cl_command_queue qq = getQueue(q); if (haveTempDstUMats) sync = true; @@ -4316,7 +4355,28 @@ struct Program::Impl if (!param_buildExtraOptions.empty()) buildflags = joinBuildOptions(buildflags, param_buildExtraOptions); } +#if CV_OPENCL_SHOW_BUILD_OPTIONS + CV_LOG_INFO(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' options:" << buildflags); +#endif compile(ctx, src_, errmsg); +#if CV_OPENCL_SHOW_BUILD_KERNELS + if (handle) + { + size_t retsz = 0; + char kernels_buffer[4096] = {0}; + cl_int result = clGetProgramInfo(handle, CL_PROGRAM_KERNEL_NAMES, sizeof(kernels_buffer), &kernels_buffer[0], &retsz); + CV_OCL_DBG_CHECK_RESULT(result, cv::format("clGetProgramInfo(CL_PROGRAM_KERNEL_NAMES: %s/%s)", sourceModule_.c_str(), sourceName_.c_str()).c_str()); + if (result == CL_SUCCESS && retsz < sizeof(kernels_buffer)) + { + kernels_buffer[retsz] = 0; + CV_LOG_INFO(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' kernels: '" << kernels_buffer << "'"); + } + else + { + CV_LOG_ERROR(NULL, "OpenCL program '" << sourceModule_ << "/" << sourceName_ << "' can't retrieve kernel names!"); + } + } +#endif } bool compile(const Context& ctx, const ProgramSource::Impl* src_, String& errmsg) @@ -4548,7 +4608,6 @@ struct Program::Impl CV_LOG_INFO(NULL, result << ": Kernels='" << kernels_buffer << "'"); } #endif - } return handle != NULL; } @@ -6668,6 +6727,10 @@ void convertFromImage(void* cl_mem_image, UMat& dst) depth = CV_32F; break; + case CL_HALF_FLOAT: + depth = CV_16F; + break; + default: CV_Error(cv::Error::OpenCLApiCallError, "Not supported image_channel_data_type"); } @@ -6676,9 +6739,23 @@ void convertFromImage(void* cl_mem_image, UMat& dst) switch (fmt.image_channel_order) { case CL_R: + case CL_A: + case CL_INTENSITY: + case CL_LUMINANCE: type = CV_MAKE_TYPE(depth, 1); break; + case CL_RG: + case CL_RA: + type = CV_MAKE_TYPE(depth, 2); + break; + + // CL_RGB has no mappings to OpenCV types because CL_RGB can only be used with + // CL_UNORM_SHORT_565, CL_UNORM_SHORT_555, or CL_UNORM_INT_101010. + /*case CL_RGB: + type = CV_MAKE_TYPE(depth, 3); + break;*/ + case CL_RGBA: case CL_BGRA: case CL_ARGB: @@ -7068,6 +7145,13 @@ static std::string kerToStr(const Mat & k) stream << "DIG(" << data[i] << "f)"; stream << "DIG(" << data[width] << "f)"; } + else if (depth == CV_16F) + { + stream.setf(std::ios_base::showpoint); + for (int i = 0; i < width; ++i) + stream << "DIG(" << (float)data[i] << "h)"; + stream << "DIG(" << (float)data[width] << "h)"; + } else { for (int i = 0; i < width; ++i) @@ -7091,7 +7175,7 @@ String kernelToStr(InputArray _kernel, int ddepth, const char * name) typedef std::string (* func_t)(const Mat &); static const func_t funcs[] = { kerToStr, kerToStr, kerToStr, kerToStr, - kerToStr, kerToStr, kerToStr, 0 }; + kerToStr, kerToStr, kerToStr, kerToStr }; const func_t func = funcs[ddepth]; CV_Assert(func != 0); @@ -7130,14 +7214,14 @@ int predictOptimalVectorWidth(InputArray src1, InputArray src2, InputArray src3, int vectorWidths[] = { d.preferredVectorWidthChar(), d.preferredVectorWidthChar(), d.preferredVectorWidthShort(), d.preferredVectorWidthShort(), d.preferredVectorWidthInt(), d.preferredVectorWidthFloat(), - d.preferredVectorWidthDouble(), -1 }; + d.preferredVectorWidthDouble(), d.preferredVectorWidthHalf() }; // if the device says don't use vectors if (vectorWidths[0] == 1) { // it's heuristic vectorWidths[CV_8U] = vectorWidths[CV_8S] = 4; - vectorWidths[CV_16U] = vectorWidths[CV_16S] = 2; + vectorWidths[CV_16U] = vectorWidths[CV_16S] = vectorWidths[CV_16F] = 2; vectorWidths[CV_32S] = vectorWidths[CV_32F] = vectorWidths[CV_64F] = 1; } @@ -7225,10 +7309,12 @@ struct Image2D::Impl { cl_image_format format; static const int channelTypes[] = { CL_UNSIGNED_INT8, CL_SIGNED_INT8, CL_UNSIGNED_INT16, - CL_SIGNED_INT16, CL_SIGNED_INT32, CL_FLOAT, -1, -1 }; + CL_SIGNED_INT16, CL_SIGNED_INT32, CL_FLOAT, -1, CL_HALF_FLOAT }; static const int channelTypesNorm[] = { CL_UNORM_INT8, CL_SNORM_INT8, CL_UNORM_INT16, CL_SNORM_INT16, -1, -1, -1, -1 }; - static const int channelOrders[] = { -1, CL_R, CL_RG, -1, CL_RGBA }; + // CL_RGB has no mappings to OpenCV types because CL_RGB can only be used with + // CL_UNORM_SHORT_565, CL_UNORM_SHORT_555, or CL_UNORM_INT_101010. + static const int channelOrders[] = { -1, CL_R, CL_RG, /*CL_RGB*/ -1, CL_RGBA }; int channelType = norm ? channelTypesNorm[depth] : channelTypes[depth]; int channelOrder = channelOrders[cn]; diff --git a/modules/core/src/opencl/intel_gemm.cl b/modules/core/src/opencl/intel_gemm.cl index 6cea8d7efd..53ae790779 100644 --- a/modules/core/src/opencl/intel_gemm.cl +++ b/modules/core/src/opencl/intel_gemm.cl @@ -392,6 +392,15 @@ __kernel void intelblas_gemm_buffer_NN( #define TILE_N 8 #define SLM_BLOCK 512 +/* + A K B.t() K D N + ----------- ----------- ----------- + | | | | | | + M | | x N | | => M | | + | | | | | | + ----------- ----------- ----------- +*/ + __attribute__((reqd_work_group_size(8, LWG_HEIGHT, 1))) __kernel void intelblas_gemm_buffer_NT( const __global float *src0, int off0, @@ -422,59 +431,79 @@ __kernel void intelblas_gemm_buffer_NT( float8 dot06 = 0.f; float8 dot07 = 0.f; - float4 brow0; - float4 brow1; - float4 brow2; - float4 brow3; - float4 brow4; - float4 brow5; - float4 brow6; - float4 brow7; + const int dst_row = (global_y * TILE_M); + __global float *dst_write0 = dst + global_x + dst_row * ldC + offd; - __global float *dst_write0 = dst + local_x * VEC_SIZE + ( group_x * TILE_N ) + ( group_y * LWG_HEIGHT * TILE_M + local_y * TILE_M) * ldC + offd; + const __global float *src0_read00 = src0 + off0; + const int a_row_base = global_y * TILE_M; + const int a_col_base = local_x * (TILE_K / 8); // <= TILE_K - 4 - const __global float *src0_read = src0 + local_x * ( TILE_K / 8 ) + ( group_y * LWG_HEIGHT * TILE_M + local_y * TILE_M ) * ldA + off0; - - const __global float *src1_read0 = src1 + ( group_x * TILE_N ) * ldB + off1; + const __global float *src1_read00 = src1 + off1; + const int b_row_base = (group_x * TILE_N); + //const int b_col_base = 0; __local float slm_brow[8 * SLM_BLOCK]; - __local float* slm_brow0; int local_index = mad24(local_y, 8, local_x) * 4; - int w; - for(int b_tile = 0; b_tile < K; b_tile += SLM_BLOCK) { + int w = 0; + for (int b_tile = 0; b_tile < K; b_tile += SLM_BLOCK) + { +#define UPDATE_BROW(_row) \ + { \ + float4 brow; \ + int b_row = b_row_base + _row; \ + int b_col = b_tile + local_index; \ + if (b_row < N && b_col <= K - 4 /*vload4*/) \ + brow = vload4(0, src1_read00 + mad24(b_row, ldB, b_col)); \ + else \ + brow = (float4)0; \ + vstore4(brow, 0, slm_brow + mad24(_row, SLM_BLOCK, local_index)); \ + } + barrier(CLK_LOCAL_MEM_FENCE); - vstore4(vload4(0, src1_read0 + mad24(0, ldB, local_index)), 0, slm_brow + mad24(0, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(1, ldB, local_index)), 0, slm_brow + mad24(1, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(2, ldB, local_index)), 0, slm_brow + mad24(2, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(3, ldB, local_index)), 0, slm_brow + mad24(3, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(4, ldB, local_index)), 0, slm_brow + mad24(4, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(5, ldB, local_index)), 0, slm_brow + mad24(5, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(6, ldB, local_index)), 0, slm_brow + mad24(6, SLM_BLOCK, local_index)); - vstore4(vload4(0, src1_read0 + mad24(7, ldB, local_index)), 0, slm_brow + mad24(7, SLM_BLOCK, local_index)); + UPDATE_BROW(0); + UPDATE_BROW(1); + UPDATE_BROW(2); + UPDATE_BROW(3); + UPDATE_BROW(4); + UPDATE_BROW(5); + UPDATE_BROW(6); + UPDATE_BROW(7); barrier(CLK_LOCAL_MEM_FENCE); +#undef UPDATE_BROW - slm_brow0 = slm_brow + local_x * (TILE_K / 8); - w = b_tile; - int end_w = min(b_tile + SLM_BLOCK, K); - while( w + TILE_K <= end_w ) { - float4 arow; + for (int k_tile_offset = 0; k_tile_offset < SLM_BLOCK; k_tile_offset += TILE_K) + { + int a_col = a_col_base + b_tile + k_tile_offset; - brow0 = vload4(0, slm_brow0 + 0 * SLM_BLOCK); - brow1 = vload4(0, slm_brow0 + 1 * SLM_BLOCK); - brow2 = vload4(0, slm_brow0 + 2 * SLM_BLOCK); - brow3 = vload4(0, slm_brow0 + 3 * SLM_BLOCK); - brow4 = vload4(0, slm_brow0 + 4 * SLM_BLOCK); - brow5 = vload4(0, slm_brow0 + 5 * SLM_BLOCK); - brow6 = vload4(0, slm_brow0 + 6 * SLM_BLOCK); - brow7 = vload4(0, slm_brow0 + 7 * SLM_BLOCK); + if (a_col > K - 4 /*vload4*/) + break; -#define MM_DOT_PRODUCT(_row,_dot) \ - arow = vload4(0, src0_read + _row * ldA); \ - _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ - _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ - _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ - _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); + int slm_brow_col = a_col_base + k_tile_offset; // <= SLM_BLOCK - 4 +#define READ_SLM_BROW(_row) \ + float4 brow##_row = vload4(0, slm_brow + mad24(_row, SLM_BLOCK, slm_brow_col)); + + READ_SLM_BROW(0); + READ_SLM_BROW(1); + READ_SLM_BROW(2); + READ_SLM_BROW(3); + READ_SLM_BROW(4); + READ_SLM_BROW(5); + READ_SLM_BROW(6); + READ_SLM_BROW(7); +#undef READ_SLM_BROW + +#define MM_DOT_PRODUCT(_row,_dot) \ + { \ + int a_row = a_row_base + _row; \ + if (a_row < M) { \ + float4 arow = vload4(0, src0_read00 + mad24(a_row, ldA, a_col)); \ + _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ + _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ + _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ + _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); \ + } \ + } MM_DOT_PRODUCT(0,dot00); MM_DOT_PRODUCT(1,dot01); @@ -485,53 +514,7 @@ __kernel void intelblas_gemm_buffer_NT( MM_DOT_PRODUCT(6,dot06); MM_DOT_PRODUCT(7,dot07); #undef MM_DOT_PRODUCT - - src0_read += TILE_K; - slm_brow0 += TILE_K; - w += TILE_K; } - src1_read0 += SLM_BLOCK; - } - - if(w < K) { - float4 arow; - -#define READ_BROW(_brow,_row) \ - _brow = vload4(0, slm_brow0 + _row * SLM_BLOCK); \ - _brow.x = (mad24(local_x, 4, w) < K) ? _brow.x : 0.0f; \ - _brow.y = (mad24(local_x, 4, w + 1) < K) ? _brow.y : 0.0f; \ - _brow.z = (mad24(local_x, 4, w + 2) < K) ? _brow.z : 0.0f; \ - _brow.w = (mad24(local_x, 4, w + 3) < K) ? _brow.w : 0.0f; - - READ_BROW(brow0,0); - READ_BROW(brow1,1); - READ_BROW(brow2,2); - READ_BROW(brow3,3); - READ_BROW(brow4,4); - READ_BROW(brow5,5); - READ_BROW(brow6,6); - READ_BROW(brow7,7); - -#define MM_DOT_PRODUCT(_row,_dot) \ - arow = vload4(0, src0_read + _row * ldA); \ - arow.x = (mad24(local_x, 4, w) < K) ? arow.x : 0.0f; \ - arow.y = (mad24(local_x, 4, w + 1) < K) ? arow.y : 0.0f; \ - arow.z = (mad24(local_x, 4, w + 2) < K) ? arow.z : 0.0f; \ - arow.w = (mad24(local_x, 4, w + 3) < K) ? arow.w : 0.0f; \ - _dot = mad( (float8)(arow.x), (float8)(brow0.x, brow1.x, brow2.x, brow3.x, brow4.x, brow5.x, brow6.x, brow7.x), _dot ); \ - _dot = mad( (float8)(arow.y), (float8)(brow0.y, brow1.y, brow2.y, brow3.y, brow4.y, brow5.y, brow6.y, brow7.y), _dot ); \ - _dot = mad( (float8)(arow.z), (float8)(brow0.z, brow1.z, brow2.z, brow3.z, brow4.z, brow5.z, brow6.z, brow7.z), _dot ); \ - _dot = mad( (float8)(arow.w), (float8)(brow0.w, brow1.w, brow2.w, brow3.w, brow4.w, brow5.w, brow6.w, brow7.w), _dot ); - - MM_DOT_PRODUCT(0,dot00); - MM_DOT_PRODUCT(1,dot01); - MM_DOT_PRODUCT(2,dot02); - MM_DOT_PRODUCT(3,dot03); - MM_DOT_PRODUCT(4,dot04); - MM_DOT_PRODUCT(5,dot05); - MM_DOT_PRODUCT(6,dot06); - MM_DOT_PRODUCT(7,dot07); -#undef MM_DOT_PRODUCT } #define REDUCE(_dot) \ @@ -572,21 +555,22 @@ __kernel void intelblas_gemm_buffer_NT( output = (local_x == 5) ? _dot.s5 : output; \ output = (local_x == 6) ? _dot.s6 : output; \ output = (local_x == 7) ? _dot.s7 : output; \ - if (beta != 0.0) \ + if (beta != 0.0f) \ dst_write0[0] = mad(output, (float)alpha, ((float)beta * dst_write0[0])); \ else \ dst_write0[0] = output * (float)alpha; \ dst_write0 += ldC; - if(global_x < N && global_y * 8 < M) { - OUTPUT(dot00); - if(mad24(global_y, 8, 1) < M) { OUTPUT(dot01); } - if(mad24(global_y, 8, 2) < M) { OUTPUT(dot02); } - if(mad24(global_y, 8, 3) < M) { OUTPUT(dot03); } - if(mad24(global_y, 8, 4) < M) { OUTPUT(dot04); } - if(mad24(global_y, 8, 5) < M) { OUTPUT(dot05); } - if(mad24(global_y, 8, 6) < M) { OUTPUT(dot06); } - if(mad24(global_y, 8, 7) < M) { OUTPUT(dot07); } + if (global_x < N && dst_row < M) + { + /*if (dst_row + 0 < M)*/ { OUTPUT(dot00); } + if (dst_row + 1 < M) { OUTPUT(dot01); } + if (dst_row + 2 < M) { OUTPUT(dot02); } + if (dst_row + 3 < M) { OUTPUT(dot03); } + if (dst_row + 4 < M) { OUTPUT(dot04); } + if (dst_row + 5 < M) { OUTPUT(dot05); } + if (dst_row + 6 < M) { OUTPUT(dot06); } + if (dst_row + 7 < M) { OUTPUT(dot07); } } #undef OUTPUT } diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index 879d80cdb1..684a1a2ee3 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -56,9 +56,9 @@ #undef abs #endif -#if defined __linux__ || defined __APPLE__ || defined __GLIBC__ \ - || defined __HAIKU__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ \ - || defined __OpenBSD__ +#if defined __unix__ || defined __APPLE__ || defined __GLIBC__ \ + || defined __HAIKU__ || defined __EMSCRIPTEN__ \ + || defined __FreeBSD__ || defined __NetBSD__ || defined __OpenBSD__ #include #include #include @@ -72,7 +72,7 @@ #endif #endif -#if defined CV_CXX11 +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #include #endif @@ -884,9 +884,11 @@ T minNonZero(const T& val_1, const T& val_2) return (val_1 != 0) ? val_1 : val_2; } +#ifndef OPENCV_DISABLE_THREAD_SUPPORT static int getNumberOfCPUs_() { +#ifndef OPENCV_SEMIHOSTING /* * Logic here is to try different methods of getting CPU counts and return * the minimum most value as it has high probablity of being right and safe. @@ -978,6 +980,9 @@ int getNumberOfCPUs_() #endif return ncpus != 0 ? ncpus : 1; +#else // OPENCV_SEMIHOSTING + return 1; +#endif //OPENCV_SEMIHOSTING } int getNumberOfCPUs() @@ -986,6 +991,13 @@ int getNumberOfCPUs() return nCPUs; // cached value } +#else // OPENCV_DISABLE_THREAD_SUPPORT +int getNumberOfCPUs() +{ + return 1; +} +#endif // OPENCV_DISABLE_THREAD_SUPPORT + const char* currentParallelFramework() { std::shared_ptr& api = getCurrentParallelForAPI(); diff --git a/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp b/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp index a5649b60c0..b9b3105d20 100644 --- a/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp +++ b/modules/core/src/parallel/plugin_parallel_wrapper.impl.hpp @@ -223,6 +223,9 @@ std::vector getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_DEBUG(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } diff --git a/modules/core/src/parallel_impl.cpp b/modules/core/src/parallel_impl.cpp index 25bf4adce8..c118bcd3cb 100644 --- a/modules/core/src/parallel_impl.cpp +++ b/modules/core/src/parallel_impl.cpp @@ -580,6 +580,11 @@ void ThreadPool::run(const Range& range, const ParallelLoopBody& body, double ns { WorkerThread& thread = *(threads[i].get()); if ( +#if defined(__clang__) && defined(__has_feature) +#if __has_feature(thread_sanitizer) + 1 || // Robust workaround to avoid data race warning of `thread.job` +#endif +#endif #if !defined(CV_USE_GLOBAL_WORKERS_COND_VAR) thread.isActive || #endif diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 4bf52a3134..291931b5ae 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -4,6 +4,8 @@ #include "precomp.hpp" #include "persistence.hpp" +#include "persistence_impl.hpp" +#include "persistence_base64_encoding.hpp" #include #include @@ -143,17 +145,17 @@ static const char symbols[9] = "ucwsifdh"; static char typeSymbol(int depth) { CV_StaticAssert(CV_64F == 6, ""); - CV_Assert(depth >=0 && depth <= CV_64F); + CV_CheckDepth(depth, depth >=0 && depth <= CV_16F, ""); return symbols[depth]; } static int symbolToType(char c) { - const char* pos = strchr( symbols, c ); - if( !pos ) - CV_Error( CV_StsBadArg, "Invalid data type specification" ); if (c == 'r') return CV_SEQ_ELTYPE_PTR; + const char* pos = strchr( symbols, c ); + if( !pos ) + CV_Error( cv::Error::StsBadArg, "Invalid data type specification" ); return static_cast(pos - symbols); } @@ -192,7 +194,7 @@ int decodeFormat( const char* dt, int* fmt_pairs, int max_len ) } if( count <= 0 ) - CV_Error( CV_StsBadArg, "Invalid data type specification" ); + CV_Error( cv::Error::StsBadArg, "Invalid data type specification" ); fmt_pairs[i] = count; } @@ -208,7 +210,7 @@ int decodeFormat( const char* dt, int* fmt_pairs, int max_len ) { i += 2; if( i >= max_len ) - CV_Error( CV_StsBadArg, "Too long data type specification" ); + CV_Error( cv::Error::StsBadArg, "Too long data type specification" ); } fmt_pairs[i] = 0; } @@ -245,8 +247,12 @@ int calcStructSize( const char* dt, int initial_size ) { int size = calcElemSize( dt, initial_size ); size_t elem_max_size = 0; - for ( const char * type = dt; *type != '\0'; type++ ) { - switch ( *type ) + for ( const char * type = dt; *type != '\0'; type++ ) + { + char v = *type; + if (v >= '0' && v <= '9') + continue; // skip vector size + switch (v) { case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } @@ -255,7 +261,9 @@ int calcStructSize( const char* dt, int initial_size ) case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } - default: break; + case 'h': { elem_max_size = std::max(elem_max_size, sizeof(float16_t)); break; } + default: + CV_Error_(Error::StsNotImplemented, ("Unknown type identifier: '%c' in '%s'", (char)(*type), dt)); } } size = cvAlign( size, static_cast(elem_max_size) ); @@ -269,7 +277,7 @@ int decodeSimpleFormat( const char* dt ) fmt_pair_count = decodeFormat( dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS ); if( fmt_pair_count != 1 || fmt_pairs[0] >= CV_CN_MAX) - CV_Error( CV_StsError, "Too complex format for the matrix" ); + CV_Error( cv::Error::StsError, "Too complex format for the matrix" ); elem_type = CV_MAKETYPE( fmt_pairs[1], fmt_pairs[0] ); @@ -339,1449 +347,1483 @@ static inline void writeReal(uchar* p, double fval) #endif } -class FileStorage::Impl : public FileStorage_API -{ -public: - void init() - { - flags = 0; - buffer.clear(); - bufofs = 0; - state = UNDEFINED; - is_opened = false; - dummy_eof = false; - write_mode = false; - mem_mode = false; - space = 0; - wrap_margin = 71; - fmt = 0; - file = 0; - gzfile = 0; - empty_stream = true; - strbufv.clear(); - strbuf = 0; - strbufsize = strbufpos = 0; - roots.clear(); - fs_data.clear(); - fs_data_ptrs.clear(); - fs_data_blksz.clear(); - freeSpaceOfs = 0; - - str_hash.clear(); - str_hash_data.clear(); - str_hash_data.resize(1); - str_hash_data[0] = '\0'; - - filename.clear(); - lineno = 0; - } - - Impl(FileStorage* _fs) - { - fs_ext = _fs; - init(); - } - - virtual ~Impl() - { - release(); - } - - void release(String* out=0) - { - if( is_opened ) - { - if(out) - out->clear(); - if( write_mode ) - { - while( write_stack.size() > 1 ) - { - endWriteStruct(); - } - flush(); - if( fmt == FileStorage::FORMAT_XML ) - puts( "\n" ); - else if ( fmt == FileStorage::FORMAT_JSON ) - puts( "}\n" ); - } - if( mem_mode && out ) - { - *out = cv::String(outbuf.begin(), outbuf.end()); - } - } - closeFile(); - init(); - } - - void analyze_file_name( const std::string& file_name, std::vector& params ) - { - params.clear(); - static const char not_file_name = '\n'; - static const char parameter_begin = '?'; - static const char parameter_separator = '&'; - - if( file_name.find(not_file_name, (size_t)0) != std::string::npos ) - return; - - size_t beg = file_name.find_last_of(parameter_begin); - params.push_back(file_name.substr((size_t)0, beg)); - - if( beg != std::string::npos ) - { - size_t end = file_name.size(); - beg++; - for( size_t param_beg = beg, param_end = beg; - param_end < end; - param_beg = param_end + 1 ) - { - param_end = file_name.find_first_of( parameter_separator, param_beg ); - if( (param_end == std::string::npos || param_end != param_beg) && param_beg + 1 < end ) - { - params.push_back( file_name.substr( param_beg, param_end - param_beg ) ); - } - } - } - } - - bool open( const char* filename_or_buf, int _flags, const char* encoding ) - { - _flags &= ~FileStorage::BASE64; - - bool ok = true; - release(); - - bool append = (_flags & 3) == FileStorage::APPEND; - mem_mode = (_flags & FileStorage::MEMORY) != 0; - - write_mode = (_flags & 3) != 0; - - bool isGZ = false; - size_t fnamelen = 0; - - std::vector params; - //if ( !mem_mode ) - { - analyze_file_name( filename_or_buf, params ); - if( !params.empty() ) - filename = params[0]; - - /*if( !write_base64 && params.size() >= 2 && - std::find(params.begin()+1, params.end(), std::string("base64")) != params.end()) - write_base64 = (write_mode || append);*/ - } - - if( filename.size() == 0 && !mem_mode && !write_mode ) - CV_Error( CV_StsNullPtr, "NULL or empty filename" ); - - if( mem_mode && append ) - CV_Error( CV_StsBadFlag, "FileStorage::APPEND and FileStorage::MEMORY are not currently compatible" ); - - flags = _flags; - - if( !mem_mode ) - { - char* dot_pos = strrchr((char*)filename.c_str(), '.'); - char compression = '\0'; - - if( dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && - (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0')) ) - { - if( append ) - { - CV_Error(CV_StsNotImplemented, "Appending data to compressed file is not implemented" ); - } - isGZ = true; - compression = dot_pos[3]; - if( compression ) - dot_pos[3] = '\0', fnamelen--; - } - - if( !isGZ ) - { - file = fopen(filename.c_str(), !write_mode ? "rt" : !append ? "wt" : "a+t" ); - if( !file ) - return false; - } - else - { -#if USE_ZLIB - char mode[] = { write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0' }; - gzfile = gzopen(filename.c_str(), mode); - if( !gzfile ) - return false; -#else - CV_Error(CV_StsNotImplemented, "There is no compressed file storage support in this configuration"); -#endif - } - } - - roots.clear(); - fs_data.clear(); - wrap_margin = 71; - fmt = FileStorage::FORMAT_AUTO; - - if( write_mode ) - { - fmt = flags & FileStorage::FORMAT_MASK; - - if( mem_mode ) - outbuf.clear(); - - if( fmt == FileStorage::FORMAT_AUTO && !filename.empty() ) - { - const char* dot_pos = NULL; - const char* dot_pos2 = NULL; - // like strrchr() implementation, but save two last positions simultaneously - for (const char* pos = &filename[0]; pos[0] != 0; pos++) - { - if( pos[0] == '.' ) - { - dot_pos2 = dot_pos; - dot_pos = pos; - } - } - if (fs::strcasecmp(dot_pos, ".gz") == 0 && dot_pos2 != NULL) - { - dot_pos = dot_pos2; - } - fmt = (fs::strcasecmp(dot_pos, ".xml") == 0 || fs::strcasecmp(dot_pos, ".xml.gz") == 0 ) - ? FileStorage::FORMAT_XML - : (fs::strcasecmp(dot_pos, ".json") == 0 || fs::strcasecmp(dot_pos, ".json.gz") == 0) - ? FileStorage::FORMAT_JSON - : FileStorage::FORMAT_YAML; - } - else if( fmt == FileStorage::FORMAT_AUTO ) - { - fmt = FileStorage::FORMAT_XML; - } - - // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") - // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) - int buf_size = CV_FS_MAX_LEN*(fmt == FileStorage::FORMAT_XML ? 6 : 4) + 1024; - - if (append) - { - fseek( file, 0, SEEK_END ); - if (ftell(file) == 0) - append = false; - } - - write_stack.clear(); - empty_stream = true; - write_stack.push_back(FStructData("", FileNode::MAP | FileNode::EMPTY, 0)); - buffer.reserve(buf_size + 1024); - buffer.resize(buf_size); - bufofs = 0; - - if( fmt == FileStorage::FORMAT_XML ) - { - size_t file_size = file ? (size_t)ftell(file) : (size_t)0; - if( !append || file_size == 0 ) - { - if( encoding && *encoding != '\0' ) - { - if( fs::strcasecmp(encoding, "UTF-16" ) == 0 ) - { - release(); - CV_Error( CV_StsBadArg, "UTF-16 XML encoding is not supported! Use 8-bit encoding\n"); - } - - CV_Assert( strlen(encoding) < 1000 ); - char buf[1100]; - sprintf(buf, "\n", encoding); - puts( buf ); - } - else - puts( "\n" ); - puts( "\n" ); - } - else - { - int xml_buf_size = 1 << 10; - char substr[] = ""; - int last_occurrence = -1; - xml_buf_size = MIN(xml_buf_size, int(file_size)); - fseek( file, -xml_buf_size, SEEK_END ); - // find the last occurrence of - for(;;) - { - int line_offset = (int)ftell( file ); - const char* ptr0 = this->gets(xml_buf_size); - const char* ptr = NULL; - if( !ptr0 ) - break; - ptr = ptr0; - for(;;) - { - ptr = strstr( ptr, substr ); - if( !ptr ) - break; - last_occurrence = line_offset + (int)(ptr - ptr0); - ptr += strlen(substr); - } - } - if( last_occurrence < 0 ) - { - release(); - CV_Error( CV_StsError, "Could not find in the end of file.\n" ); - } - closeFile(); - file = fopen( filename.c_str(), "r+t" ); - CV_Assert(file != 0); - fseek( file, last_occurrence, SEEK_SET ); - // replace the last "" with " ", which has the same length - puts( " " ); - fseek( file, 0, SEEK_END ); - puts( "\n" ); - } - - emitter = createXMLEmitter(this); - } - else if( fmt == FileStorage::FORMAT_YAML ) - { - if( !append) - puts( "%YAML:1.0\n---\n" ); - else - puts( "...\n---\n" ); - - emitter = createYAMLEmitter(this); - } - else - { - CV_Assert( fmt == FileStorage::FORMAT_JSON ); - if( !append ) - puts( "{\n" ); - else - { - bool valid = false; - long roffset = 0; - for ( ; - fseek( file, roffset, SEEK_END ) == 0; - roffset -= 1 ) - { - const char end_mark = '}'; - if ( fgetc( file ) == end_mark ) - { - fseek( file, roffset, SEEK_END ); - valid = true; - break; - } - } - - if ( valid ) - { - closeFile(); - file = fopen( filename.c_str(), "r+t" ); - CV_Assert(file != 0); - fseek( file, roffset, SEEK_END ); - fputs( ",", file ); - } - else - { - CV_Error( CV_StsError, "Could not find '}' in the end of file.\n" ); - } - } - write_stack.back().indent = 4; - emitter = createJSONEmitter(this); - } - is_opened = true; - } - else - { - const size_t buf_size0 = 40; - buffer.resize(buf_size0); - if( mem_mode ) - { - strbuf = (char*)filename_or_buf; - strbufsize = strlen(strbuf); - } - - const char* yaml_signature = "%YAML"; - const char* json_signature = "{"; - const char* xml_signature = "gets(16); - CV_Assert(buf); - char* bufPtr = cv_skip_BOM(buf); - size_t bufOffset = bufPtr - buf; - - if(strncmp( bufPtr, yaml_signature, strlen(yaml_signature) ) == 0) - fmt = FileStorage::FORMAT_YAML; - else if(strncmp( bufPtr, json_signature, strlen(json_signature) ) == 0) - fmt = FileStorage::FORMAT_JSON; - else if(strncmp( bufPtr, xml_signature, strlen(xml_signature) ) == 0) - fmt = FileStorage::FORMAT_XML; - else if(strbufsize == bufOffset) - CV_Error(CV_BADARG_ERR, "Input file is invalid"); - else - CV_Error(CV_BADARG_ERR, "Unsupported file storage format"); - - rewind(); - strbufpos = bufOffset; - bufofs = 0; - - try - { - char* ptr = bufferStart(); - ptr[0] = ptr[1] = ptr[2] = '\0'; - FileNode root_nodes(fs_ext, 0, 0); - - uchar* rptr = reserveNodeSpace(root_nodes, 9); - *rptr = FileNode::SEQ; - writeInt(rptr + 1, 4); - writeInt(rptr + 5, 0); - - roots.clear(); - - switch (fmt) - { - case FileStorage::FORMAT_XML: parser = createXMLParser(this); break; - case FileStorage::FORMAT_YAML: parser = createYAMLParser(this); break; - case FileStorage::FORMAT_JSON: parser = createJSONParser(this); break; - default: parser = Ptr(); - } - - if( !parser.empty() ) - { - ok = parser->parse(ptr); - if( ok ) - { - finalizeCollection(root_nodes); - - CV_Assert( !fs_data_ptrs.empty() ); - FileNode roots_node(fs_ext, 0, 0); - size_t i, nroots = roots_node.size(); - FileNodeIterator it = roots_node.begin(); - - for( i = 0; i < nroots; i++, ++it ) - roots.push_back(*it); - } - } - } - catch(...) - { - is_opened = true; - release(); - throw; - } - - // release resources that we do not need anymore - closeFile(); - is_opened = true; - std::vector tmpbuf; - std::swap(buffer, tmpbuf); - bufofs = 0; - } - return ok; - } - - void puts( const char* str ) - { - CV_Assert( write_mode ); - if( mem_mode ) - std::copy(str, str + strlen(str), std::back_inserter(outbuf)); - else if( file ) - fputs( str, file ); -#if USE_ZLIB - else if( gzfile ) - gzputs( gzfile, str ); -#endif - else - CV_Error( CV_StsError, "The storage is not opened" ); - } - - char* getsFromFile( char* buf, int count ) - { - if( file ) - return fgets( buf, count, file ); - #if USE_ZLIB - if( gzfile ) - return gzgets( gzfile, buf, count ); - #endif - CV_Error(CV_StsError, "The storage is not opened"); - } - - char* gets( size_t maxCount ) - { - if( strbuf ) - { - size_t i = strbufpos, len = strbufsize; - const char* instr = strbuf; - for( ; i < len; i++ ) - { - char c = instr[i]; - if( c == '\0' || c == '\n' ) - { - if( c == '\n' ) - i++; - break; - } - } - size_t count = i - strbufpos; - if( maxCount == 0 || maxCount > count ) - maxCount = count; - buffer.resize(std::max(buffer.size(), maxCount + 8)); - memcpy(&buffer[0], instr + strbufpos, maxCount); - buffer[maxCount] = '\0'; - strbufpos = i; - return maxCount > 0 ? &buffer[0] : 0; - } - - const size_t MAX_BLOCK_SIZE = INT_MAX/2; // hopefully, that will be enough - if( maxCount == 0 ) - maxCount = MAX_BLOCK_SIZE; - else - CV_Assert(maxCount < MAX_BLOCK_SIZE); - size_t ofs = 0; - - for(;;) - { - int count = (int)std::min(buffer.size() - ofs - 16, maxCount); - char* ptr = getsFromFile( &buffer[ofs], count+1 ); - if( !ptr ) - break; - int delta = (int)strlen(ptr); - ofs += delta; - maxCount -= delta; - if( ptr[delta-1] == '\n' || maxCount == 0 ) - break; - if( delta == count ) - buffer.resize((size_t)(buffer.size()*1.5)); - } - return ofs > 0 ? &buffer[0] : 0; - } - - char* gets() - { - char* ptr = this->gets(0); - if( !ptr ) - { - ptr = bufferStart(); // FIXIT Why do we need this hack? What is about other parsers JSON/YAML? - *ptr = '\0'; - setEof(); - return 0; - } - else - { - size_t l = strlen(ptr); - if( l > 0 && ptr[l-1] != '\n' && ptr[l-1] != '\r' && !eof() ) - { - ptr[l] = '\n'; - ptr[l+1] = '\0'; - } - } - lineno++; - return ptr; - } - - bool eof() - { - if( dummy_eof ) - return true; - if( strbuf ) - return strbufpos >= strbufsize; - if( file ) - return feof(file) != 0; -#if USE_ZLIB - if( gzfile ) - return gzeof(gzfile) != 0; -#endif - return false; - } - - void setEof() - { - dummy_eof = true; - } - - void closeFile() - { - if( file ) - fclose( file ); -#if USE_ZLIB - else if( gzfile ) - gzclose( gzfile ); -#endif - file = 0; - gzfile = 0; - strbuf = 0; - strbufpos = 0; - is_opened = false; - } - - void rewind() - { - if( file ) - ::rewind(file); -#if USE_ZLIB - else if( gzfile ) - gzrewind(gzfile); -#endif - strbufpos = 0; - } - - char* resizeWriteBuffer( char* ptr, int len ) - { - const char* buffer_end = &buffer[0] + buffer.size(); - if( ptr + len < buffer_end ) - return ptr; - - const char* buffer_start = &buffer[0]; - int written_len = (int)(ptr - buffer_start); - - CV_Assert(written_len <= (int)buffer.size()); - int new_size = (int)((buffer_end - buffer_start)*3/2); - new_size = MAX( written_len + len, new_size ); - buffer.reserve( new_size + 256 ); - buffer.resize( new_size ); - bufofs = written_len; - return &buffer[0] + bufofs; - } - - char* flush() - { - char* buffer_start = &buffer[0]; - char* ptr = buffer_start + bufofs; - - if( ptr > buffer_start + space ) - { - ptr[0] = '\n'; - ptr[1] = '\0'; - puts( buffer_start ); - bufofs = 0; - } - - int indent = write_stack.back().indent; - - if( space != indent ) - { - memset( buffer_start, ' ', indent ); - space = indent; - } - bufofs = space; - ptr = buffer_start + bufofs; - - return ptr; - } - - void endWriteStruct() - { - CV_Assert( write_mode ); - CV_Assert( !write_stack.empty() ); - - FStructData& current_struct = write_stack.back(); - if( fmt == FileStorage::FORMAT_JSON && !FileNode::isFlow(current_struct.flags) && write_stack.size() > 1 ) - current_struct.indent = write_stack[write_stack.size() - 2].indent; - - emitter->endWriteStruct(current_struct); - - write_stack.pop_back(); - if( !write_stack.empty() ) - write_stack.back().flags &= ~FileNode::EMPTY; - } - - void startWriteStruct( const char* key, int struct_flags, - const char* type_name ) - { - CV_Assert( write_mode ); - - struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; - if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, - "Some collection type: FileNode::SEQ or FileNode::MAP must be specified" ); - - if( type_name && type_name[0] == '\0' ) - type_name = 0; - - FStructData s = emitter->startWriteStruct( write_stack.back(), key, struct_flags, type_name ); - write_stack.push_back(s); - size_t write_stack_size = write_stack.size(); - if( write_stack_size > 1 ) - write_stack[write_stack_size-2].flags &= ~FileNode::EMPTY; - - if( !FileNode::isFlow(s.flags) ) - flush(); - - if( fmt == FileStorage::FORMAT_JSON && type_name && type_name[0] && FileNode::isMap(struct_flags)) - { - emitter->write("type_id", type_name, false); - } - } - - void writeComment( const char* comment, bool eol_comment ) - { - CV_Assert(write_mode); - emitter->writeComment( comment, eol_comment ); - } - - void startNextStream() - { - CV_Assert(write_mode); - if( !empty_stream ) - { - while( !write_stack.empty() ) +void FileStorage::Impl::init() { + flags = 0; + buffer.clear(); + bufofs = 0; + state = UNDEFINED; + is_using_base64 = false; + state_of_writing_base64 = FileStorage_API::Base64State::Uncertain; + is_write_struct_delayed = false; + delayed_struct_key = nullptr; + delayed_struct_flags = 0; + delayed_type_name = nullptr; + base64_writer = nullptr; + is_opened = false; + dummy_eof = false; + write_mode = false; + mem_mode = false; + space = 0; + wrap_margin = 71; + fmt = 0; + file = 0; + gzfile = 0; + empty_stream = true; + + strbufv.clear(); + strbuf = 0; + strbufsize = strbufpos = 0; + roots.clear(); + + fs_data.clear(); + fs_data_ptrs.clear(); + fs_data_blksz.clear(); + freeSpaceOfs = 0; + + str_hash.clear(); + str_hash_data.clear(); + str_hash_data.resize(1); + str_hash_data[0] = '\0'; + + filename.clear(); + lineno = 0; +} + +FileStorage::Impl::Impl(FileStorage *_fs) { + fs_ext = _fs; + init(); +} + +FileStorage::Impl::~Impl() { + release(); +} + +void FileStorage::Impl::release(String *out) { + if (is_opened) { + if (out) + out->clear(); + if (write_mode) { + while (write_stack.size() > 1) { endWriteStruct(); + } flush(); - emitter->startNextStream(); - empty_stream = true; - write_stack.push_back(FStructData("", FileNode::EMPTY, 0)); - bufofs = 0; + if (fmt == FileStorage::FORMAT_XML) + puts("\n"); + else if (fmt == FileStorage::FORMAT_JSON) + puts("}\n"); + } + if (mem_mode && out) { + *out = cv::String(outbuf.begin(), outbuf.end()); + } + } + closeFile(); + init(); +} + +void FileStorage::Impl::analyze_file_name(const std::string &file_name, std::vector ¶ms) { + params.clear(); + static const char not_file_name = '\n'; + static const char parameter_begin = '?'; + static const char parameter_separator = '&'; + + if (file_name.find(not_file_name, (size_t) 0) != std::string::npos) + return; + + size_t beg = file_name.find_last_of(parameter_begin); + params.push_back(file_name.substr((size_t) 0, beg)); + + if (beg != std::string::npos) { + size_t end = file_name.size(); + beg++; + for (size_t param_beg = beg, param_end = beg; + param_end < end; + param_beg = param_end + 1) { + param_end = file_name.find_first_of(parameter_separator, param_beg); + if ((param_end == std::string::npos || param_end != param_beg) && param_beg + 1 < end) { + params.push_back(file_name.substr(param_beg, param_end - param_beg)); + } + } + } +} + +bool FileStorage::Impl::open(const char *filename_or_buf, int _flags, const char *encoding) { + bool ok = true; + release(); + + bool append = (_flags & 3) == FileStorage::APPEND; + mem_mode = (_flags & FileStorage::MEMORY) != 0; + + write_mode = (_flags & 3) != 0; + bool write_base64 = (write_mode || append) && (_flags & FileStorage::BASE64) != 0; + + bool isGZ = false; + size_t fnamelen = 0; + + std::vector params; + //if ( !mem_mode ) + { + analyze_file_name(filename_or_buf, params); + if (!params.empty()) + filename = params[0]; + + if (!write_base64 && params.size() >= 2 && + std::find(params.begin() + 1, params.end(), std::string("base64")) != params.end()) + write_base64 = (write_mode || append); + } + + if (filename.size() == 0 && !mem_mode && !write_mode) + CV_Error(cv::Error::StsNullPtr, "NULL or empty filename"); + + if (mem_mode && append) + CV_Error(cv::Error::StsBadFlag, "FileStorage::APPEND and FileStorage::MEMORY are not currently compatible"); + + flags = _flags; + + if (!mem_mode) { + char *dot_pos = strrchr((char *) filename.c_str(), '.'); + char compression = '\0'; + + if (dot_pos && dot_pos[1] == 'g' && dot_pos[2] == 'z' && + (dot_pos[3] == '\0' || (cv_isdigit(dot_pos[3]) && dot_pos[4] == '\0'))) { + if (append) { + CV_Error(cv::Error::StsNotImplemented, "Appending data to compressed file is not implemented"); + } + isGZ = true; + compression = dot_pos[3]; + if (compression) + dot_pos[3] = '\0', fnamelen--; + } + + if (!isGZ) { + file = fopen(filename.c_str(), !write_mode ? "rt" : !append ? "wt" : "a+t"); + if (!file) + return false; + } else { +#if USE_ZLIB + char mode[] = {write_mode ? 'w' : 'r', 'b', compression ? compression : '3', '\0'}; + gzfile = gzopen(filename.c_str(), mode); + if (!gzfile) + return false; +#else + CV_Error(cv::Error::StsNotImplemented, "There is no compressed file storage support in this configuration"); +#endif } } - void write( const String& key, int value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value); - } + roots.clear(); + fs_data.clear(); + wrap_margin = 71; + fmt = FileStorage::FORMAT_AUTO; - void write( const String& key, double value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value); - } + if (write_mode) { + fmt = flags & FileStorage::FORMAT_MASK; - void write( const String& key, const String& value ) - { - CV_Assert(write_mode); - emitter->write(key.c_str(), value.c_str(), false); - } + if (mem_mode) + outbuf.clear(); - void writeRawData( const std::string& dt, const void* _data, size_t len ) - { - CV_Assert(write_mode); - - size_t elemSize = fs::calcStructSize(dt.c_str(), 0); - CV_Assert( len % elemSize == 0 ); - len /= elemSize; - - bool explicitZero = fmt == FileStorage::FORMAT_JSON; - const uchar* data0 = (const uchar*)_data; - int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2], k, fmt_pair_count; - char buf[256] = ""; - - fmt_pair_count = fs::decodeFormat( dt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS ); - - if( !len ) - return; - - if( !data0 ) - CV_Error( CV_StsNullPtr, "Null data pointer" ); - - if( fmt_pair_count == 1 ) - { - fmt_pairs[0] *= (int)len; - len = 1; + if (fmt == FileStorage::FORMAT_AUTO && !filename.empty()) { + const char *dot_pos = NULL; + const char *dot_pos2 = NULL; + // like strrchr() implementation, but save two last positions simultaneously + for (const char *pos = &filename[0]; pos[0] != 0; pos++) { + if (pos[0] == '.') { + dot_pos2 = dot_pos; + dot_pos = pos; + } + } + if (fs::strcasecmp(dot_pos, ".gz") == 0 && dot_pos2 != NULL) { + dot_pos = dot_pos2; + } + fmt = (fs::strcasecmp(dot_pos, ".xml") == 0 || fs::strcasecmp(dot_pos, ".xml.gz") == 0) + ? FileStorage::FORMAT_XML + : (fs::strcasecmp(dot_pos, ".json") == 0 || fs::strcasecmp(dot_pos, ".json.gz") == 0) + ? FileStorage::FORMAT_JSON + : FileStorage::FORMAT_YAML; + } else if (fmt == FileStorage::FORMAT_AUTO) { + fmt = FileStorage::FORMAT_XML; } - for(;len--; data0 += elemSize) - { - int offset = 0; - for( k = 0; k < fmt_pair_count; k++ ) - { - int i, count = fmt_pairs[k*2]; - int elem_type = fmt_pairs[k*2+1]; - int elem_size = CV_ELEM_SIZE(elem_type); - const char *ptr; + // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") + // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) + int buf_size = CV_FS_MAX_LEN * (fmt == FileStorage::FORMAT_XML ? 6 : 4) + 1024; - offset = cvAlign( offset, elem_size ); - const uchar* data = data0 + offset; + if (append) { + fseek(file, 0, SEEK_END); + if (ftell(file) == 0) + append = false; + } - for( i = 0; i < count; i++ ) - { - switch( elem_type ) - { + write_stack.clear(); + empty_stream = true; + write_stack.push_back(FStructData("", FileNode::MAP | FileNode::EMPTY, 0)); + buffer.reserve(buf_size + 1024); + buffer.resize(buf_size); + bufofs = 0; + is_using_base64 = write_base64; + state_of_writing_base64 = FileStorage_API::Base64State::Uncertain; + + if (fmt == FileStorage::FORMAT_XML) { + size_t file_size = file ? (size_t) ftell(file) : (size_t) 0; + if (!append || file_size == 0) { + if (encoding && *encoding != '\0') { + if (fs::strcasecmp(encoding, "UTF-16") == 0) { + release(); + CV_Error(cv::Error::StsBadArg, "UTF-16 XML encoding is not supported! Use 8-bit encoding\n"); + } + + CV_Assert(strlen(encoding) < 1000); + char buf[1100]; + sprintf(buf, "\n", encoding); + puts(buf); + } else + puts("\n"); + puts("\n"); + } else { + int xml_buf_size = 1 << 10; + char substr[] = ""; + int last_occurrence = -1; + xml_buf_size = MIN(xml_buf_size, int(file_size)); + fseek(file, -xml_buf_size, SEEK_END); + // find the last occurrence of + for (;;) { + int line_offset = (int) ftell(file); + const char *ptr0 = this->gets(xml_buf_size); + const char *ptr = NULL; + if (!ptr0) + break; + ptr = ptr0; + for (;;) { + ptr = strstr(ptr, substr); + if (!ptr) + break; + last_occurrence = line_offset + (int) (ptr - ptr0); + ptr += strlen(substr); + } + } + if (last_occurrence < 0) { + release(); + CV_Error(cv::Error::StsError, "Could not find in the end of file.\n"); + } + closeFile(); + file = fopen(filename.c_str(), "r+t"); + CV_Assert(file != 0); + fseek(file, last_occurrence, SEEK_SET); + // replace the last "" with " ", which has the same length + puts(" "); + fseek(file, 0, SEEK_END); + puts("\n"); + } + + emitter = createXMLEmitter(this); + } else if (fmt == FileStorage::FORMAT_YAML) { + if (!append) + puts("%YAML:1.0\n---\n"); + else + puts("...\n---\n"); + + emitter = createYAMLEmitter(this); + } else { + CV_Assert(fmt == FileStorage::FORMAT_JSON); + if (!append) + puts("{\n"); + else { + bool valid = false; + long roffset = 0; + for (; + fseek(file, roffset, SEEK_END) == 0; + roffset -= 1) { + const char end_mark = '}'; + if (fgetc(file) == end_mark) { + fseek(file, roffset, SEEK_END); + valid = true; + break; + } + } + + if (valid) { + closeFile(); + file = fopen(filename.c_str(), "r+t"); + CV_Assert(file != 0); + fseek(file, roffset, SEEK_END); + fputs(",", file); + } else { + CV_Error(cv::Error::StsError, "Could not find '}' in the end of file.\n"); + } + } + write_stack.back().indent = 4; + emitter = createJSONEmitter(this); + } + is_opened = true; + } else { + const size_t buf_size0 = 40; + buffer.resize(buf_size0); + if (mem_mode) { + strbuf = (char *) filename_or_buf; + strbufsize = strlen(strbuf); + } + + const char *yaml_signature = "%YAML"; + const char *json_signature = "{"; + const char *xml_signature = "gets(16); + CV_Assert(buf); + char *bufPtr = cv_skip_BOM(buf); + size_t bufOffset = bufPtr - buf; + + if (strncmp(bufPtr, yaml_signature, strlen(yaml_signature)) == 0) + fmt = FileStorage::FORMAT_YAML; + else if (strncmp(bufPtr, json_signature, strlen(json_signature)) == 0) + fmt = FileStorage::FORMAT_JSON; + else if (strncmp(bufPtr, xml_signature, strlen(xml_signature)) == 0) + fmt = FileStorage::FORMAT_XML; + else if (strbufsize == bufOffset) + CV_Error(cv::Error::StsBadArg, "Input file is invalid"); + else + CV_Error(cv::Error::StsBadArg, "Unsupported file storage format"); + + rewind(); + strbufpos = bufOffset; + bufofs = 0; + + try { + char *ptr = bufferStart(); + ptr[0] = ptr[1] = ptr[2] = '\0'; + FileNode root_nodes(fs_ext, 0, 0); + + uchar *rptr = reserveNodeSpace(root_nodes, 9); + *rptr = FileNode::SEQ; + writeInt(rptr + 1, 4); + writeInt(rptr + 5, 0); + + roots.clear(); + + switch (fmt) { + case FileStorage::FORMAT_XML: + parser = createXMLParser(this); + break; + case FileStorage::FORMAT_YAML: + parser = createYAMLParser(this); + break; + case FileStorage::FORMAT_JSON: + parser = createJSONParser(this); + break; + default: + parser = Ptr(); + } + + if (!parser.empty()) { + ok = parser->parse(ptr); + if (ok) { + finalizeCollection(root_nodes); + + CV_Assert(!fs_data_ptrs.empty()); + FileNode roots_node(fs_ext, 0, 0); + size_t i, nroots = roots_node.size(); + FileNodeIterator it = roots_node.begin(); + + for (i = 0; i < nroots; i++, ++it) + roots.push_back(*it); + } + } + } + catch (...) { + is_opened = true; + release(); + throw; + } + + // release resources that we do not need anymore + closeFile(); + is_opened = true; + std::vector tmpbuf; + std::swap(buffer, tmpbuf); + bufofs = 0; + } + return ok; +} + +void FileStorage::Impl::puts(const char *str) { + CV_Assert(write_mode); + if (mem_mode) + std::copy(str, str + strlen(str), std::back_inserter(outbuf)); + else if (file) + fputs(str, file); +#if USE_ZLIB + else if (gzfile) + gzputs(gzfile, str); +#endif + else + CV_Error(cv::Error::StsError, "The storage is not opened"); +} + +char *FileStorage::Impl::getsFromFile(char *buf, int count) { + if (file) + return fgets(buf, count, file); +#if USE_ZLIB + if (gzfile) + return gzgets(gzfile, buf, count); +#endif + CV_Error(cv::Error::StsError, "The storage is not opened"); +} + +char *FileStorage::Impl::gets(size_t maxCount) { + if (strbuf) { + size_t i = strbufpos, len = strbufsize; + const char *instr = strbuf; + for (; i < len; i++) { + char c = instr[i]; + if (c == '\0' || c == '\n') { + if (c == '\n') + i++; + break; + } + } + size_t count = i - strbufpos; + if (maxCount == 0 || maxCount > count) + maxCount = count; + buffer.resize(std::max(buffer.size(), maxCount + 8)); + memcpy(&buffer[0], instr + strbufpos, maxCount); + buffer[maxCount] = '\0'; + strbufpos = i; + return maxCount > 0 ? &buffer[0] : 0; + } + + const size_t MAX_BLOCK_SIZE = INT_MAX / 2; // hopefully, that will be enough + if (maxCount == 0) + maxCount = MAX_BLOCK_SIZE; + else + CV_Assert(maxCount < MAX_BLOCK_SIZE); + size_t ofs = 0; + + for (;;) { + int count = (int) std::min(buffer.size() - ofs - 16, maxCount); + char *ptr = getsFromFile(&buffer[ofs], count + 1); + if (!ptr) + break; + int delta = (int) strlen(ptr); + ofs += delta; + maxCount -= delta; + if (ptr[delta - 1] == '\n' || maxCount == 0) + break; + if (delta == count) + buffer.resize((size_t) (buffer.size() * 1.5)); + } + return ofs > 0 ? &buffer[0] : 0; +} + +char *FileStorage::Impl::gets() { + char *ptr = this->gets(0); + if (!ptr) { + ptr = bufferStart(); // FIXIT Why do we need this hack? What is about other parsers JSON/YAML? + *ptr = '\0'; + setEof(); + return 0; + } else { + size_t l = strlen(ptr); + if (l > 0 && ptr[l - 1] != '\n' && ptr[l - 1] != '\r' && !eof()) { + ptr[l] = '\n'; + ptr[l + 1] = '\0'; + } + } + lineno++; + return ptr; +} + +bool FileStorage::Impl::eof() { + if (dummy_eof) + return true; + if (strbuf) + return strbufpos >= strbufsize; + if (file) + return feof(file) != 0; +#if USE_ZLIB + if (gzfile) + return gzeof(gzfile) != 0; +#endif + return false; +} + +void FileStorage::Impl::setEof() { + dummy_eof = true; +} + +void FileStorage::Impl::closeFile() { + if (file) + fclose(file); +#if USE_ZLIB + else if (gzfile) + gzclose(gzfile); +#endif + file = 0; + gzfile = 0; + strbuf = 0; + strbufpos = 0; + is_opened = false; +} + +void FileStorage::Impl::rewind() { + if (file) + ::rewind(file); +#if USE_ZLIB + else if (gzfile) + gzrewind(gzfile); +#endif + strbufpos = 0; +} + +char *FileStorage::Impl::resizeWriteBuffer(char *ptr, int len) { + const char *buffer_end = &buffer[0] + buffer.size(); + if (ptr + len < buffer_end) + return ptr; + + const char *buffer_start = &buffer[0]; + int written_len = (int) (ptr - buffer_start); + + CV_Assert(written_len <= (int) buffer.size()); + int new_size = (int) ((buffer_end - buffer_start) * 3 / 2); + new_size = MAX(written_len + len, new_size); + buffer.reserve(new_size + 256); + buffer.resize(new_size); + bufofs = written_len; + return &buffer[0] + bufofs; +} + +char *FileStorage::Impl::flush() { + char *buffer_start = &buffer[0]; + char *ptr = buffer_start + bufofs; + + if (ptr > buffer_start + space) { + ptr[0] = '\n'; + ptr[1] = '\0'; + puts(buffer_start); + bufofs = 0; + } + + int indent = write_stack.back().indent; + + if (space != indent) { + memset(buffer_start, ' ', indent); + space = indent; + } + bufofs = space; + ptr = buffer_start + bufofs; + + return ptr; +} + +void FileStorage::Impl::endWriteStruct() { + CV_Assert(write_mode); + + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + + CV_Assert(!write_stack.empty()); + + FStructData ¤t_struct = write_stack.back(); + if (fmt == FileStorage::FORMAT_JSON && !FileNode::isFlow(current_struct.flags) && write_stack.size() > 1) + current_struct.indent = write_stack[write_stack.size() - 2].indent; + + emitter->endWriteStruct(current_struct); + + write_stack.pop_back(); + if (!write_stack.empty()) + write_stack.back().flags &= ~FileNode::EMPTY; +} + +void FileStorage::Impl::startWriteStruct_helper(const char *key, int struct_flags, + const char *type_name) { + CV_Assert(write_mode); + + struct_flags = (struct_flags & (FileNode::TYPE_MASK | FileNode::FLOW)) | FileNode::EMPTY; + if (!FileNode::isCollection(struct_flags)) + CV_Error(cv::Error::StsBadArg, + "Some collection type: FileNode::SEQ or FileNode::MAP must be specified"); + + if (type_name && type_name[0] == '\0') + type_name = 0; + + FStructData s = emitter->startWriteStruct(write_stack.back(), key, struct_flags, type_name); + + write_stack.push_back(s); + size_t write_stack_size = write_stack.size(); + if (write_stack_size > 1) + write_stack[write_stack_size - 2].flags &= ~FileNode::EMPTY; + + if (fmt != FileStorage::FORMAT_JSON && !FileNode::isFlow(s.flags)) + flush(); + + if (fmt == FileStorage::FORMAT_JSON && type_name && type_name[0] && FileNode::isMap(struct_flags)) { + emitter->write("type_id", type_name, false); + } +} + +void FileStorage::Impl::startWriteStruct(const char *key, int struct_flags, + const char *type_name) { + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 == FileStorage_API::NotUse) + switch_to_Base64_state(FileStorage_API::Uncertain); + + if (state_of_writing_base64 == FileStorage_API::Uncertain && FileNode::isSeq(struct_flags) + && is_using_base64 && type_name == 0) { + /* Uncertain whether output Base64 data */ + make_write_struct_delayed(key, struct_flags, type_name); + } else if (type_name && memcmp(type_name, "binary", 6) == 0) { + /* Must output Base64 data */ + if ((FileNode::TYPE_MASK & struct_flags) != FileNode::SEQ) + CV_Error(cv::Error::StsBadArg, "must set 'struct_flags |= CV_NODE_SEQ' if using Base64."); + else if (state_of_writing_base64 != FileStorage_API::Uncertain) + CV_Error(cv::Error::StsError, "function \'cvStartWriteStruct\' calls cannot be nested if using Base64."); + + startWriteStruct_helper(key, struct_flags, "binary"); + + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::InUse); + } else { + /* Won't output Base64 data */ + if (state_of_writing_base64 == FileStorage_API::InUse) + CV_Error(cv::Error::StsError, "At the end of the output Base64, `cvEndWriteStruct` is needed."); + + startWriteStruct_helper(key, struct_flags, type_name); + + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::NotUse); + } +} + +void FileStorage::Impl::writeComment(const char *comment, bool eol_comment) { + CV_Assert(write_mode); + emitter->writeComment(comment, eol_comment); +} + +void FileStorage::Impl::startNextStream() { + CV_Assert(write_mode); + if (!empty_stream) { + while (!write_stack.empty()) + endWriteStruct(); + flush(); + emitter->startNextStream(); + empty_stream = true; + write_stack.push_back(FStructData("", FileNode::EMPTY, 0)); + bufofs = 0; + } +} + +void FileStorage::Impl::write(const String &key, int value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value); +} + +void FileStorage::Impl::write(const String &key, double value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value); +} + +void FileStorage::Impl::write(const String &key, const String &value) { + CV_Assert(write_mode); + emitter->write(key.c_str(), value.c_str(), false); +} + +void FileStorage::Impl::writeRawData(const std::string &dt, const void *_data, size_t len) { + CV_Assert(write_mode); + + if (is_using_base64 || state_of_writing_base64 == FileStorage_API::Base64State::InUse) { + writeRawDataBase64(_data, len, dt.c_str()); + return; + } else if (state_of_writing_base64 == FileStorage_API::Base64State::Uncertain) { + switch_to_Base64_state(FileStorage_API::Base64State::NotUse); + } + + size_t elemSize = fs::calcStructSize(dt.c_str(), 0); + CV_Assert(elemSize); + CV_Assert(len % elemSize == 0); + len /= elemSize; + + bool explicitZero = fmt == FileStorage::FORMAT_JSON; + const uchar *data0 = (const uchar *) _data; + int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2], k, fmt_pair_count; + char buf[256] = ""; + + fmt_pair_count = fs::decodeFormat(dt.c_str(), fmt_pairs, CV_FS_MAX_FMT_PAIRS); + + if (!len) + return; + + if (!data0) + CV_Error(cv::Error::StsNullPtr, "Null data pointer"); + + if (fmt_pair_count == 1) { + fmt_pairs[0] *= (int) len; + len = 1; + } + + for (; len--; data0 += elemSize) { + int offset = 0; + for (k = 0; k < fmt_pair_count; k++) { + int i, count = fmt_pairs[k * 2]; + int elem_type = fmt_pairs[k * 2 + 1]; + int elem_size = CV_ELEM_SIZE(elem_type); + const char *ptr; + + offset = cvAlign(offset, elem_size); + const uchar *data = data0 + offset; + + for (i = 0; i < count; i++) { + switch (elem_type) { case CV_8U: - ptr = fs::itoa( *(uchar*)data, buf, 10 ); + ptr = fs::itoa(*(uchar *) data, buf, 10); data++; break; case CV_8S: - ptr = fs::itoa( *(char*)data, buf, 10 ); + ptr = fs::itoa(*(char *) data, buf, 10); data++; break; case CV_16U: - ptr = fs::itoa( *(ushort*)data, buf, 10 ); + ptr = fs::itoa(*(ushort *) data, buf, 10); data += sizeof(ushort); break; case CV_16S: - ptr = fs::itoa( *(short*)data, buf, 10 ); + ptr = fs::itoa(*(short *) data, buf, 10); data += sizeof(short); break; case CV_32S: - ptr = fs::itoa( *(int*)data, buf, 10 ); + ptr = fs::itoa(*(int *) data, buf, 10); data += sizeof(int); break; case CV_32F: - ptr = fs::floatToString( buf, *(float*)data, false, explicitZero ); + ptr = fs::floatToString(buf, *(float *) data, false, explicitZero); data += sizeof(float); break; case CV_64F: - ptr = fs::doubleToString( buf, *(double*)data, explicitZero ); + ptr = fs::doubleToString(buf, *(double *) data, explicitZero); data += sizeof(double); break; case CV_16F: /* reference */ - ptr = fs::floatToString( buf, (float)*(float16_t*)data, true, explicitZero ); + ptr = fs::floatToString(buf, (float) *(float16_t *) data, true, explicitZero); data += sizeof(float16_t); break; default: - CV_Error( CV_StsUnsupportedFormat, "Unsupported type" ); + CV_Error(cv::Error::StsUnsupportedFormat, "Unsupported type"); return; - } - - emitter->writeScalar(0, ptr); } - offset = (int)(data - data0); + emitter->writeScalar(0, ptr); } + + offset = (int) (data - data0); } } +} - void writeRawDataBase64(const void* /*data*/, int /*len*/, const char* /*dt*/ ) - { +void FileStorage::Impl::workaround() { + check_if_write_struct_is_delayed(false); + if (state_of_writing_base64 != FileStorage_API::Base64State::Uncertain) + switch_to_Base64_state(FileStorage_API::Base64State::Uncertain); +} + +void FileStorage::Impl::switch_to_Base64_state(FileStorage_API::Base64State new_state) { + const char *err_unkonwn_state = "Unexpected error, unable to determine the Base64 state."; + const char *err_unable_to_switch = "Unexpected error, unable to switch to this state."; + + /* like a finite state machine */ + switch (state_of_writing_base64) { + case FileStorage_API::Base64State::Uncertain: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + { + CV_DbgAssert(base64_writer == 0); + bool can_indent = (fmt != cv::FileStorage::Mode::FORMAT_JSON); + base64_writer = new base64::Base64Writer(*this, can_indent); + if (!can_indent) { + char *ptr = bufferPtr(); + *ptr++ = '\0'; + puts(bufferStart()); + setBufferPtr(bufferStart()); + memset(bufferStart(), 0, static_cast(space)); + puts("\"$base64$"); + } + break; + } + case FileStorage_API::Base64State::Uncertain: + break; + case FileStorage_API::Base64State::NotUse: + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + case FileStorage_API::Base64State::InUse: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + case FileStorage_API::Base64State::NotUse: + CV_Error(cv::Error::StsError, err_unable_to_switch); + break; + case FileStorage_API::Base64State::Uncertain: + delete base64_writer; + base64_writer = 0; + if ( fmt == cv::FileStorage::FORMAT_JSON ) + { + puts("\""); + setBufferPtr(bufferStart()); + flush(); + memset(bufferStart(), 0, static_cast(space) ); + setBufferPtr(bufferStart()); + } + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + case FileStorage_API::Base64State::NotUse: + switch (new_state) { + case FileStorage_API::Base64State::InUse: + case FileStorage_API::Base64State::NotUse: + CV_Error(cv::Error::StsError, err_unable_to_switch); + break; + case FileStorage_API::Base64State::Uncertain: + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; + } + break; + default: + CV_Error(cv::Error::StsError, err_unkonwn_state); + break; } - String releaseAndGetString(); + state_of_writing_base64 = new_state; +} - FileNode getFirstTopLevelNode() const - { - return roots.empty() ? FileNode() : roots[0]; +void FileStorage::Impl::make_write_struct_delayed(const char *key, int struct_flags, const char *type_name) { + CV_Assert(is_write_struct_delayed == false); + CV_DbgAssert(delayed_struct_key == nullptr); + CV_DbgAssert(delayed_struct_flags == 0); + CV_DbgAssert(delayed_type_name == nullptr); + + delayed_struct_flags = struct_flags; + + if (key != nullptr) { + delayed_struct_key = new char[strlen(key) + 1U]; + strcpy(delayed_struct_key, key); } - FileNode root(int streamIdx=0) const - { - return streamIdx >= 0 && streamIdx < (int)roots.size() ? roots[streamIdx] : FileNode(); + if (type_name != nullptr) { + delayed_type_name = new char[strlen(type_name) + 1U]; + strcpy(delayed_type_name, type_name); } - FileNode operator[](const String& nodename) const - { - return this->operator[](nodename.c_str()); - } + is_write_struct_delayed = true; +} - FileNode operator[](const char* /*nodename*/) const - { - return FileNode(); - } +void FileStorage::Impl::check_if_write_struct_is_delayed(bool change_type_to_base64) { + if (is_write_struct_delayed) { + /* save data to prevent recursive call errors */ + std::string struct_key; + std::string type_name; + int struct_flags = delayed_struct_flags; - int getFormat() const { return fmt; } - - char* bufferPtr() const { return (char*)(&buffer[0] + bufofs); } - char* bufferStart() const { return (char*)&buffer[0]; } - char* bufferEnd() const { return (char*)(&buffer[0] + buffer.size()); } - void setBufferPtr(char* ptr) - { - char* bufferstart = bufferStart(); - CV_Assert( ptr >= bufferstart && ptr <= bufferEnd() ); - bufofs = ptr - bufferstart; - } - int wrapMargin() const { return wrap_margin; } - - FStructData& getCurrentStruct() - { - CV_Assert(!write_stack.empty()); - return write_stack.back(); - } - - void setNonEmpty() - { - empty_stream = false; - } - - void processSpecialDouble( char* buf, double* value, char** endptr ) - { - FileStorage_API* fs = this; - char c = buf[0]; - int inf_hi = 0x7ff00000; - - if( c == '-' || c == '+' ) - { - inf_hi = c == '-' ? 0xfff00000 : 0x7ff00000; - c = *++buf; + if (delayed_struct_key != nullptr && *delayed_struct_key != '\0') { + struct_key.assign(delayed_struct_key); + } + if (delayed_type_name != nullptr && *delayed_type_name != '\0') { + type_name.assign(delayed_type_name); } - if( c != '.' ) - CV_PARSE_ERROR_CPP( "Bad format of floating-point constant" ); + /* reset */ + delete[] delayed_struct_key; + delete[] delayed_type_name; + delayed_struct_key = nullptr; + delayed_struct_flags = 0; + delayed_type_name = nullptr; - Cv64suf v; - v.f = 0.; - if( toupper(buf[1]) == 'I' && toupper(buf[2]) == 'N' && toupper(buf[3]) == 'F' ) - v.u = (uint64)inf_hi << 32; - else if( toupper(buf[1]) == 'N' && toupper(buf[2]) == 'A' && toupper(buf[3]) == 'N' ) - v.u = (uint64)-1; + is_write_struct_delayed = false; + + /* call */ + if (change_type_to_base64) { + startWriteStruct_helper(struct_key.c_str(), struct_flags, "binary"); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::InUse); + } else { + startWriteStruct_helper(struct_key.c_str(), struct_flags, type_name.c_str()); + if (state_of_writing_base64 != FileStorage_API::Uncertain) + switch_to_Base64_state(FileStorage_API::Uncertain); + switch_to_Base64_state(FileStorage_API::NotUse); + } + } +} + +void FileStorage::Impl::writeRawDataBase64(const void *_data, size_t len, const char *dt) { + CV_Assert(write_mode); + + check_if_write_struct_is_delayed(true); + + if (state_of_writing_base64 == FileStorage_API::Base64State::Uncertain) { + switch_to_Base64_state(FileStorage_API::Base64State::InUse); + } else if (state_of_writing_base64 != FileStorage_API::Base64State::InUse) { + CV_Error(cv::Error::StsError, "Base64 should not be used at present."); + } + + base64_writer->write(_data, len, dt); +} + +FileNode FileStorage::Impl::getFirstTopLevelNode() const { + return roots.empty() ? FileNode() : roots[0]; +} + +FileNode FileStorage::Impl::root(int streamIdx) const { + return streamIdx >= 0 && streamIdx < (int) roots.size() ? roots[streamIdx] : FileNode(); +} + +FileNode FileStorage::Impl::operator[](const String &nodename) const { + return this->operator[](nodename.c_str()); +} + +FileNode FileStorage::Impl::operator[](const char * /*nodename*/) const { + return FileNode(); +} + +int FileStorage::Impl::getFormat() const { return fmt; } + +char *FileStorage::Impl::bufferPtr() const { return (char *) (&buffer[0] + bufofs); } + +char *FileStorage::Impl::bufferStart() const { return (char *) &buffer[0]; } + +char *FileStorage::Impl::bufferEnd() const { return (char *) (&buffer[0] + buffer.size()); } + +void FileStorage::Impl::setBufferPtr(char *ptr) { + char *bufferstart = bufferStart(); + CV_Assert(ptr >= bufferstart && ptr <= bufferEnd()); + bufofs = ptr - bufferstart; +} + +int FileStorage::Impl::wrapMargin() const { return wrap_margin; } + +FStructData &FileStorage::Impl::getCurrentStruct() { + CV_Assert(!write_stack.empty()); + return write_stack.back(); +} + +void FileStorage::Impl::setNonEmpty() { + empty_stream = false; +} + +void FileStorage::Impl::processSpecialDouble(char *buf, double *value, char **endptr) { + FileStorage_API *fs = this; + char c = buf[0]; + int inf_hi = 0x7ff00000; + + if (c == '-' || c == '+') { + inf_hi = c == '-' ? 0xfff00000 : 0x7ff00000; + c = *++buf; + } + + if (c != '.') + CV_PARSE_ERROR_CPP("Bad format of floating-point constant"); + + Cv64suf v; + v.f = 0.; + if (toupper(buf[1]) == 'I' && toupper(buf[2]) == 'N' && toupper(buf[3]) == 'F') + v.u = (uint64) inf_hi << 32; + else if (toupper(buf[1]) == 'N' && toupper(buf[2]) == 'A' && toupper(buf[3]) == 'N') + v.u = (uint64) -1; + else + CV_PARSE_ERROR_CPP("Bad format of floating-point constant"); + *value = v.f; + *endptr = buf + 4; +} + +double FileStorage::Impl::strtod(char *ptr, char **endptr) { + double fval = ::strtod(ptr, endptr); + if (**endptr == '.') { + char *dot_pos = *endptr; + *dot_pos = ','; + double fval2 = ::strtod(ptr, endptr); + *dot_pos = '.'; + if (*endptr > dot_pos) + fval = fval2; else - CV_PARSE_ERROR_CPP( "Bad format of floating-point constant" ); - *value = v.f; - *endptr = buf + 4; + *endptr = dot_pos; } - double strtod( char* ptr, char** endptr ) - { - double fval = ::strtod( ptr, endptr ); - if( **endptr == '.' ) - { - char* dot_pos = *endptr; - *dot_pos = ','; - double fval2 = ::strtod( ptr, endptr ); - *dot_pos = '.'; - if( *endptr > dot_pos ) - fval = fval2; - else - *endptr = dot_pos; - } + if (*endptr == ptr || cv_isalpha(**endptr)) + processSpecialDouble(ptr, &fval, endptr); - if( *endptr == ptr || cv_isalpha(**endptr) ) - processSpecialDouble( ptr, &fval, endptr ); + return fval; +} - return fval; +void FileStorage::Impl::convertToCollection(int type, FileNode &node) { + CV_Assert(type == FileNode::SEQ || type == FileNode::MAP); + + int node_type = node.type(); + if (node_type == type) + return; + + bool named = node.isNamed(); + uchar *ptr = node.ptr() + 1 + (named ? 4 : 0); + + int ival = 0; + double fval = 0; + std::string sval; + bool add_first_scalar = false; + + if (node_type != FileNode::NONE) { + // scalar nodes can only be converted to sequences, e.g. in XML: + // 5[parser_position]... => create 5 with name "a" + // 5 6[parser_position]... => 5 is converted to [5] and then 6 is added to it + // + // otherwise we don't know where to get the element names from + CV_Assert(type == FileNode::SEQ); + if (node_type == FileNode::INT) { + ival = readInt(ptr); + add_first_scalar = true; + } else if (node_type == FileNode::REAL) { + fval = readReal(ptr); + add_first_scalar = true; + } else if (node_type == FileNode::STRING) { + sval = std::string(node); + add_first_scalar = true; + } else + CV_Error_(Error::StsError, ("The node of type %d cannot be converted to collection", node_type)); } - void convertToCollection(int type, FileNode& node) - { - CV_Assert( type == FileNode::SEQ || type == FileNode::MAP ); + ptr = reserveNodeSpace(node, 1 + (named ? 4 : 0) + 4 + 4); + *ptr++ = (uchar) (type | (named ? FileNode::NAMED : 0)); + // name has been copied automatically + if (named) + ptr += 4; + // set raw_size(collection)==4, nelems(collection)==1 + writeInt(ptr, 4); + writeInt(ptr + 4, 0); - int node_type = node.type(); - if( node_type == type ) - return; + if (add_first_scalar) + addNode(node, std::string(), node_type, + node_type == FileNode::INT ? (const void *) &ival : + node_type == FileNode::REAL ? (const void *) &fval : + node_type == FileNode::STRING ? (const void *) sval.c_str() : 0, + -1); +} - bool named = node.isNamed(); - uchar* ptr = node.ptr() + 1 + (named ? 4 : 0); +// a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or +// b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). +// If there is no enough space in the current block (it should be the last block added so far), +// the last block is shrunk so that it ends immediately before the reallocated node. Then, +// a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. +// The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. +// In the case (b) the existing tag and the name are copied automatically. +uchar *FileStorage::Impl::reserveNodeSpace(FileNode &node, size_t sz) { + bool shrinkBlock = false; + size_t shrinkBlockIdx = 0, shrinkSize = 0; - int ival = 0; - double fval = 0; - std::string sval; - bool add_first_scalar = false; + uchar *ptr = 0, *blockEnd = 0; - if( node_type != FileNode::NONE ) - { - // scalar nodes can only be converted to sequences, e.g. in XML: - // 5[parser_position]... => create 5 with name "a" - // 5 6[parser_position]... => 5 is converted to [5] and then 6 is added to it - // - // otherwise we don't know where to get the element names from - CV_Assert( type == FileNode::SEQ ); - if( node_type == FileNode::INT ) - { - ival = readInt(ptr); - add_first_scalar = true; - } - else if( node_type == FileNode::REAL ) - { - fval = readReal(ptr); - add_first_scalar = true; - } - else if( node_type == FileNode::STRING ) - { - sval = std::string(node); - add_first_scalar = true; - } - else - CV_Error_(Error::StsError, ("The node of type %d cannot be converted to collection", node_type)); + if (!fs_data_ptrs.empty()) { + size_t blockIdx = node.blockIdx; + size_t ofs = node.ofs; + CV_Assert(blockIdx == fs_data_ptrs.size() - 1); + CV_Assert(ofs <= fs_data_blksz[blockIdx]); + CV_Assert(freeSpaceOfs <= fs_data_blksz[blockIdx]); + //CV_Assert( freeSpaceOfs <= ofs + sz ); + + ptr = fs_data_ptrs[blockIdx] + ofs; + blockEnd = fs_data_ptrs[blockIdx] + fs_data_blksz[blockIdx]; + + CV_Assert(ptr >= fs_data_ptrs[blockIdx] && ptr <= blockEnd); + if (ptr + sz <= blockEnd) { + freeSpaceOfs = ofs + sz; + return ptr; } - ptr = reserveNodeSpace(node, 1 + (named ? 4 : 0) + 4 + 4); - *ptr++ = (uchar)(type | (named ? FileNode::NAMED : 0)); - // name has been copied automatically - if( named ) - ptr += 4; - // set raw_size(collection)==4, nelems(collection)==1 + if (ofs == + 0) // FileNode is a first component of this block. Resize current block instead of allocation of new one. + { + fs_data[blockIdx]->resize(sz); + ptr = &fs_data[blockIdx]->at(0); + fs_data_ptrs[blockIdx] = ptr; + fs_data_blksz[blockIdx] = sz; + freeSpaceOfs = sz; + return ptr; + } + + shrinkBlock = true; + shrinkBlockIdx = blockIdx; + shrinkSize = ofs; + } + + size_t blockSize = std::max((size_t) CV_FS_MAX_LEN * 4 - 256, sz) + 256; + Ptr > pv = makePtr >(blockSize); + fs_data.push_back(pv); + uchar *new_ptr = &pv->at(0); + fs_data_ptrs.push_back(new_ptr); + fs_data_blksz.push_back(blockSize); + node.blockIdx = fs_data_ptrs.size() - 1; + node.ofs = 0; + freeSpaceOfs = sz; + + if (ptr && ptr + 5 <= blockEnd) { + new_ptr[0] = ptr[0]; + if (ptr[0] & FileNode::NAMED) { + new_ptr[1] = ptr[1]; + new_ptr[2] = ptr[2]; + new_ptr[3] = ptr[3]; + new_ptr[4] = ptr[4]; + } + } + + if (shrinkBlock) { + fs_data[shrinkBlockIdx]->resize(shrinkSize); + fs_data_blksz[shrinkBlockIdx] = shrinkSize; + } + + return new_ptr; +} + +unsigned FileStorage::Impl::getStringOfs(const std::string &key) const { + str_hash_t::const_iterator it = str_hash.find(key); + return it != str_hash.end() ? it->second : 0; +} + +FileNode FileStorage::Impl::addNode(FileNode &collection, const std::string &key, + int elem_type, const void *value, int len) { + FileStorage_API *fs = this; + bool noname = key.empty() || (fmt == FileStorage::FORMAT_XML && strcmp(key.c_str(), "_") == 0); + convertToCollection(noname ? FileNode::SEQ : FileNode::MAP, collection); + + bool isseq = collection.empty() ? false : collection.isSeq(); + if (noname != isseq) + CV_PARSE_ERROR_CPP(noname ? "Map element should have a name" : + "Sequence element should not have name (use <_>)"); + unsigned strofs = 0; + if (!noname) { + strofs = getStringOfs(key); + if (!strofs) { + strofs = (unsigned) str_hash_data.size(); + size_t keysize = key.size() + 1; + str_hash_data.resize(strofs + keysize); + memcpy(&str_hash_data[0] + strofs, &key[0], keysize); + str_hash.insert(std::make_pair(key, strofs)); + } + } + + uchar *cp = collection.ptr(); + + size_t blockIdx = fs_data_ptrs.size() - 1; + size_t ofs = freeSpaceOfs; + FileNode node(fs_ext, blockIdx, ofs); + + size_t sz0 = 1 + (noname ? 0 : 4) + 8; + uchar *ptr = reserveNodeSpace(node, sz0); + + *ptr++ = (uchar) (elem_type | (noname ? 0 : FileNode::NAMED)); + if (elem_type == FileNode::NONE) + freeSpaceOfs -= 8; + + if (!noname) { + writeInt(ptr, (int) strofs); + ptr += 4; + } + + if (elem_type == FileNode::SEQ || elem_type == FileNode::MAP) { writeInt(ptr, 4); - writeInt(ptr + 4, 0); - - if( add_first_scalar ) - addNode(node, std::string(), node_type, - node_type == FileNode::INT ? (const void*)&ival : - node_type == FileNode::REAL ? (const void*)&fval : - node_type == FileNode::STRING ? (const void*)sval.c_str() : 0, - -1); + writeInt(ptr, 0); } - // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or - // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). - // If there is no enough space in the current block (it should be the last block added so far), - // the last block is shrunk so that it ends immediately before the reallocated node. Then, - // a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. - // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. - // In the case (b) the existing tag and the name are copied automatically. - uchar* reserveNodeSpace(FileNode& node, size_t sz) - { - bool shrinkBlock = false; - size_t shrinkBlockIdx = 0, shrinkSize = 0; + if (value) + node.setValue(elem_type, value, len); - uchar *ptr = 0, *blockEnd = 0; + if (collection.isNamed()) + cp += 4; + int nelems = readInt(cp + 5); + writeInt(cp + 5, nelems + 1); - if( !fs_data_ptrs.empty() ) - { - size_t blockIdx = node.blockIdx; - size_t ofs = node.ofs; - CV_Assert( blockIdx == fs_data_ptrs.size()-1 ); - CV_Assert( ofs <= fs_data_blksz[blockIdx] ); - CV_Assert( freeSpaceOfs <= fs_data_blksz[blockIdx] ); - //CV_Assert( freeSpaceOfs <= ofs + sz ); + return node; +} - ptr = fs_data_ptrs[blockIdx] + ofs; - blockEnd = fs_data_ptrs[blockIdx] + fs_data_blksz[blockIdx]; +void FileStorage::Impl::finalizeCollection(FileNode &collection) { + if (!collection.isSeq() && !collection.isMap()) + return; + uchar *ptr0 = collection.ptr(), *ptr = ptr0 + 1; + if (*ptr0 & FileNode::NAMED) + ptr += 4; + size_t blockIdx = collection.blockIdx; + size_t ofs = collection.ofs + (size_t) (ptr + 8 - ptr0); + size_t rawSize = 4; + unsigned sz = (unsigned) readInt(ptr + 4); + if (sz > 0) { + size_t lastBlockIdx = fs_data_ptrs.size() - 1; - CV_Assert(ptr >= fs_data_ptrs[blockIdx] && ptr <= blockEnd); - if( ptr + sz <= blockEnd ) - { - freeSpaceOfs = ofs + sz; - return ptr; - } - - if (ofs == 0) // FileNode is a first component of this block. Resize current block instead of allocation of new one. - { - fs_data[blockIdx]->resize(sz); - ptr = &fs_data[blockIdx]->at(0); - fs_data_ptrs[blockIdx] = ptr; - fs_data_blksz[blockIdx] = sz; - freeSpaceOfs = sz; - return ptr; - } - - shrinkBlock = true; - shrinkBlockIdx = blockIdx; - shrinkSize = ofs; - } - - size_t blockSize = std::max((size_t)CV_FS_MAX_LEN*4 - 256, sz) + 256; - Ptr > pv = makePtr >(blockSize); - fs_data.push_back(pv); - uchar* new_ptr = &pv->at(0); - fs_data_ptrs.push_back(new_ptr); - fs_data_blksz.push_back(blockSize); - node.blockIdx = fs_data_ptrs.size()-1; - node.ofs = 0; - freeSpaceOfs = sz; - - if( ptr && ptr + 5 <= blockEnd ) - { - new_ptr[0] = ptr[0]; - if( ptr[0] & FileNode::NAMED ) - { - new_ptr[1] = ptr[1]; - new_ptr[2] = ptr[2]; - new_ptr[3] = ptr[3]; - new_ptr[4] = ptr[4]; - } - } - - if (shrinkBlock) - { - fs_data[shrinkBlockIdx]->resize(shrinkSize); - fs_data_blksz[shrinkBlockIdx] = shrinkSize; - } - - return new_ptr; - } - - unsigned getStringOfs( const std::string& key ) const - { - str_hash_t::const_iterator it = str_hash.find(key); - return it != str_hash.end() ? it->second : 0; - } - - FileNode addNode( FileNode& collection, const std::string& key, - int elem_type, const void* value, int len ) - { - FileStorage_API* fs = this; - bool noname = key.empty() || (fmt == FileStorage::FORMAT_XML && strcmp(key.c_str(), "_") == 0); - convertToCollection( noname ? FileNode::SEQ : FileNode::MAP, collection ); - - bool isseq = collection.empty() ? false : collection.isSeq(); - if( noname != isseq ) - CV_PARSE_ERROR_CPP( noname ? "Map element should have a name" : - "Sequence element should not have name (use <_>)" ); - unsigned strofs = 0; - if( !noname ) - { - strofs = getStringOfs(key); - if( !strofs ) - { - strofs = (unsigned)str_hash_data.size(); - size_t keysize = key.size() + 1; - str_hash_data.resize(strofs + keysize); - memcpy(&str_hash_data[0] + strofs, &key[0], keysize); - str_hash.insert(std::make_pair(key, strofs)); - } - } - - uchar* cp = collection.ptr(); - - size_t blockIdx = fs_data_ptrs.size() - 1; - size_t ofs = freeSpaceOfs; - FileNode node(fs_ext, blockIdx, ofs); - - size_t sz0 = 1 + (noname ? 0 : 4) + 8; - uchar* ptr = reserveNodeSpace(node, sz0); - - *ptr++ = (uchar)(elem_type | (noname ? 0 : FileNode::NAMED)); - if( elem_type == FileNode::NONE ) - freeSpaceOfs -= 8; - - if( !noname ) - { - writeInt(ptr, (int)strofs); - ptr += 4; - } - - if( elem_type == FileNode::SEQ || elem_type == FileNode::MAP ) - { - writeInt(ptr, 4); - writeInt(ptr, 0); - } - - if( value ) - node.setValue(elem_type, value, len); - - if( collection.isNamed() ) - cp += 4; - int nelems = readInt(cp + 5); - writeInt(cp + 5, nelems + 1); - - return node; - } - - void finalizeCollection( FileNode& collection ) - { - if( !collection.isSeq() && !collection.isMap() ) - return; - uchar* ptr0 = collection.ptr(), *ptr = ptr0 + 1; - if( *ptr0 & FileNode::NAMED ) - ptr += 4; - size_t blockIdx = collection.blockIdx; - size_t ofs = collection.ofs + (size_t)(ptr + 8 - ptr0); - size_t rawSize = 4; - unsigned sz = (unsigned)readInt(ptr + 4); - if( sz > 0 ) - { - size_t lastBlockIdx = fs_data_ptrs.size() - 1; - - for( ; blockIdx < lastBlockIdx; blockIdx++ ) - { - rawSize += fs_data_blksz[blockIdx] - ofs; - ofs = 0; - } - } - rawSize += freeSpaceOfs - ofs; - writeInt(ptr, (int)rawSize); - } - - void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const - { - while( ofs >= fs_data_blksz[blockIdx] ) - { - if( blockIdx == fs_data_blksz.size() - 1 ) - { - CV_Assert( ofs == fs_data_blksz[blockIdx] ); - break; - } - ofs -= fs_data_blksz[blockIdx]; - blockIdx++; - } - } - - class Base64Decoder - { - public: - Base64Decoder() { ofs = 0; ptr = 0; indent = 0; totalchars = 0; eos = true; } - void init(Ptr& _parser, char* _ptr, int _indent) - { - parser = _parser; - ptr = _ptr; - indent = _indent; - encoded.clear(); - decoded.clear(); + for (; blockIdx < lastBlockIdx; blockIdx++) { + rawSize += fs_data_blksz[blockIdx] - ofs; ofs = 0; - totalchars = 0; - eos = false; } + } + rawSize += freeSpaceOfs - ofs; + writeInt(ptr, (int) rawSize); +} - bool readMore(int needed) - { - static const uchar base64tab[] = +void FileStorage::Impl::normalizeNodeOfs(size_t &blockIdx, size_t &ofs) const { + while (ofs >= fs_data_blksz[blockIdx]) { + if (blockIdx == fs_data_blksz.size() - 1) { + CV_Assert(ofs == fs_data_blksz[blockIdx]); + break; + } + ofs -= fs_data_blksz[blockIdx]; + blockIdx++; + } +} + +FileStorage::Impl::Base64State FileStorage::Impl::get_state_of_writing_base64() { + return state_of_writing_base64; +} + +int FileStorage::Impl::get_space() { + return space; +} + + +FileStorage::Impl::Base64Decoder::Base64Decoder() { + ofs = 0; + ptr = 0; + indent = 0; + totalchars = 0; + eos = true; +} + +void FileStorage::Impl::Base64Decoder::init(Ptr &_parser, char *_ptr, int _indent) { + parser = _parser; + ptr = _ptr; + indent = _indent; + encoded.clear(); + decoded.clear(); + ofs = 0; + totalchars = 0; + eos = false; +} + +bool FileStorage::Impl::Base64Decoder::readMore(int needed) { + static const uchar base64tab[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - if( eos ) - return false; + if (eos) + return false; - size_t sz = decoded.size(); - CV_Assert( ofs <= sz ); - sz -= ofs; - for( size_t i = 0; i < sz; i++ ) - decoded[i] = decoded[ofs + i]; + size_t sz = decoded.size(); + CV_Assert(ofs <= sz); + sz -= ofs; + for (size_t i = 0; i < sz; i++) + decoded[i] = decoded[ofs + i]; - decoded.resize(sz); - ofs = 0; + decoded.resize(sz); + ofs = 0; - CV_Assert( !parser.empty() && ptr ); - char *beg = 0, *end = 0; - bool ok = parser->getBase64Row(ptr, indent, beg, end); - ptr = end; - std::copy(beg, end, std::back_inserter(encoded)); - totalchars += end - beg; + CV_Assert(!parser.empty() && ptr); + char *beg = 0, *end = 0; + bool ok = parser->getBase64Row(ptr, indent, beg, end); + ptr = end; + std::copy(beg, end, std::back_inserter(encoded)); + totalchars += end - beg; - if( !ok || beg == end ) - { - // in the end of base64 sequence pad it with '=' characters so that - // its total length is multiple of - eos = true; - size_t tc = totalchars; - for( ; tc % 4 != 0; tc++ ) - encoded.push_back('='); - } + if (!ok || beg == end) { + // in the end of base64 sequence pad it with '=' characters so that + // its total length is multiple of + eos = true; + size_t tc = totalchars; + for (; tc % 4 != 0; tc++) + encoded.push_back('='); + } - int i = 0, j, n = (int)encoded.size(); - if( n > 0 ) - { - const uchar* tab = base64tab; - char* src = &encoded[0]; + int i = 0, j, n = (int) encoded.size(); + if (n > 0) { + const uchar *tab = base64tab; + char *src = &encoded[0]; - for( ; i <= n - 4; i += 4 ) - { - // dddddd cccccc bbbbbb aaaaaa => ddddddcc ccccbbbb bbaaaaaa - uchar d = tab[(int)(uchar)src[i]], c = tab[(int)(uchar)src[i+1]]; - uchar b = tab[(int)(uchar)src[i+2]], a = tab[(int)(uchar)src[i+3]]; + for (; i <= n - 4; i += 4) { + // dddddd cccccc bbbbbb aaaaaa => ddddddcc ccccbbbb bbaaaaaa + uchar d = tab[(int) (uchar) src[i]], c = tab[(int) (uchar) src[i + 1]]; + uchar b = tab[(int) (uchar) src[i + 2]], a = tab[(int) (uchar) src[i + 3]]; - decoded.push_back((uchar)((d << 2) | (c >> 4))); - decoded.push_back((uchar)((c << 4) | (b >> 2))); - decoded.push_back((uchar)((b << 6) | a)); - } - } - - if( i > 0 && encoded[i-1] == '=' ) - { - if( i > 1 && encoded[i-2] == '=' && !decoded.empty() ) - decoded.pop_back(); - if( !decoded.empty() ) - decoded.pop_back(); - } - - n -= i; - for( j = 0; j < n; j++ ) - encoded[j] = encoded[i + j]; - encoded.resize(n); - - return (int)decoded.size() >= needed; + decoded.push_back((uchar) ((d << 2) | (c >> 4))); + decoded.push_back((uchar) ((c << 4) | (b >> 2))); + decoded.push_back((uchar) ((b << 6) | a)); } + } - uchar getUInt8() - { - size_t sz = decoded.size(); - if( ofs >= sz && !readMore(1) ) - return (uchar)0; - return decoded[ofs++]; - } + if (i > 0 && encoded[i - 1] == '=') { + if (i > 1 && encoded[i - 2] == '=' && !decoded.empty()) + decoded.pop_back(); + if (!decoded.empty()) + decoded.pop_back(); + } - ushort getUInt16() - { - size_t sz = decoded.size(); - if( ofs + 2 > sz && !readMore(2) ) - return (ushort)0; - ushort val = (decoded[ofs] + (decoded[ofs + 1] << 8)); - ofs += 2; - return val; - } + n -= i; + for (j = 0; j < n; j++) + encoded[j] = encoded[i + j]; + encoded.resize(n); - int getInt32() - { - size_t sz = decoded.size(); - if( ofs + 4 > sz && !readMore(4) ) - return 0; - int ival = readInt(&decoded[ofs]); - ofs += 4; - return ival; - } + return (int) decoded.size() >= needed; +} - double getFloat64() - { - size_t sz = decoded.size(); - if( ofs + 8 > sz && !readMore(8) ) - return 0; - double fval = readReal(&decoded[ofs]); - ofs += 8; - return fval; - } +uchar FileStorage::Impl::Base64Decoder::getUInt8() { + size_t sz = decoded.size(); + if (ofs >= sz && !readMore(1)) + return (uchar) 0; + return decoded[ofs++]; +} - bool endOfStream() const { return eos; } - char* getPtr() const { return ptr; } - protected: +ushort FileStorage::Impl::Base64Decoder::getUInt16() { + size_t sz = decoded.size(); + if (ofs + 2 > sz && !readMore(2)) + return (ushort) 0; + ushort val = (decoded[ofs] + (decoded[ofs + 1] << 8)); + ofs += 2; + return val; +} - Ptr parser; - char* ptr; - int indent; - std::vector encoded; - std::vector decoded; - size_t ofs; - size_t totalchars; - bool eos; - }; +int FileStorage::Impl::Base64Decoder::getInt32() { + size_t sz = decoded.size(); + if (ofs + 4 > sz && !readMore(4)) + return 0; + int ival = readInt(&decoded[ofs]); + ofs += 4; + return ival; +} - char* parseBase64(char* ptr, int indent, FileNode& collection) - { - const int BASE64_HDR_SIZE = 24; - char dt[BASE64_HDR_SIZE+1] = {0}; - base64decoder.init(parser, ptr, indent); +double FileStorage::Impl::Base64Decoder::getFloat64() { + size_t sz = decoded.size(); + if (ofs + 8 > sz && !readMore(8)) + return 0; + double fval = readReal(&decoded[ofs]); + ofs += 8; + return fval; +} - int i, k; +bool FileStorage::Impl::Base64Decoder::endOfStream() const { return eos; } - for( i = 0; i < BASE64_HDR_SIZE; i++ ) - dt[i] = (char)base64decoder.getUInt8(); - for( i = 0; i < BASE64_HDR_SIZE; i++ ) - if( isspace(dt[i])) - break; - dt[i] = '\0'; +char *FileStorage::Impl::Base64Decoder::getPtr() const { return ptr; } - CV_Assert( !base64decoder.endOfStream() ); - int fmt_pairs[CV_FS_MAX_FMT_PAIRS*2]; - int fmt_pair_count = fs::decodeFormat( dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS ); - int ival = 0; - double fval = 0; +char *FileStorage::Impl::parseBase64(char *ptr, int indent, FileNode &collection) { + const int BASE64_HDR_SIZE = 24; + char dt[BASE64_HDR_SIZE + 1] = {0}; + base64decoder.init(parser, ptr, indent); - for(;;) - { - for( k = 0; k < fmt_pair_count; k++ ) - { - int elem_type = fmt_pairs[k*2+1]; - int count = fmt_pairs[k*2]; + int i, k; - for( i = 0; i < count; i++ ) - { - int node_type = FileNode::INT; - switch( elem_type ) - { + for (i = 0; i < BASE64_HDR_SIZE; i++) + dt[i] = (char) base64decoder.getUInt8(); + for (i = 0; i < BASE64_HDR_SIZE; i++) + if (isspace(dt[i])) + break; + dt[i] = '\0'; + + CV_Assert(!base64decoder.endOfStream()); + + int fmt_pairs[CV_FS_MAX_FMT_PAIRS * 2]; + int fmt_pair_count = fs::decodeFormat(dt, fmt_pairs, CV_FS_MAX_FMT_PAIRS); + int ival = 0; + double fval = 0; + + for (;;) { + for (k = 0; k < fmt_pair_count; k++) { + int elem_type = fmt_pairs[k * 2 + 1]; + int count = fmt_pairs[k * 2]; + + for (i = 0; i < count; i++) { + int node_type = FileNode::INT; + switch (elem_type) { case CV_8U: ival = base64decoder.getUInt8(); break; case CV_8S: - ival = (char)base64decoder.getUInt8(); + ival = (char) base64decoder.getUInt8(); break; case CV_16U: ival = base64decoder.getUInt16(); break; case CV_16S: - ival = (short)base64decoder.getUInt16(); + ival = (short) base64decoder.getUInt16(); break; case CV_32S: ival = base64decoder.getInt32(); break; - case CV_32F: - { + case CV_32F: { Cv32suf v; v.i = base64decoder.getInt32(); fval = v.f; node_type = FileNode::REAL; - } + } break; case CV_64F: fval = base64decoder.getFloat64(); node_type = FileNode::REAL; break; case CV_16F: - fval = (float)float16_t::fromBits(base64decoder.getUInt16()); + fval = (float) float16_t::fromBits(base64decoder.getUInt16()); node_type = FileNode::REAL; break; default: - CV_Error( Error::StsUnsupportedFormat, "Unsupported type" ); - } - - if( base64decoder.endOfStream() ) - break; - addNode(collection, std::string(), node_type, - node_type == FileNode::INT ? (void*)&ival : (void*)&fval, -1); + CV_Error(Error::StsUnsupportedFormat, "Unsupported type"); } + + if (base64decoder.endOfStream()) + break; + addNode(collection, std::string(), node_type, + node_type == FileNode::INT ? (void *) &ival : (void *) &fval, -1); } - if( base64decoder.endOfStream() ) - break; } - - finalizeCollection(collection); - return base64decoder.getPtr(); + if (base64decoder.endOfStream()) + break; } - void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line ) - { - std::string msg = format("%s(%d): %s", filename.c_str(), lineno, err_msg.c_str()); - error(Error::StsParseError, func_name, msg.c_str(), source_file, source_line ); - } + finalizeCollection(collection); + return base64decoder.getPtr(); +} - const uchar* getNodePtr(size_t blockIdx, size_t ofs) const - { - CV_Assert( blockIdx < fs_data_ptrs.size()); - CV_Assert( ofs < fs_data_blksz[blockIdx]); +void FileStorage::Impl::parseError(const char *func_name, const std::string &err_msg, const char *source_file, + int source_line) { + std::string msg = format("%s(%d): %s", filename.c_str(), lineno, err_msg.c_str()); + error(Error::StsParseError, func_name, msg.c_str(), source_file, source_line); +} - return fs_data_ptrs[blockIdx] + ofs; - } +const uchar *FileStorage::Impl::getNodePtr(size_t blockIdx, size_t ofs) const { + CV_Assert(blockIdx < fs_data_ptrs.size()); + CV_Assert(ofs < fs_data_blksz[blockIdx]); - std::string getName( size_t nameofs ) const - { - CV_Assert( nameofs < str_hash_data.size() ); - return std::string(&str_hash_data[nameofs]); - } + return fs_data_ptrs[blockIdx] + ofs; +} - FileStorage* getFS() { return fs_ext; } +std::string FileStorage::Impl::getName(size_t nameofs) const { + CV_Assert(nameofs < str_hash_data.size()); + return std::string(&str_hash_data[nameofs]); +} - FileStorage* fs_ext; +FileStorage *FileStorage::Impl::getFS() { return fs_ext; } - std::string filename; - int flags; - bool empty_stream; - - FILE* file; - gzFile gzfile; - - bool is_opened; - bool dummy_eof; - bool write_mode; - bool mem_mode; - int fmt; - - State state; //!< current state of the FileStorage (used only for writing) - int space, wrap_margin; - std::deque write_stack; - std::vector buffer; - size_t bufofs; - - std::deque outbuf; - - Ptr emitter; - Ptr parser; - Base64Decoder base64decoder; - - std::vector roots; - std::vector > > fs_data; - std::vector fs_data_ptrs; - std::vector fs_data_blksz; - size_t freeSpaceOfs; - typedef std::unordered_map str_hash_t; - str_hash_t str_hash; - std::vector str_hash_data; - - std::vector strbufv; - char* strbuf; - size_t strbufsize; - size_t strbufpos; - int lineno; -}; FileStorage::FileStorage() : state(0) @@ -1800,7 +1842,7 @@ FileStorage::FileStorage(const String& filename, int flags, const String& encodi void FileStorage::startWriteStruct(const String& name, int struct_flags, const String& typeName) { - p->startWriteStruct(name.c_str(), struct_flags, typeName.c_str()); + p->startWriteStruct(name.size() ? name.c_str() : 0, struct_flags, typeName.size() ? typeName.c_str() : 0); elname = String(); if ((struct_flags & FileNode::TYPE_MASK) == FileNode::SEQ) state = FileStorage::VALUE_EXPECTED; @@ -1875,7 +1917,7 @@ std::string FileStorage::getDefaultObjectName(const std::string& _filename) } ptr++; if( ptr == ptr2 ) - CV_Error( CV_StsBadArg, "Invalid filename" ); + CV_Error( cv::Error::StsBadArg, "Invalid filename" ); char* name = name_buf.data(); @@ -1998,12 +2040,14 @@ FileStorage& operator << (FileStorage& fs, const String& str) if( c == '}' || c == ']' ) { if( fs_impl->write_stack.empty() ) - CV_Error_( CV_StsError, ("Extra closing '%c'", *_str) ); + CV_Error_( cv::Error::StsError, ("Extra closing '%c'", *_str) ); + + fs_impl->workaround(); int struct_flags = fs_impl->write_stack.back().flags; char expected_bracket = FileNode::isMap(struct_flags) ? '}' : ']'; if( c != expected_bracket ) - CV_Error_( CV_StsError, ("The closing '%c' does not match the opening '%c'", c, expected_bracket)); + CV_Error_( cv::Error::StsError, ("The closing '%c' does not match the opening '%c'", c, expected_bracket)); fs_impl->endWriteStruct(); CV_Assert(!fs_impl->write_stack.empty()); struct_flags = fs_impl->write_stack.back().flags; @@ -2013,7 +2057,7 @@ FileStorage& operator << (FileStorage& fs, const String& str) else if( fs.state == NAME_EXPECTED + INSIDE_MAP ) { if (!cv_isalpha(c) && c != '_') - CV_Error_( CV_StsError, ("Incorrect element name %s; should start with a letter or '_'", _str) ); + CV_Error_( cv::Error::StsError, ("Incorrect element name %s; should start with a letter or '_'", _str) ); fs.elname = str; fs.state = VALUE_EXPECTED + INSIDE_MAP; } @@ -2042,7 +2086,7 @@ FileStorage& operator << (FileStorage& fs, const String& str) } } else - CV_Error( CV_StsError, "Invalid fs.state" ); + CV_Error( cv::Error::StsError, "Invalid fs.state" ); return fs; } diff --git a/modules/core/src/persistence.hpp b/modules/core/src/persistence.hpp index 05c7adc17c..1a9dbecf7c 100644 --- a/modules/core/src/persistence.hpp +++ b/modules/core/src/persistence.hpp @@ -163,6 +163,24 @@ public: CV_NORETURN virtual void parseError(const char* funcname, const std::string& msg, const char* filename, int lineno) = 0; + +private: + enum Base64State{ + Uncertain, + NotUse, + InUse, + }; + + friend class cv::FileStorage::Impl; + friend class cv::FileStorage; + friend class JSONEmitter; + friend class XMLEmitter; + friend class YAMLEmitter; + + virtual void check_if_write_struct_is_delayed(bool change_type_to_base64 = false) = 0; + virtual void switch_to_Base64_state(Base64State state) = 0; + virtual Base64State get_state_of_writing_base64() = 0; + virtual int get_space() = 0; }; class FileStorageEmitter diff --git a/modules/core/src/persistence_base64_encoding.cpp b/modules/core/src/persistence_base64_encoding.cpp new file mode 100644 index 0000000000..7d90fd422b --- /dev/null +++ b/modules/core/src/persistence_base64_encoding.cpp @@ -0,0 +1,370 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include "precomp.hpp" +#include "persistence_impl.hpp" +#include "persistence_base64_encoding.hpp" + +namespace cv +{ + +class base64::Base64ContextEmitter +{ +public: + explicit Base64ContextEmitter(cv::FileStorage::Impl& fs, bool needs_indent_) + : file_storage(fs) + , needs_indent(needs_indent_) + , binary_buffer(BUFFER_LEN) + , base64_buffer(base64_encode_buffer_size(BUFFER_LEN)) + , src_beg(0) + , src_cur(0) + , src_end(0) + { + src_beg = binary_buffer.data(); + src_end = src_beg + BUFFER_LEN; + src_cur = src_beg; + + CV_Assert(fs.write_mode); + + if (needs_indent) + { + file_storage.flush(); + } + } + + ~Base64ContextEmitter() + { + /* cleaning */ + if (src_cur != src_beg) + flush(); /* encode the rest binary data to base64 buffer */ + } + + Base64ContextEmitter & write(const uchar * beg, const uchar * end) + { + if (beg >= end) + return *this; + + while (beg < end) { + /* collect binary data and copy to binary buffer */ + size_t len = std::min(end - beg, src_end - src_cur); + std::memcpy(src_cur, beg, len); + beg += len; + src_cur += len; + + if (src_cur >= src_end) { + /* binary buffer is full. */ + /* encode it to base64 and send result to fs */ + flush(); + } + } + + return *this; + } + + /* + * a convertor must provide : + * - `operator >> (uchar * & dst)` for writing current binary data to `dst` and moving to next data. + * - `operator bool` for checking if current loaction is valid and not the end. + */ + template inline + Base64ContextEmitter & write(_to_binary_convertor_t & convertor) + { + static const size_t BUFFER_MAX_LEN = 1024U; + + std::vector buffer(BUFFER_MAX_LEN); + uchar * beg = buffer.data(); + uchar * end = beg; + + while (convertor) { + convertor >> end; + write(beg, end); + end = beg; + } + + return *this; + } + + bool flush() + { + /* control line width, so on. */ + size_t len = base64_encode(src_beg, base64_buffer.data(), 0U, src_cur - src_beg); + if (len == 0U) + return false; + + src_cur = src_beg; + + if ( !needs_indent) + { + file_storage.puts((const char*)base64_buffer.data()); + } + else + { + const char newline[] = "\n"; + char space[80]; + int ident = file_storage.write_stack.back().indent; + memset(space, ' ', static_cast(ident)); + space[ident] = '\0'; + + file_storage.puts(space); + file_storage.puts((const char*)base64_buffer.data()); + file_storage.puts(newline); + file_storage.flush(); + } + + return true; + } + +private: + /* because of Base64, we must keep its length a multiple of 3 */ + static const size_t BUFFER_LEN = 48U; + // static_assert(BUFFER_LEN % 3 == 0, "BUFFER_LEN is invalid"); + +private: + cv::FileStorage::Impl& file_storage; + bool needs_indent; + + std::vector binary_buffer; + std::vector base64_buffer; + uchar * src_beg; + uchar * src_cur; + uchar * src_end; +}; + +std::string base64::make_base64_header(const char *dt) { + std::ostringstream oss; + oss << dt << ' '; + std::string buffer(oss.str()); + CV_Assert(buffer.size() < ::base64::HEADER_SIZE); + + buffer.reserve(::base64::HEADER_SIZE); + while (buffer.size() < ::base64::HEADER_SIZE) + buffer += ' '; + + return buffer; +} + +size_t base64::base64_encode(const uint8_t *src, uint8_t *dst, size_t off, size_t cnt) { + if (!src || !dst || !cnt) + return 0; + + /* initialize beginning and end */ + uint8_t * dst_beg = dst; + uint8_t * dst_cur = dst_beg; + + uint8_t const * src_beg = src + off; + uint8_t const * src_cur = src_beg; + uint8_t const * src_end = src_cur + cnt / 3U * 3U; + + /* integer multiples part */ + while (src_cur < src_end) { + uint8_t _2 = *src_cur++; + uint8_t _1 = *src_cur++; + uint8_t _0 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_1 & 0xF0U) >> 4U | (_2 & 0x03U) << 4U]; + *dst_cur++ = base64_mapping[(_0 & 0xC0U) >> 6U | (_1 & 0x0FU) << 2U]; + *dst_cur++ = base64_mapping[ _0 & 0x3FU]; + } + + /* remainder part */ + size_t rst = src_beg + cnt - src_cur; + if (rst == 1U) { + uint8_t _2 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U]; + } else if (rst == 2U) { + uint8_t _2 = *src_cur++; + uint8_t _1 = *src_cur++; + *dst_cur++ = base64_mapping[ _2 >> 2U]; + *dst_cur++ = base64_mapping[(_2 & 0x03U) << 4U | (_1 & 0xF0U) >> 4U]; + *dst_cur++ = base64_mapping[(_1 & 0x0FU) << 2U]; + } + + /* padding */ + switch (rst) + { + case 1U: *dst_cur++ = base64_padding; + /* fallthrough */ + case 2U: *dst_cur++ = base64_padding; + /* fallthrough */ + default: *dst_cur = 0; + break; + } + + return static_cast(dst_cur - dst_beg); +} + +int base64::icvCalcStructSize(const char *dt, int initial_size) { + int size = cv::fs::calcElemSize( dt, initial_size ); + size_t elem_max_size = 0; + for ( const char * type = dt; *type != '\0'; type++ ) { + switch ( *type ) + { + case 'u': { elem_max_size = std::max( elem_max_size, sizeof(uchar ) ); break; } + case 'c': { elem_max_size = std::max( elem_max_size, sizeof(schar ) ); break; } + case 'w': { elem_max_size = std::max( elem_max_size, sizeof(ushort) ); break; } + case 's': { elem_max_size = std::max( elem_max_size, sizeof(short ) ); break; } + case 'i': { elem_max_size = std::max( elem_max_size, sizeof(int ) ); break; } + case 'f': { elem_max_size = std::max( elem_max_size, sizeof(float ) ); break; } + case 'd': { elem_max_size = std::max( elem_max_size, sizeof(double) ); break; } + default: break; + } + } + size = cvAlign( size, static_cast(elem_max_size) ); + return size; +} + +size_t base64::base64_encode_buffer_size(size_t cnt, bool is_end_with_zero) { + size_t additional = static_cast(is_end_with_zero == true); + return (cnt + 2U) / 3U * 4U + additional; +} + +base64::Base64Writer::Base64Writer(cv::FileStorage::Impl& fs, bool can_indent) + : emitter(new Base64ContextEmitter(fs, can_indent)) + , data_type_string() +{ + CV_Assert(fs.write_mode); +} + +void base64::Base64Writer::write(const void* _data, size_t len, const char* dt) +{ + check_dt(dt); + RawDataToBinaryConvertor convertor(_data, static_cast(len), data_type_string); + emitter->write(convertor); +} + +template inline +void base64::Base64Writer::write(_to_binary_convertor_t & convertor, const char* dt) +{ + check_dt(dt); + emitter->write(convertor); +} + +base64::Base64Writer::~Base64Writer() +{ + delete emitter; +} + +void base64::Base64Writer::check_dt(const char* dt) +{ + if ( dt == 0 ) + CV_Error( cv::Error::StsBadArg, "Invalid \'dt\'." ); + else if (data_type_string.empty()) { + data_type_string = dt; + + /* output header */ + std::string buffer = make_base64_header(dt); + const uchar * beg = reinterpret_cast(buffer.data()); + const uchar * end = beg + buffer.size(); + + emitter->write(beg, end); + } else if ( data_type_string != dt ) + CV_Error( cv::Error::StsBadArg, "\'dt\' does not match." ); +} + +base64::RawDataToBinaryConvertor::RawDataToBinaryConvertor(const void* src, int len, const std::string & dt) + : beg(reinterpret_cast(src)) + , cur(0) + , end(0) +{ + CV_Assert(src); + CV_Assert(!dt.empty()); + CV_Assert(len > 0); + + /* calc step and to_binary_funcs */ + step_packed = make_to_binary_funcs(dt); + + end = beg; + cur = beg; + + step = icvCalcStructSize(dt.c_str(), 0); + end = beg + static_cast(len); +} + +inline base64::RawDataToBinaryConvertor& base64::RawDataToBinaryConvertor::operator >>(uchar * & dst) +{ + CV_DbgAssert(*this); + + for (size_t i = 0U, n = to_binary_funcs.size(); i < n; i++) { + elem_to_binary_t & pack = to_binary_funcs[i]; + pack.func(cur + pack.offset, dst + pack.offset_packed); + } + cur += step; + dst += step_packed; + + return *this; +} + +inline base64::RawDataToBinaryConvertor::operator bool() const +{ + return cur < end; +} + +size_t base64::RawDataToBinaryConvertor::make_to_binary_funcs(const std::string &dt) +{ + size_t cnt = 0; + size_t offset = 0; + size_t offset_packed = 0; + char type = '\0'; + + std::istringstream iss(dt); + while (!iss.eof()) { + if (!(iss >> cnt)) { + iss.clear(); + cnt = 1; + } + CV_Assert(cnt > 0U); + if (!(iss >> type)) + break; + + while (cnt-- > 0) + { + elem_to_binary_t pack; + + size_t size = 0; + switch (type) + { + case 'u': + case 'c': + size = sizeof(uchar); + pack.func = to_binary; + break; + case 'w': + case 's': + size = sizeof(ushort); + pack.func = to_binary; + break; + case 'i': + size = sizeof(uint); + pack.func = to_binary; + break; + case 'f': + size = sizeof(float); + pack.func = to_binary; + break; + case 'd': + size = sizeof(double); + pack.func = to_binary; + break; + case 'r': + default: + CV_Error(cv::Error::StsError, "type is not supported"); + }; + + offset = static_cast(cvAlign(static_cast(offset), static_cast(size))); + pack.offset = offset; + offset += size; + + pack.offset_packed = offset_packed; + offset_packed += size; + + to_binary_funcs.push_back(pack); + } + } + + CV_Assert(iss.eof()); + return offset_packed; +} + +} \ No newline at end of file diff --git a/modules/core/src/persistence_base64_encoding.hpp b/modules/core/src/persistence_base64_encoding.hpp new file mode 100644 index 0000000000..1ee5201e14 --- /dev/null +++ b/modules/core/src/persistence_base64_encoding.hpp @@ -0,0 +1,127 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_CORE_BASE64_ENCODING_HPP +#define OPENCV_CORE_BASE64_ENCODING_HPP + +namespace cv +{ + +namespace base64 +{ +/* A decorator for CvFileStorage +* - no copyable +* - not safe for now +* - move constructor may be needed if C++11 +*/ +uint8_t const base64_mapping[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; + +uint8_t const base64_padding = '='; + +std::string make_base64_header(const char * dt); + +size_t base64_encode(uint8_t const * src, uint8_t * dst, size_t off, size_t cnt); + + +int icvCalcStructSize( const char* dt, int initial_size ); + +class Base64ContextEmitter; +class Impl; + +class Base64Writer +{ +public: + Base64Writer(cv::FileStorage::Impl& fs, bool can_indent); + ~Base64Writer(); + void write(const void* _data, size_t len, const char* dt); + template void write(_to_binary_convertor_t & convertor, const char* dt); + +private: + void check_dt(const char* dt); + +private: + // disable copy and assignment + Base64Writer(const Base64Writer &); + Base64Writer & operator=(const Base64Writer &); + +private: + + Base64ContextEmitter * emitter; + std::string data_type_string; +}; + +size_t base64_encode_buffer_size(size_t cnt, bool is_end_with_zero = true); + +template inline size_t +to_binary(_uint_t val, uchar * cur) +{ + size_t delta = CHAR_BIT; + size_t cnt = sizeof(_uint_t); + while (cnt --> static_cast(0U)) { + *cur++ = static_cast(val); + val >>= delta; + } + return sizeof(_uint_t); +} + +template<> inline size_t to_binary(double val, uchar * cur) +{ + Cv64suf bit64; + bit64.f = val; + return to_binary(bit64.u, cur); +} + +template<> inline size_t to_binary(float val, uchar * cur) +{ + Cv32suf bit32; + bit32.f = val; + return to_binary(bit32.u, cur); +} + +template inline size_t +to_binary(uchar const * val, uchar * cur) +{ + return to_binary<_primitive_t>(*reinterpret_cast<_primitive_t const *>(val), cur); +} + + + +class RawDataToBinaryConvertor +{ +public: + // NOTE: len is already multiplied by element size here + RawDataToBinaryConvertor(const void* src, int len, const std::string & dt); + + inline RawDataToBinaryConvertor & operator >>(uchar * & dst); + inline operator bool() const; + +private: + typedef size_t(*to_binary_t)(const uchar *, uchar *); + struct elem_to_binary_t + { + size_t offset; + size_t offset_packed; + to_binary_t func; + }; + +private: + size_t make_to_binary_funcs(const std::string &dt); + +private: + const uchar * beg; + const uchar * cur; + const uchar * end; + + size_t step; + size_t step_packed; + std::vector to_binary_funcs; +}; + +} + +} +#endif \ No newline at end of file diff --git a/modules/core/src/persistence_impl.hpp b/modules/core/src/persistence_impl.hpp new file mode 100644 index 0000000000..4ea2dc3502 --- /dev/null +++ b/modules/core/src/persistence_impl.hpp @@ -0,0 +1,231 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#ifndef OPENCV_CORE_PERSISTENCE_IMPL_HPP +#define OPENCV_CORE_PERSISTENCE_IMPL_HPP + +#include "persistence.hpp" +#include "persistence_base64_encoding.hpp" +#include +#include + + +namespace cv +{ + +enum Base64State{ + Uncertain, + NotUse, + InUse, +}; + +class cv::FileStorage::Impl : public FileStorage_API +{ +public: + void init(); + + Impl(FileStorage* _fs); + + virtual ~Impl(); + + void release(String* out=0); + + void analyze_file_name( const std::string& file_name, std::vector& params ); + + bool open( const char* filename_or_buf, int _flags, const char* encoding ); + + void puts( const char* str ); + + char* getsFromFile( char* buf, int count ); + + char* gets( size_t maxCount ); + + char* gets(); + + bool eof(); + + void setEof(); + + void closeFile(); + + void rewind(); + + char* resizeWriteBuffer( char* ptr, int len ); + + char* flush(); + + void endWriteStruct(); + + void startWriteStruct_helper( const char* key, int struct_flags, + const char* type_name ); + + void startWriteStruct( const char* key, int struct_flags, + const char* type_name ); + + void writeComment( const char* comment, bool eol_comment ); + + void startNextStream(); + + void write( const String& key, int value ); + + void write( const String& key, double value ); + + void write( const String& key, const String& value ); + + void writeRawData( const std::string& dt, const void* _data, size_t len ); + + void workaround(); + + void switch_to_Base64_state( FileStorage_API::Base64State new_state); + + void make_write_struct_delayed( const char* key, int struct_flags, const char* type_name ); + + void check_if_write_struct_is_delayed( bool change_type_to_base64 ); + + void writeRawDataBase64(const void* _data, size_t len, const char* dt ); + + String releaseAndGetString(); + + FileNode getFirstTopLevelNode() const; + + FileNode root(int streamIdx=0) const; + + FileNode operator[](const String& nodename) const; + + FileNode operator[](const char* /*nodename*/) const; + + int getFormat() const; + + char* bufferPtr() const; + char* bufferStart() const; + char* bufferEnd() const; + void setBufferPtr(char* ptr); + int wrapMargin() const; + + FStructData& getCurrentStruct(); + + void setNonEmpty(); + + void processSpecialDouble( char* buf, double* value, char** endptr ); + + double strtod( char* ptr, char** endptr ); + + void convertToCollection(int type, FileNode& node); + + // a) allocates new FileNode (for that just set blockIdx to the last block and ofs to freeSpaceOfs) or + // b) reallocates just created new node (blockIdx and ofs must be taken from FileNode). + // If there is no enough space in the current block (it should be the last block added so far), + // the last block is shrunk so that it ends immediately before the reallocated node. Then, + // a new block of sufficient size is allocated and the FileNode is placed in the beginning of it. + // The case (a) can be used to allocate the very first node by setting blockIdx == ofs == 0. + // In the case (b) the existing tag and the name are copied automatically. + uchar* reserveNodeSpace(FileNode& node, size_t sz); + + unsigned getStringOfs( const std::string& key ) const; + + FileNode addNode( FileNode& collection, const std::string& key, + int elem_type, const void* value, int len ); + + void finalizeCollection( FileNode& collection ); + + void normalizeNodeOfs(size_t& blockIdx, size_t& ofs) const; + + Base64State get_state_of_writing_base64(); + + int get_space(); + + class Base64Decoder + { + public: + Base64Decoder(); + void init(Ptr& _parser, char* _ptr, int _indent); + + bool readMore(int needed); + + uchar getUInt8(); + + ushort getUInt16(); + + int getInt32(); + + double getFloat64(); + + bool endOfStream() const; + char* getPtr() const; + protected: + + Ptr parser; + char* ptr; + int indent; + std::vector encoded; + std::vector decoded; + size_t ofs; + size_t totalchars; + bool eos; + }; + + char* parseBase64(char* ptr, int indent, FileNode& collection); + + void parseError( const char* func_name, const std::string& err_msg, const char* source_file, int source_line ); + + const uchar* getNodePtr(size_t blockIdx, size_t ofs) const; + + std::string getName( size_t nameofs ) const; + + FileStorage* getFS(); + + FileStorage* fs_ext; + + std::string filename; + int flags; + bool empty_stream; + + FILE* file; + gzFile gzfile; + + bool is_opened; + bool dummy_eof; + bool write_mode; + bool mem_mode; + int fmt; + + State state; //!< current state of the FileStorage (used only for writing) + bool is_using_base64; + bool is_write_struct_delayed; + char* delayed_struct_key; + int delayed_struct_flags; + char* delayed_type_name; + FileStorage_API::Base64State state_of_writing_base64; + + int space, wrap_margin; + std::deque write_stack; + std::vector buffer; + size_t bufofs; + + std::deque outbuf; + + Ptr emitter; + Ptr parser; + Base64Decoder base64decoder; + base64::Base64Writer* base64_writer; + + std::vector roots; + std::vector > > fs_data; + std::vector fs_data_ptrs; + std::vector fs_data_blksz; + size_t freeSpaceOfs; + typedef std::unordered_map str_hash_t; + str_hash_t str_hash; + std::vector str_hash_data; + + std::vector strbufv; + char* strbuf; + size_t strbufsize; + size_t strbufpos; + int lineno; +}; + +} + +#endif \ No newline at end of file diff --git a/modules/core/src/persistence_json.cpp b/modules/core/src/persistence_json.cpp index 667895fbc5..12a58e80bf 100644 --- a/modules/core/src/persistence_json.cpp +++ b/modules/core/src/persistence_json.cpp @@ -23,7 +23,7 @@ public: struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, + CV_Error( cv::Error::StsBadArg, "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" ); if( type_name && *type_name == '\0' ) @@ -53,29 +53,26 @@ public: void endWriteStruct(const FStructData& current_struct) { int struct_flags = current_struct.flags; - CV_Assert( FileNode::isCollection(struct_flags) ); - if( !FileNode::isFlow(struct_flags) ) - { -#if 0 - if ( fs->bufferPtr() <= fs->bufferStart() + fs->space ) - { - /* some bad code for base64_writer... */ - ptr = fs->bufferPtr(); - *ptr++ = '\n'; - *ptr++ = '\0'; - fs->puts( fs->bufferStart() ); - fs->setBufferPtr(fs->bufferStart()); + if (FileNode::isCollection(struct_flags)) { + if (!FileNode::isFlow(struct_flags)) { + if (fs->bufferPtr() <= fs->bufferStart() + fs->get_space()) { + /* some bad code for base64_writer... */ + char *ptr = fs->bufferPtr(); + *ptr++ = '\n'; + *ptr++ = '\0'; + fs->puts(fs->bufferStart()); + fs->setBufferPtr(fs->bufferStart()); + } + fs->flush(); } -#endif - fs->flush(); - } - char* ptr = fs->bufferPtr(); - if( ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags) ) - *ptr++ = ' '; - *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']'; - fs->setBufferPtr(ptr); + char *ptr = fs->bufferPtr(); + if (ptr > fs->bufferStart() + current_struct.indent && !FileNode::isEmptyCollection(struct_flags)) + *ptr++ = ' '; + *ptr++ = FileNode::isMap(struct_flags) ? '}' : ']'; + fs->setBufferPtr(ptr); + } } void write(const char* key, int value) @@ -97,11 +94,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { @@ -136,6 +133,20 @@ public: void writeScalar(const char* key, const char* data) { + /* check write_struct */ + + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + + /* check parameters */ + size_t key_len = 0u; if( key && *key == '\0' ) key = 0; @@ -143,9 +154,9 @@ public: { key_len = strlen(key); if ( key_len == 0u ) - CV_Error( CV_StsBadArg, "The key is an empty" ); + CV_Error( cv::Error::StsBadArg, "The key is an empty" ); else if ( static_cast(key_len) > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The key is too long" ); + CV_Error( cv::Error::StsBadArg, "The key is too long" ); } size_t data_len = 0u; @@ -157,7 +168,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if ( (FileNode::isMap(struct_flags) ^ (key != 0)) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else { fs->setNonEmpty(); @@ -199,7 +210,7 @@ public: if( key ) { if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, static_cast(key_len) ); *ptr++ = '\"'; @@ -210,7 +221,7 @@ public: ptr[i] = c; if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) - CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); } ptr += key_len; @@ -233,7 +244,7 @@ public: void writeComment(const char* comment, bool eol_comment) { if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); int len = static_cast(strlen(comment)); char* ptr = fs->bufferPtr(); diff --git a/modules/core/src/persistence_xml.cpp b/modules/core/src/persistence_xml.cpp index 52b5374425..62b7b1eb59 100644 --- a/modules/core/src/persistence_xml.cpp +++ b/modules/core/src/persistence_xml.cpp @@ -45,7 +45,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if( FileNode::isMap(struct_flags) ^ (key != 0) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else @@ -61,26 +61,26 @@ public: if( !key ) key = "_"; else if( key[0] == '_' && key[1] == '\0' ) - CV_Error( CV_StsBadArg, "A single _ is a reserved tag name" ); + CV_Error( cv::Error::StsBadArg, "A single _ is a reserved tag name" ); len = (int)strlen( key ); *ptr++ = '<'; if( tag_type == CV_XML_CLOSING_TAG ) { if( !attrlist.empty() ) - CV_Error( CV_StsBadArg, "Closing tag should not include any attributes" ); + CV_Error( cv::Error::StsBadArg, "Closing tag should not include any attributes" ); *ptr++ = '/'; } if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key should start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key should start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, len ); for( i = 0; i < len; i++ ) { char c = key[i]; if( !cv_isalnum(c) && c != '_' && c != '-' ) - CV_Error( CV_StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" ); + CV_Error( cv::Error::StsBadArg, "Key name may only contain alphanumeric characters [a-zA-Z0-9], '-' and '_'" ); ptr[i] = c; } ptr += len; @@ -158,11 +158,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != '\"' || str[0] != str[len-1] ) { @@ -233,6 +233,16 @@ public: void writeScalar(const char* key, const char* data) { + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + int len = (int)strlen(data); if( key && *key == '\0' ) key = 0; @@ -255,7 +265,7 @@ public: int new_offset = (int)(ptr - fs->bufferStart()) + len; if( key ) - CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" ); + CV_Error( cv::Error::StsBadArg, "elements with keys can not be written to sequence" ); current_struct.flags = FileNode::SEQ; @@ -281,10 +291,10 @@ public: char* ptr; if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); if( strstr(comment, "--") != 0 ) - CV_Error( CV_StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); + CV_Error( cv::Error::StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); len = (int)strlen(comment); eol = strchr(comment, '\n'); diff --git a/modules/core/src/persistence_yml.cpp b/modules/core/src/persistence_yml.cpp index 3f3742b8d1..95db1450c6 100644 --- a/modules/core/src/persistence_yml.cpp +++ b/modules/core/src/persistence_yml.cpp @@ -33,7 +33,7 @@ public: struct_flags = (struct_flags & (FileNode::TYPE_MASK|FileNode::FLOW)) | FileNode::EMPTY; if( !FileNode::isCollection(struct_flags)) - CV_Error( CV_StsBadArg, + CV_Error( cv::Error::StsBadArg, "Some collection type - FileNode::SEQ or FileNode::MAP, must be specified" ); if (type_name && memcmp(type_name, "binary", 6) == 0) @@ -120,11 +120,11 @@ public: int i, len; if( !str ) - CV_Error( CV_StsNullPtr, "Null string pointer" ); + CV_Error( cv::Error::StsNullPtr, "Null string pointer" ); len = (int)strlen(str); if( len > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The written string is too long" ); + CV_Error( cv::Error::StsBadArg, "The written string is too long" ); if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { @@ -174,6 +174,16 @@ public: void writeScalar(const char* key, const char* data) { + fs->check_if_write_struct_is_delayed(false); + if ( fs->get_state_of_writing_base64() == FileStorage_API::Uncertain ) + { + fs->switch_to_Base64_state( FileStorage_API::NotUse ); + } + else if ( fs->get_state_of_writing_base64() == FileStorage_API::InUse ) + { + CV_Error( cv::Error::StsError, "At present, output Base64 data only." ); + } + int i, keylen = 0; int datalen = 0; char* ptr; @@ -188,7 +198,7 @@ public: if( FileNode::isCollection(struct_flags) ) { if( (FileNode::isMap(struct_flags) ^ (key != 0)) ) - CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + CV_Error( cv::Error::StsBadArg, "An attempt to add element without a key to a map, " "or add element with key to sequence" ); } else @@ -201,10 +211,10 @@ public: { keylen = (int)strlen(key); if( keylen == 0 ) - CV_Error( CV_StsBadArg, "The key is an empty" ); + CV_Error( cv::Error::StsBadArg, "The key is an empty" ); if( keylen > CV_FS_MAX_LEN ) - CV_Error( CV_StsBadArg, "The key is too long" ); + CV_Error( cv::Error::StsBadArg, "The key is too long" ); } if( data ) @@ -238,7 +248,7 @@ public: if( key ) { if( !cv_isalpha(key[0]) && key[0] != '_' ) - CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + CV_Error( cv::Error::StsBadArg, "Key must start with a letter or _" ); ptr = fs->resizeWriteBuffer( ptr, keylen ); @@ -248,7 +258,7 @@ public: ptr[i] = c; if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) - CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + CV_Error( cv::Error::StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); } ptr += keylen; @@ -271,7 +281,7 @@ public: void writeComment(const char* comment, bool eol_comment) { if( !comment ) - CV_Error( CV_StsNullPtr, "Null comment" ); + CV_Error( cv::Error::StsNullPtr, "Null comment" ); int len = (int)strlen(comment); const char* eol = strchr(comment, '\n'); diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index c001de3aac..9fddf98718 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -55,6 +55,18 @@ #include +#ifndef OPENCV_WITH_THREAD_SANITIZER + #if defined(__clang__) && defined(__has_feature) + #if __has_feature(thread_sanitizer) + #define OPENCV_WITH_THREAD_SANITIZER 1 + #include // assume C++11 + #endif + #endif +#endif +#ifndef OPENCV_WITH_THREAD_SANITIZER + #define OPENCV_WITH_THREAD_SANITIZER 0 +#endif + namespace cv { static void _initSystem() @@ -116,10 +128,14 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } #include // std::abort #endif -#if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __HAIKU__ || defined __Fuchsia__ +#if defined __ANDROID__ || defined __unix__ || defined __FreeBSD__ || defined __OpenBSD__ || defined __HAIKU__ || defined __Fuchsia__ # include # include +#if defined __QNXNTO__ +# include +#else # include +#endif #if defined __ANDROID__ || defined __linux__ # include #endif @@ -130,7 +146,7 @@ void* allocSingletonNewBuffer(size_t size) { return malloc(size); } #endif -#if (defined __ppc64__ || defined __PPC64__) && defined __linux__ +#if (defined __ppc64__ || defined __PPC64__) && defined __unix__ # include "sys/auxv.h" # ifndef AT_HWCAP2 # define AT_HWCAP2 26 @@ -216,7 +232,9 @@ std::wstring GetTempFileNameWinRT(std::wstring prefix) #endif #else +#ifndef OPENCV_DISABLE_THREAD_SUPPORT #include +#endif #include #include @@ -231,7 +249,7 @@ std::wstring GetTempFileNameWinRT(std::wstring prefix) #include "omp.h" #endif -#if defined __linux__ || defined __APPLE__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ || defined __GLIBC__ || defined __HAIKU__ +#if defined __unix__ || defined __APPLE__ || defined __EMSCRIPTEN__ || defined __FreeBSD__ || defined __GLIBC__ || defined __HAIKU__ #include #include #include @@ -533,7 +551,7 @@ struct HWFeatures } #endif // CV_CPUID_X86 - #if defined __ANDROID__ || defined __linux__ + #if defined __ANDROID__ || defined __linux__ || defined __FreeBSD__ #ifdef __aarch64__ have[CV_CPU_NEON] = true; have[CV_CPU_FP16] = true; @@ -559,7 +577,7 @@ struct HWFeatures CV_LOG_INFO(NULL, "- FP16 instructions is NOT enabled via build flags"); #endif #endif - #elif defined __arm__ + #elif defined __arm__ && !defined __FreeBSD__ int cpufile = open("/proc/self/auxv", O_RDONLY); if (cpufile >= 0) @@ -598,7 +616,7 @@ struct HWFeatures have[CV_CPU_MSA] = true; #endif - #if (defined __ppc64__ || defined __PPC64__) && defined __linux__ + #if (defined __ppc64__ || defined __PPC64__) && defined __unix__ unsigned int hwcap = getauxval(AT_HWCAP); if (hwcap & PPC_FEATURE_HAS_VSX) { hwcap = getauxval(AT_HWCAP2); @@ -812,12 +830,12 @@ int64 getTickCount(void) LARGE_INTEGER counter; QueryPerformanceCounter( &counter ); return (int64)counter.QuadPart; -#elif defined __linux || defined __linux__ +#elif defined __MACH__ && defined __APPLE__ + return (int64)mach_absolute_time(); +#elif defined __unix__ struct timespec tp; clock_gettime(CLOCK_MONOTONIC, &tp); return (int64)tp.tv_sec*1000000000 + tp.tv_nsec; -#elif defined __MACH__ && defined __APPLE__ - return (int64)mach_absolute_time(); #else struct timeval tv; gettimeofday(&tv, NULL); @@ -831,8 +849,6 @@ double getTickFrequency(void) LARGE_INTEGER freq; QueryPerformanceFrequency(&freq); return (double)freq.QuadPart; -#elif defined __linux || defined __linux__ - return 1e9; #elif defined __MACH__ && defined __APPLE__ static double freq = 0; if( freq == 0 ) @@ -842,6 +858,8 @@ double getTickFrequency(void) freq = sTimebaseInfo.denom*1e9/sTimebaseInfo.numer; } return freq; +#elif defined __unix__ + return 1e9; #else return 1e6; #endif @@ -1314,6 +1332,8 @@ bool __termination = false; namespace details { +#ifndef OPENCV_DISABLE_THREAD_SUPPORT + #ifdef _WIN32 #ifdef _MSC_VER #pragma warning(disable:4505) // unreferenced local function has been removed @@ -1323,64 +1343,62 @@ namespace details { #endif #endif -template -class DisposedSingletonMark -{ -private: - static bool mark; -protected: - DisposedSingletonMark() {} - ~DisposedSingletonMark() - { - mark = true; - } -public: - static bool isDisposed() { return mark; } -}; - // TLS platform abstraction layer -class TlsAbstraction : public DisposedSingletonMark +class TlsAbstraction { public: TlsAbstraction(); - ~TlsAbstraction(); - void* getData() const + ~TlsAbstraction() { - if (isDisposed()) // guard: static initialization order fiasco - return NULL; - return getData_(); - } - void setData(void *pData) - { - if (isDisposed()) // guard: static initialization order fiasco - return; - return setData_(pData); + // TlsAbstraction singleton should not be released + // There is no reliable way to avoid problems caused by static initialization order fiasco + // NB: Do NOT use logging here + fprintf(stderr, "OpenCV FATAL: TlsAbstraction::~TlsAbstraction() call is not expected\n"); + fflush(stderr); } + void* getData() const; + void setData(void *pData); + + void releaseSystemResources(); + private: - void* getData_() const; - void setData_(void *pData); #ifdef _WIN32 #ifndef WINRT DWORD tlsKey; + bool disposed; #endif #else // _WIN32 pthread_key_t tlsKey; +#if OPENCV_WITH_THREAD_SANITIZER + std::atomic disposed; +#else + bool disposed; +#endif #endif }; -template<> bool DisposedSingletonMark::mark = false; - -static TlsAbstraction& getTlsAbstraction_() +class TlsAbstractionReleaseGuard { - static TlsAbstraction g_tls; // disposed in atexit() handlers (required for unregistering our callbacks) - return g_tls; -} + TlsAbstraction& tls_; +public: + TlsAbstractionReleaseGuard(TlsAbstraction& tls) : tls_(tls) + { + /* nothing */ + } + ~TlsAbstractionReleaseGuard() + { + tls_.releaseSystemResources(); + } +}; + +// TODO use reference static TlsAbstraction* getTlsAbstraction() { - static TlsAbstraction* instance = &getTlsAbstraction_(); - return DisposedSingletonMark::isDisposed() ? NULL : instance; + static TlsAbstraction *g_tls = new TlsAbstraction(); // memory leak is intended here to avoid disposing of TLS container + static TlsAbstractionReleaseGuard g_tlsReleaseGuard(*g_tls); + return g_tls; } @@ -1388,15 +1406,15 @@ static TlsAbstraction* getTlsAbstraction() #ifdef WINRT static __declspec( thread ) void* tlsData = NULL; // using C++11 thread attribute for local thread data TlsAbstraction::TlsAbstraction() {} -TlsAbstraction::~TlsAbstraction() +void TlsAbstraction::releaseSystemResources() { cv::__termination = true; // DllMain is missing in static builds } -void* TlsAbstraction::getData_() const +void* TlsAbstraction::getData() const { return tlsData; } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { tlsData = pData; } @@ -1405,6 +1423,7 @@ void TlsAbstraction::setData_(void *pData) static void NTAPI opencv_fls_destructor(void* pData); #endif // CV_USE_FLS TlsAbstraction::TlsAbstraction() + : disposed(false) { #ifndef CV_USE_FLS tlsKey = TlsAlloc(); @@ -1413,9 +1432,10 @@ TlsAbstraction::TlsAbstraction() #endif // CV_USE_FLS CV_Assert(tlsKey != TLS_OUT_OF_INDEXES); } -TlsAbstraction::~TlsAbstraction() +void TlsAbstraction::releaseSystemResources() { cv::__termination = true; // DllMain is missing in static builds + disposed = true; #ifndef CV_USE_FLS TlsFree(tlsKey); #else // CV_USE_FLS @@ -1423,16 +1443,20 @@ TlsAbstraction::~TlsAbstraction() #endif // CV_USE_FLS tlsKey = TLS_OUT_OF_INDEXES; } -void* TlsAbstraction::getData_() const +void* TlsAbstraction::getData() const { + if (disposed) + return NULL; #ifndef CV_USE_FLS return TlsGetValue(tlsKey); #else // CV_USE_FLS return FlsGetValue(tlsKey); #endif // CV_USE_FLS } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { + if (disposed) + return; // no-op #ifndef CV_USE_FLS CV_Assert(TlsSetValue(tlsKey, pData) == TRUE); #else // CV_USE_FLS @@ -1443,12 +1467,14 @@ void TlsAbstraction::setData_(void *pData) #else // _WIN32 static void opencv_tls_destructor(void* pData); TlsAbstraction::TlsAbstraction() + : disposed(false) { CV_Assert(pthread_key_create(&tlsKey, opencv_tls_destructor) == 0); } -TlsAbstraction::~TlsAbstraction() +void TlsAbstraction::releaseSystemResources() { cv::__termination = true; // DllMain is missing in static builds + disposed = true; if (pthread_key_delete(tlsKey) != 0) { // Don't use logging here @@ -1456,12 +1482,16 @@ TlsAbstraction::~TlsAbstraction() fflush(stderr); } } -void* TlsAbstraction::getData_() const +void* TlsAbstraction::getData() const { + if (disposed) + return NULL; return pthread_getspecific(tlsKey); } -void TlsAbstraction::setData_(void *pData) +void TlsAbstraction::setData(void *pData) { + if (disposed) + return; // no-op CV_Assert(pthread_setspecific(tlsKey, pData) == 0); } #endif @@ -1489,6 +1519,7 @@ public: TlsStorage() : tlsSlotsSize(0) { + (void)getTlsAbstraction(); // ensure singeton initialization (for correct order of atexit calls) tlsSlots.reserve(32); threads.reserve(32); g_isTlsStorageInitialized = true; @@ -1726,14 +1757,129 @@ static void WINAPI opencv_fls_destructor(void* pData) #endif // CV_USE_FLS #endif // _WIN32 +static TlsStorage* const g_force_initialization_of_TlsStorage +#if defined __GNUC__ + __attribute__((unused)) +#endif + = &getTlsStorage(); + + +#else // OPENCV_DISABLE_THREAD_SUPPORT + +// no threading (OPENCV_DISABLE_THREAD_SUPPORT=ON) +class TlsStorage +{ +public: + TlsStorage() + { + slots.reserve(32); + } + ~TlsStorage() + { + for (size_t slotIdx = 0; slotIdx < slots.size(); slotIdx++) + { + SlotInfo& s = slots[slotIdx]; + TLSDataContainer* container = s.container; + if (container && s.data) + { + container->deleteDataInstance(s.data); // Can't use from SlotInfo destructor + s.data = nullptr; + } + } + } + + // Reserve TLS storage index + size_t reserveSlot(TLSDataContainer* container) + { + size_t slotsSize = slots.size(); + for (size_t slot = 0; slot < slotsSize; slot++) + { + SlotInfo& s = slots[slot]; + if (s.container == NULL) + { + CV_Assert(!s.data); + s.container = container; + return slot; + } + } + + // create new slot + slots.push_back(SlotInfo(container)); + return slotsSize; + } + + // Release TLS storage index and pass associated data to caller + void releaseSlot(size_t slotIdx, std::vector &dataVec, bool keepSlot = false) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + void* data = s.data; + if (data) + { + dataVec.push_back(data); + s.data = nullptr; + } + if (!keepSlot) + { + s.container = NULL; // mark slot as free (see reserveSlot() implementation) + } + } + + // Get data by TLS storage index + void* getData(size_t slotIdx) const + { + CV_Assert(slotIdx < slots.size()); + const SlotInfo& s = slots[slotIdx]; + return s.data; + } + + // Gather data from threads by TLS storage index + void gather(size_t slotIdx, std::vector &dataVec) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + void* data = s.data; + if (data) + dataVec.push_back(data); + return; + } + + // Set data to storage index + void setData(size_t slotIdx, void* pData) + { + CV_Assert(slotIdx < slots.size()); + SlotInfo& s = slots[slotIdx]; + s.data = pData; + } + +private: + struct SlotInfo + { + SlotInfo(TLSDataContainer* _container) : container(_container), data(nullptr) {} + TLSDataContainer* container; // attached container (to dispose data) + void* data; + }; + std::vector slots; +}; + +static TlsStorage& getTlsStorage() +{ + static TlsStorage g_storage; // no threading + return g_storage; +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + } // namespace details using namespace details; void releaseTlsStorageThread() { +#ifndef OPENCV_DISABLE_THREAD_SUPPORT if (!g_isTlsStorageInitialized) return; // nothing to release, so prefer to avoid creation of new global structures getTlsStorage().releaseThread(); +#endif } TLSDataContainer::TLSDataContainer() @@ -1783,7 +1929,15 @@ void* TLSDataContainer::getData() const { // Create new data instance and save it to TLS storage pData = createDataInstance(); - getTlsStorage().setData(key_, pData); + try + { + getTlsStorage().setData(key_, pData); + } + catch (...) + { + deleteDataInstance(pData); + throw; + } } return pData; } diff --git a/modules/core/src/umatrix.cpp b/modules/core/src/umatrix.cpp index c80d240ecc..846d406eca 100644 --- a/modules/core/src/umatrix.cpp +++ b/modules/core/src/umatrix.cpp @@ -56,10 +56,6 @@ void setSize(UMat& m, int _dims, const int* _sz, const size_t* _steps, void updateContinuityFlag(UMat& m); void finalizeHdr(UMat& m); -// it should be a prime number for the best hash function -enum { UMAT_NLOCKS = 31 }; -static Mutex umatLocks[UMAT_NLOCKS]; - UMatData::UMatData(const MatAllocator* allocator) { prevAllocator = currAllocator = allocator; @@ -131,6 +127,12 @@ UMatData::~UMatData() } } +#ifndef OPENCV_DISABLE_THREAD_SUPPORT + +// it should be a prime number for the best hash function +enum { UMAT_NLOCKS = 31 }; +static Mutex umatLocks[UMAT_NLOCKS]; + static size_t getUMatDataLockIndex(const UMatData* u) { size_t idx = ((size_t)(void*)u) % UMAT_NLOCKS; @@ -228,6 +230,33 @@ UMatDataAutoLock::~UMatDataAutoLock() getUMatDataAutoLocker().release(u1, u2); } +#else + +void UMatData::lock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +void UMatData::unlock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +UMatDataAutoLock::UMatDataAutoLock(UMatData* u) : u1(u), u2(NULL) +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} +UMatDataAutoLock::UMatDataAutoLock(UMatData* u1_, UMatData* u2_) : u1(u1_), u2(u2_) +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} +UMatDataAutoLock::~UMatDataAutoLock() +{ + // nothing in OPENCV_DISABLE_THREAD_SUPPORT mode +} + +#endif // OPENCV_DISABLE_THREAD_SUPPORT + //////////////////////////////// UMat //////////////////////////////// UMat::UMat(UMatUsageFlags _usageFlags) CV_NOEXCEPT @@ -597,14 +626,28 @@ UMat Mat::getUMat(AccessFlag accessFlags, UMatUsageFlags usageFlags) const CV_XADD(&(u->refcount), 1); CV_XADD(&(u->urefcount), 1); } - hdr.flags = flags; - hdr.usageFlags = usageFlags; - setSize(hdr, dims, size.p, step.p); - finalizeHdr(hdr); - hdr.u = new_u; - hdr.offset = 0; //data - datastart; - hdr.addref(); - return hdr; + try + { + hdr.flags = flags; + hdr.usageFlags = usageFlags; + setSize(hdr, dims, size.p, step.p); + finalizeHdr(hdr); + hdr.u = new_u; + hdr.offset = 0; //data - datastart; + hdr.addref(); + return hdr; + } + catch(...) + { + if (u != NULL) + { + CV_XADD(&(u->refcount), -1); + CV_XADD(&(u->urefcount), -1); + } + new_u->currAllocator->deallocate(new_u); + throw; + } + } void UMat::create(int d, const int* _sizes, int _type, UMatUsageFlags _usageFlags) @@ -756,18 +799,17 @@ UMat::UMat(const UMat& m, const Rect& roi) offset += roi.x*esz; CV_Assert( 0 <= roi.x && 0 <= roi.width && roi.x + roi.width <= m.cols && 0 <= roi.y && 0 <= roi.height && roi.y + roi.height <= m.rows ); - if( u ) - CV_XADD(&(u->urefcount), 1); if( roi.width < m.cols || roi.height < m.rows ) flags |= SUBMATRIX_FLAG; step[0] = m.step[0]; step[1] = esz; updateContinuityFlag(); + addref(); if( rows <= 0 || cols <= 0 ) { - release(); rows = cols = 0; + release(); } } @@ -951,11 +993,11 @@ UMat UMat::reshape(int new_cn, int new_rows) const return hdr; } -UMat UMat::diag(const UMat& d) +UMat UMat::diag(const UMat& d, UMatUsageFlags usageFlags) { CV_Assert( d.cols == 1 || d.rows == 1 ); int len = d.rows + d.cols - 1; - UMat m(len, len, d.type(), Scalar(0)); + UMat m(len, len, d.type(), Scalar(0), usageFlags); UMat md = m.diag(); if( d.cols == 1 ) d.copyTo(md); @@ -1033,24 +1075,29 @@ Mat UMat::getMat(AccessFlag accessFlags) const // TODO Support ACCESS_READ (ACCESS_WRITE) without unnecessary data transfers accessFlags |= ACCESS_RW; UMatDataAutoLock autolock(u); - if(CV_XADD(&u->refcount, 1) == 0) - u->currAllocator->map(u, accessFlags); - if (u->data != 0) + try { - Mat hdr(dims, size.p, type(), u->data + offset, step.p); - hdr.flags = flags; - hdr.u = u; - hdr.datastart = u->data; - hdr.data = u->data + offset; - hdr.datalimit = hdr.dataend = u->data + u->size; - return hdr; + if(CV_XADD(&u->refcount, 1) == 0) + u->currAllocator->map(u, accessFlags); + if (u->data != 0) + { + Mat hdr(dims, size.p, type(), u->data + offset, step.p); + hdr.flags = flags; + hdr.u = u; + hdr.datastart = u->data; + hdr.data = u->data + offset; + hdr.datalimit = hdr.dataend = u->data + u->size; + return hdr; + } } - else + catch(...) { CV_XADD(&u->refcount, -1); - CV_Assert(u->data != 0 && "Error mapping of UMat to host memory."); - return Mat(); + throw; } + CV_XADD(&u->refcount, -1); + CV_Assert(u->data != 0 && "Error mapping of UMat to host memory."); + return Mat(); } void* UMat::handle(AccessFlag accessFlags) const @@ -1323,34 +1370,34 @@ UMat UMat::t() const return m; } -UMat UMat::zeros(int rows, int cols, int type) +UMat UMat::zeros(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat(rows, cols, type, Scalar::all(0)); + return UMat(rows, cols, type, Scalar::all(0), usageFlags); } -UMat UMat::zeros(Size size, int type) +UMat UMat::zeros(Size size, int type, UMatUsageFlags usageFlags) { - return UMat(size, type, Scalar::all(0)); + return UMat(size, type, Scalar::all(0), usageFlags); } -UMat UMat::zeros(int ndims, const int* sz, int type) +UMat UMat::zeros(int ndims, const int* sz, int type, UMatUsageFlags usageFlags) { - return UMat(ndims, sz, type, Scalar::all(0)); + return UMat(ndims, sz, type, Scalar::all(0), usageFlags); } -UMat UMat::ones(int rows, int cols, int type) +UMat UMat::ones(int rows, int cols, int type, UMatUsageFlags usageFlags) { - return UMat::ones(Size(cols, rows), type); + return UMat(rows, cols, type, Scalar(1), usageFlags); } -UMat UMat::ones(Size size, int type) +UMat UMat::ones(Size size, int type, UMatUsageFlags usageFlags) { - return UMat(size, type, Scalar(1)); + return UMat(size, type, Scalar(1), usageFlags); } -UMat UMat::ones(int ndims, const int* sz, int type) +UMat UMat::ones(int ndims, const int* sz, int type, UMatUsageFlags usageFlags) { - return UMat(ndims, sz, type, Scalar(1)); + return UMat(ndims, sz, type, Scalar(1), usageFlags); } } diff --git a/modules/core/src/utils/logtagmanager.hpp b/modules/core/src/utils/logtagmanager.hpp index 29a1776ada..ab4bb9b7d3 100644 --- a/modules/core/src/utils/logtagmanager.hpp +++ b/modules/core/src/utils/logtagmanager.hpp @@ -37,8 +37,8 @@ private: // also, extensible functions (accepting user-provided callback) are not allowed // to call LogTagManger (to prevent iterator invalidation), which needs enforced // with a non-recursive mutex. - using MutexType = std::mutex; - using LockType = std::lock_guard; + using MutexType = cv::Mutex; + using LockType = cv::AutoLock; enum class MatchingScope { diff --git a/modules/core/test/ocl/test_gemm.cpp b/modules/core/test/ocl/test_gemm.cpp index 825b506780..cb7cb0be1a 100644 --- a/modules/core/test/ocl/test_gemm.cpp +++ b/modules/core/test/ocl/test_gemm.cpp @@ -67,6 +67,8 @@ PARAM_TEST_CASE(Gemm, double alpha, beta; + int M, N, K; + TEST_DECLARE_INPUT_PARAMETER(A); TEST_DECLARE_INPUT_PARAMETER(B); TEST_DECLARE_INPUT_PARAMETER(C); @@ -90,30 +92,27 @@ PARAM_TEST_CASE(Gemm, void generateTestData() { - // set minimum size to 20, since testing less sizes doesn't make sense - Size ARoiSize = randomSize(20, MAX_VALUE); + M = (int)randomDoubleLog(1, 100); + N = (int)randomDoubleLog(1, 100); + K = (int)randomDoubleLog(1, 1200); + + M = roundUp(M, 1); + N = roundUp(N, 1); + K = roundUp(K, 1); + + Size ARoiSize = (atrans) ? Size(M, K) : Size(K, M); Border ABorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(A, A_roi, ARoiSize, ABorder, type, -11, 11); - if (atrans) - ARoiSize = Size(ARoiSize.height, ARoiSize.width); - - Size BRoiSize = randomSize(20, MAX_VALUE); - if (btrans) - BRoiSize.width = ARoiSize.width; - else - BRoiSize.height = ARoiSize.width; - + Size BRoiSize = (btrans) ? Size(K, N) : Size(N, K); Border BBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(B, B_roi, BRoiSize, BBorder, type, -11, 11); - if (btrans) - BRoiSize = Size(BRoiSize.height, BRoiSize.width); - - Size DRoiSize = Size(BRoiSize.width, ARoiSize.height), CRoiSizeT(DRoiSize.height, DRoiSize.width); + Size CRoiSize = (ctrans) ? Size(M, N) : Size(N, M); Border CBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); - randomSubMat(C, C_roi, ctrans ? CRoiSizeT : DRoiSize, CBorder, type, -11, 11); + randomSubMat(C, C_roi, CRoiSize, CBorder, type, -11, 11); + Size DRoiSize = Size(N, M); Border DBorder = randomBorder(0, use_roi ? MAX_VALUE : 0); randomSubMat(D, D_roi, DRoiSize, DBorder, type, -11, 11); @@ -132,11 +131,12 @@ OCL_TEST_P(Gemm, Accuracy) for (int i = 0; i < test_loop_times; ++i) { generateTestData(); + SCOPED_TRACE(cv::format("i=%d: M=%d N=%d K=%d", i, M, N, K)); OCL_OFF(cv::gemm(A_roi, B_roi, alpha, C_roi, beta, D_roi, flags)); OCL_ON(cv::gemm(uA_roi, uB_roi, alpha, uC_roi, beta, uD_roi, flags)); - double eps = D_roi.size().area() * 1e-4; + double eps = D_roi.size().area() * (1e-5 * K); OCL_EXPECT_MATS_NEAR(D, eps); } } diff --git a/modules/core/test/ocl/test_matrix_expr.cpp b/modules/core/test/ocl/test_matrix_expr.cpp index 7a5ff72cb2..f11c0a6ebb 100644 --- a/modules/core/test/ocl/test_matrix_expr.cpp +++ b/modules/core/test/ocl/test_matrix_expr.cpp @@ -76,6 +76,24 @@ OCL_TEST_P(UMatExpr, Ones) } } +//////////////////////////////// with usageFlags ///////////////////////////////////////////////// + +OCL_TEST_P(UMatExpr, WithUsageFlags) +{ + for (int j = 0; j < test_loop_times; j++) + { + generateTestData(); + + UMat u0 = UMat::zeros(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + UMat u1 = UMat::ones(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + UMat u8 = UMat::eye(size, type, cv::USAGE_ALLOCATE_HOST_MEMORY); + + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u0.usageFlags); + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u1.usageFlags); + EXPECT_EQ(cv::USAGE_ALLOCATE_HOST_MEMORY, u8.usageFlags); + } +} + //////////////////////////////// Instantiation ///////////////////////////////////////////////// OCL_INSTANTIATE_TEST_CASE_P(MatrixOperation, UMatExpr, Combine(OCL_ALL_DEPTHS_16F, OCL_ALL_CHANNELS)); diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp index effb0e68e0..9e8e242d60 100644 --- a/modules/core/test/test_arithm.cpp +++ b/modules/core/test/test_arithm.cpp @@ -2166,6 +2166,15 @@ TEST(Core_Norm, IPP_regression_NORM_L1_16UC3_small) EXPECT_EQ((double)20*cn, cv::norm(a, b, NORM_L1, mask)); } +TEST(Core_Norm, NORM_L2_8UC4) +{ + // Tests there is no integer overflow in norm computation for multiple channels. + const int kSide = 100; + cv::Mat4b a(kSide, kSide, cv::Scalar(255, 255, 255, 255)); + cv::Mat4b b = cv::Mat4b::zeros(kSide, kSide); + const double kNorm = 2.*kSide*255.; + EXPECT_EQ(kNorm, cv::norm(a, b, NORM_L2)); +} TEST(Core_ConvertTo, regression_12121) { diff --git a/modules/core/test/test_async.cpp b/modules/core/test/test_async.cpp index f898a22878..58bcfddcd7 100644 --- a/modules/core/test/test_async.cpp +++ b/modules/core/test/test_async.cpp @@ -7,7 +7,7 @@ #include -#ifdef CV_CXX11 +#if defined(CV_CXX11) && !defined(OPENCV_DISABLE_THREAD_SUPPORT) #include #include #endif @@ -85,7 +85,8 @@ TEST(Core_Async, LikePythonTest) } -#ifdef CV_CXX11 +#if defined(CV_CXX11) && !defined(OPENCV_DISABLE_THREAD_SUPPORT) + TEST(Core_Async, AsyncThread_Simple) { Mat m(3, 3, CV_32FC1, Scalar::all(5.0f)); diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index d30c485368..3712be9f2e 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -586,6 +586,7 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); CV_Assert(test_info); std::string name = (std::string(test_info->test_case_name()) + "--" + test_info->name() + suffix_name); + std::string name_34 = string(cvtest::TS::ptr()->get_data_path()) + "io/3_4/" + name; if (!testReadWrite) name = string(cvtest::TS::ptr()->get_data_path()) + "io/" + name; @@ -661,7 +662,23 @@ static void test_filestorage_basic(int write_flags, const char* suffix_name, boo std::ifstream f(name.c_str(), std::ios::in|std::ios::binary); f.seekg(0, std::fstream::end); sz = (size_t)f.tellg(); + + f.seekg(0, std::ios::beg); + std::vector test_data(sz); + f.read(&test_data[0], sz); f.close(); + + std::ifstream reference(name_34.c_str(), std::ios::in|std::ios::binary); + ASSERT_TRUE(reference.is_open()); + reference.seekg(0, std::fstream::end); + size_t ref_sz = (size_t)reference.tellg(); + + reference.seekg(0, std::ios::beg); + std::vector reference_data(ref_sz); + reference.read(&reference_data[0], ref_sz); + reference.close(); + + EXPECT_EQ(reference_data, test_data); } std::cout << "Storage size: " << sz << std::endl; EXPECT_LE(sz, (size_t)6000); @@ -757,27 +774,27 @@ TEST(Core_InputOutput, filestorage_base64_basic_read_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", false); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_XML) +TEST(Core_InputOutput, filestorage_base64_basic_rw_XML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_YAML) +TEST(Core_InputOutput, filestorage_base64_basic_rw_YAML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_rw_JSON) +TEST(Core_InputOutput, filestorage_base64_basic_rw_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_XML) +TEST(Core_InputOutput, filestorage_base64_basic_memory_XML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".xml", true, true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_YAML) +TEST(Core_InputOutput, filestorage_base64_basic_memory_YAML) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".yml", true, true); } -TEST(Core_InputOutput, DISABLED_filestorage_base64_basic_memory_JSON) +TEST(Core_InputOutput, filestorage_base64_basic_memory_JSON) { test_filestorage_basic(cv::FileStorage::WRITE_BASE64, ".json", true, true); } @@ -1837,4 +1854,69 @@ TEST(Core_InputOutput, FileStorage_copy_constructor_17412_heap) EXPECT_EQ(0, remove(fname.c_str())); } + +static void test_20279(FileStorage& fs) +{ + Mat m32fc1(5, 10, CV_32FC1, Scalar::all(0)); + for (size_t i = 0; i < m32fc1.total(); i++) + { + float v = (float)i; + m32fc1.at((int)i) = v * 0.5f; + } + Mat m16fc1; + // produces CV_16S output: convertFp16(m32fc1, m16fc1); + m32fc1.convertTo(m16fc1, CV_16FC1); + EXPECT_EQ(CV_16FC1, m16fc1.type()) << typeToString(m16fc1.type()); + //std::cout << m16fc1 << std::endl; + + Mat m32fc3(4, 3, CV_32FC3, Scalar::all(0)); + for (size_t i = 0; i < m32fc3.total(); i++) + { + float v = (float)i; + m32fc3.at((int)i) = Vec3f(v, v * 0.2f, -v); + } + Mat m16fc3; + m32fc3.convertTo(m16fc3, CV_16FC3); + EXPECT_EQ(CV_16FC3, m16fc3.type()) << typeToString(m16fc3.type()); + //std::cout << m16fc3 << std::endl; + + fs << "m16fc1" << m16fc1; + fs << "m16fc3" << m16fc3; + + string content = fs.releaseAndGetString(); + if (cvtest::debugLevel > 0) std::cout << content << std::endl; + + FileStorage fs_read(content, FileStorage::READ + FileStorage::MEMORY); + Mat m16fc1_result; + Mat m16fc3_result; + fs_read["m16fc1"] >> m16fc1_result; + ASSERT_FALSE(m16fc1_result.empty()); + EXPECT_EQ(CV_16FC1, m16fc1_result.type()) << typeToString(m16fc1_result.type()); + EXPECT_LE(cvtest::norm(m16fc1_result, m16fc1, NORM_INF), 1e-2); + + fs_read["m16fc3"] >> m16fc3_result; + ASSERT_FALSE(m16fc3_result.empty()); + EXPECT_EQ(CV_16FC3, m16fc3_result.type()) << typeToString(m16fc3_result.type()); + EXPECT_LE(cvtest::norm(m16fc3_result, m16fc3, NORM_INF), 1e-2); +} + +TEST(Core_InputOutput, FileStorage_16F_xml) +{ + FileStorage fs("test.xml", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + +TEST(Core_InputOutput, FileStorage_16F_yml) +{ + FileStorage fs("test.yml", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + +TEST(Core_InputOutput, FileStorage_16F_json) +{ + FileStorage fs("test.json", cv::FileStorage::WRITE | cv::FileStorage::MEMORY); + test_20279(fs); +} + + }} // namespace diff --git a/modules/core/test/test_misc.cpp b/modules/core/test/test_misc.cpp index 67d0a53995..55615b0d5f 100644 --- a/modules/core/test/test_misc.cpp +++ b/modules/core/test/test_misc.cpp @@ -798,4 +798,27 @@ TEST(Core_Allocation, alignedAllocation) } } + +#if !(defined(__GNUC__) && __GNUC__ < 5) // GCC 4.8 emits: 'is_trivially_copyable' is not a member of 'std' +TEST(Core_Types, trivially_copyable) +{ + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + //EXPECT_TRUE(std::is_trivially_copyable::value); // derived from Vec (Matx) +} + +TEST(Core_Types, trivially_copyable_extra) +{ + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); + EXPECT_TRUE(std::is_trivially_copyable::value); +} +#endif + }} // namespace diff --git a/modules/core/test/test_umat.cpp b/modules/core/test/test_umat.cpp index c2bd6eceba..c323d17c06 100644 --- a/modules/core/test/test_umat.cpp +++ b/modules/core/test/test_umat.cpp @@ -1419,4 +1419,37 @@ TEST(UMat, resize_Mat_issue_13577) cv::ocl::setUseOpenCL(useOCL); // restore state } +TEST(UMat, exceptions_refcounts_issue_20594) +{ + if (!cv::ocl::useOpenCL()) + { + // skip test, difficult to create exception scenario without OpenCL + std::cout << "OpenCL is not enabled. Skip test" << std::endl; + return; + } + + UMat umat1(10, 10, CV_8UC1); + EXPECT_EQ(0, umat1.u->refcount); + + // cause exception in underlying allocator + void* const original_handle = umat1.u->handle; + umat1.u->handle = NULL; + try + { + Mat mat1 = umat1.getMat(ACCESS_RW); + } + catch (...) + { + // nothing + } + + // check for correct refcount, and no change of intentional bad handle + EXPECT_EQ(0, umat1.u->refcount); + EXPECT_EQ(NULL, umat1.u->handle); + + // reset UMat to good state + umat1.u->refcount = 0; + umat1.u->handle = original_handle; +} + } } // namespace opencv_test::ocl diff --git a/modules/core/test/test_utils.cpp b/modules/core/test/test_utils.cpp index ed5f34603d..c31ca75667 100644 --- a/modules/core/test/test_utils.cpp +++ b/modules/core/test/test_utils.cpp @@ -8,9 +8,12 @@ #include "opencv2/core/utils/logger.hpp" #include "opencv2/core/utils/buffer_area.private.hpp" -#include "test_utils_tls.impl.hpp" #include "opencv2/core/utils/filesystem.private.hpp" +#ifndef OPENCV_DISABLE_THREAD_SUPPORT +#include "test_utils_tls.impl.hpp" +#endif + namespace opencv_test { namespace { static const char * const keys = diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 4c8129cbda..c1236c4653 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -8,7 +8,8 @@ endif() set(the_description "Deep neural network module. It allows to load models from different frameworks and to make forward pass") -ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX) +ocv_add_dispatched_file_force_all("layers/layers_common" AVX AVX2 AVX512_SKX RVV) +ocv_add_dispatched_file_force_all("int8layers/layers_common" AVX2 AVX512_SKX) ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js) diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 24d35646df..9fde7ad4c4 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -165,6 +165,40 @@ CV__DNN_INLINE_NS_BEGIN int outputNameToIndex(const String& outputName) CV_OVERRIDE; }; + /** @brief GRU recurrent one-layer + * + * Accepts input sequence and computes the final hidden state for each element in the batch. + * + * - input[0] containing the features of the input sequence. + * input[0] should have shape [`T`, `N`, `data_dims`] where `T` is sequence length, `N` is batch size, `data_dims` is input size + * - output would have shape [`T`, `N`, `D` * `hidden_size`] where `D = 2` if layer is bidirectional otherwise `D = 1` + * + * Depends on the following attributes: + * - hidden_size - Number of neurons in the hidden layer + * - direction - RNN could be bidirectional or forward + * + * The final hidden state @f$ h_t @f$ computes by the following formulas: + * + @f{eqnarray*}{ + r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ + z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\ + n_t = \tanh(W_{in} x_t + b_{in} + r_t \odot (W_{hn} h_{(t-1)}+ b_{hn})) \\ + h_t = (1 - z_t) \odot n_t + z_t \odot h_{(t-1)} \\ + @f} + * Where @f$x_t@f$ is current input, @f$h_{(t-1)}@f$ is previous or initial hidden state. + * + * @f$W_{x?}@f$, @f$W_{h?}@f$ and @f$b_{?}@f$ are learned weights represented as matrices: + * @f$W_{x?} \in R^{N_h \times N_x}@f$, @f$W_{h?} \in R^{N_h \times N_h}@f$, @f$b_? \in R^{N_h}@f$. + * + * @f$\odot@f$ is per-element multiply operation. + */ + class CV_EXPORTS GRULayer : public Layer + { + public: + /** Creates instance of GRU layer */ + static Ptr create(const LayerParams& params); + }; + /** @brief Classical recurrent layer Accepts two inputs @f$x_t@f$ and @f$h_{t-1}@f$ and compute two outputs @f$o_t@f$ and @f$h_t@f$. @@ -224,6 +258,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ConvolutionLayerInt8 : public BaseConvolutionLayer + { + public: + int input_zp, output_zp; + float output_sc; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS DeconvolutionLayer : public BaseConvolutionLayer { public: @@ -266,6 +308,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS PoolingLayerInt8 : public PoolingLayer + { + public: + int input_zp, output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS SoftmaxLayer : public Layer { public: @@ -274,6 +323,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS SoftmaxLayerInt8 : public SoftmaxLayer + { + public: + float output_sc; + int output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS InnerProductLayer : public Layer { public: @@ -281,6 +338,13 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS InnerProductLayerInt8 : public InnerProductLayer + { + public: + int output_zp; + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS MVNLayer : public Layer { public: @@ -307,6 +371,29 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS QuantizeLayer : public Layer + { + public: + float scale; + int zeropoint; + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS DequantizeLayer : public Layer + { + public: + float scale; + int zeropoint; + static Ptr create(const LayerParams ¶ms); + }; + + class CV_EXPORTS RequantizeLayer : public Layer + { + public: + float scale, shift; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ConcatLayer : public Layer { public: @@ -318,6 +405,7 @@ CV__DNN_INLINE_NS_BEGIN * Details: https://github.com/torch/nn/blob/master/doc/containers.md#depthconcat */ bool padding; + int paddingValue; static Ptr create(const LayerParams ¶ms); }; @@ -425,7 +513,11 @@ CV__DNN_INLINE_NS_BEGIN { public: virtual void forwardSlice(const float* src, float* dst, int len, - size_t outPlaneSize, int cn0, int cn1) const = 0; + size_t outPlaneSize, int cn0, int cn1) const {}; + virtual void forwardSlice(const int* src, const int* lut, int* dst, int len, + size_t outPlaneSize, int cn0, int cn1) const {}; + virtual void forwardSlice(const int8_t* src, const int8_t* lut, int8_t* dst, int len, + size_t outPlaneSize, int cn0, int cn1) const {}; }; class CV_EXPORTS ReLULayer : public ActivationLayer @@ -508,6 +600,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + /* Layers used in semantic segmentation */ class CV_EXPORTS CropLayer : public Layer @@ -519,7 +617,7 @@ CV__DNN_INLINE_NS_BEGIN /** @brief Element wise operation on inputs Extra optional parameters: - - "operation" as string. Values are "sum" (default), "prod", "max", "div" + - "operation" as string. Values are "sum" (default), "prod", "max", "div", "min" - "coeff" as float array. Specify weights of inputs for SUM operation - "output_channels_mode" as string. Values are "same" (default, all input must have the same layout), "input_0", "input_0_truncate", "max_input_channels" */ @@ -529,6 +627,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS EltwiseLayerInt8 : public Layer + { + public: + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS BatchNormLayer : public ActivationLayer { public: @@ -538,6 +642,14 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams ¶ms); }; + class CV_EXPORTS BatchNormLayerInt8 : public BatchNormLayer + { + public: + float input_sc, output_sc; + int input_zp, output_zp; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS MaxUnpoolLayer : public Layer { public: @@ -557,12 +669,26 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ScaleLayerInt8 : public ScaleLayer + { + public: + float output_sc; + int output_zp; + static Ptr create(const LayerParams ¶ms); + }; + class CV_EXPORTS ShiftLayer : public Layer { public: static Ptr create(const LayerParams& params); }; + class CV_EXPORTS ShiftLayerInt8 : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS DataAugmentationLayer : public Layer { public: @@ -689,6 +815,15 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS CumSumLayer : public Layer + { + public: + int exclusive; + int reverse; + + static Ptr create(const LayerParams& params); + }; + //! @} //! @} CV__DNN_INLINE_NS_END diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 255b41de88..8b4e5e21d4 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -235,6 +235,15 @@ CV__DNN_INLINE_NS_BEGIN */ virtual void forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals); + /** @brief Tries to quantize the given layer and compute the quantization parameters required for fixed point implementation. + * @param[in] scales input and output scales. + * @param[in] zeropoints input and output zeropoints. + * @param[out] params Quantized parameters required for fixed point implementation of that layer. + * @returns True if layer can be quantized. + */ + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params); + /** @brief Given the @p input blobs, computes the output @p blobs. * @param[in] inputs the input blobs. * @param[out] outputs allocated output blobs, which will store results of the computation. @@ -368,6 +377,16 @@ CV__DNN_INLINE_NS_BEGIN */ virtual void getScaleShift(Mat& scale, Mat& shift) const; + /** + * @brief Returns scale and zeropoint of layers + * @param[out] scale Output scale + * @param[out] zeropoint Output zeropoint + * + * By default, @p scale is 1 and @p zeropoint is 0. + */ + virtual void getScaleZeropoint(float& scale, int& zeropoint) const; + + /** * @brief "Deattaches" all the layers, attached to particular layer. */ @@ -453,13 +472,21 @@ CV__DNN_INLINE_NS_BEGIN /** @brief Adds new layer to the net. * @param name unique name of the adding layer. * @param type typename of the adding layer (type must be registered in LayerRegister). + * @param dtype datatype of output blobs. * @param params parameters which will be used to initialize the creating layer. * @returns unique identifier of created layer, or -1 if a failure will happen. */ + int addLayer(const String &name, const String &type, const int &dtype, LayerParams ¶ms); + + /** @overload Datatype of output blobs set to default CV_32F */ int addLayer(const String &name, const String &type, LayerParams ¶ms); + /** @brief Adds new layer and connects its first input to the first output of previously added layer. * @see addLayer() */ + int addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams ¶ms); + + /** @overload */ int addLayerToPrev(const String &name, const String &type, LayerParams ¶ms); /** @brief Converts string name of the layer to the integer identifier. @@ -551,6 +578,25 @@ CV__DNN_INLINE_NS_BEGIN CV_WRAP_AS(forwardAndRetrieve) void forward(CV_OUT std::vector >& outputBlobs, const std::vector& outBlobNames); + /** @brief Returns a quantized Net from a floating-point Net. + * @param calibData Calibration data to compute the quantization parameters. + * @param inputsDtype Datatype of quantized net's inputs. Can be CV_32F or CV_8S. + * @param outputsDtype Datatype of quantized net's outputs. Can be CV_32F or CV_8S. + */ + CV_WRAP Net quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype); + + /** @brief Returns input scale and zeropoint for a quantized Net. + * @param scales output parameter for returning input scales. + * @param zeropoints output parameter for returning input zeropoints. + */ + CV_WRAP void getInputDetails(CV_OUT std::vector& scales, CV_OUT std::vector& zeropoints) const; + + /** @brief Returns output scale and zeropoint for a quantized Net. + * @param scales output parameter for returning output scales. + * @param zeropoints output parameter for returning output zeropoints. + */ + CV_WRAP void getOutputDetails(CV_OUT std::vector& scales, CV_OUT std::vector& zeropoints) const; + /** * @brief Compile Halide layers. * @param[in] scheduler Path to YAML file with scheduling directives. @@ -1084,6 +1130,39 @@ CV__DNN_INLINE_NS_BEGIN CV_OUT std::vector& indices, const float eta = 1.f, const int top_k = 0); + /** + * @brief Enum of Soft NMS methods. + * @see softNMSBoxes + */ + enum class SoftNMSMethod + { + SOFTNMS_LINEAR = 1, + SOFTNMS_GAUSSIAN = 2 + }; + + /** @brief Performs soft non maximum suppression given boxes and corresponding scores. + * Reference: https://arxiv.org/abs/1704.04503 + * @param bboxes a set of bounding boxes to apply Soft NMS. + * @param scores a set of corresponding confidences. + * @param updated_scores a set of corresponding updated confidences. + * @param score_threshold a threshold used to filter boxes by score. + * @param nms_threshold a threshold used in non maximum suppression. + * @param indices the kept indices of bboxes after NMS. + * @param top_k keep at most @p top_k picked indices. + * @param sigma parameter of Gaussian weighting. + * @param method Gaussian or linear. + * @see SoftNMSMethod + */ + CV_EXPORTS_W void softNMSBoxes(const std::vector& bboxes, + const std::vector& scores, + CV_OUT std::vector& updated_scores, + const float score_threshold, + const float nms_threshold, + CV_OUT std::vector& indices, + size_t top_k = 0, + const float sigma = 0.5, + SoftNMSMethod method = SoftNMSMethod::SOFTNMS_GAUSSIAN); + /** @brief This class is presented high-level API for neural networks. * @@ -1373,7 +1452,9 @@ public: /** * @brief Set the decoding method of translating the network output into string - * @param[in] decodeType The decoding method of translating the network output into string: {'CTC-greedy': greedy decoding for the output of CTC-based methods} + * @param[in] decodeType The decoding method of translating the network output into string, currently supported type: + * - `"CTC-greedy"` greedy decoding for the output of CTC-based methods + * - `"CTC-prefix-beam-search"` Prefix beam search decoding for the output of CTC-based methods */ CV_WRAP TextRecognitionModel& setDecodeType(const std::string& decodeType); @@ -1385,6 +1466,15 @@ public: CV_WRAP const std::string& getDecodeType() const; + /** + * @brief Set the decoding method options for `"CTC-prefix-beam-search"` decode usage + * @param[in] beamSize Beam size for search + * @param[in] vocPruneSize Parameter to optimize big vocabulary search, + * only take top @p vocPruneSize tokens in each search step, @p vocPruneSize <= 0 stands for disable this prune. + */ + CV_WRAP + TextRecognitionModel& setDecodeOptsCTCPrefixBeamSearch(int beamSize, int vocPruneSize = 0); + /** * @brief Set the vocabulary for recognition. * @param[in] vocabulary the associated vocabulary of the network. diff --git a/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp b/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp index 46a58f09bc..e944644f8f 100644 --- a/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp +++ b/modules/dnn/include/opencv2/dnn/layer_reg.private.hpp @@ -12,10 +12,16 @@ CV__DNN_INLINE_NS_BEGIN //! @addtogroup dnn //! @{ -//! Register layer types of DNN model. typedef std::map > LayerFactory_Impl; + +//! Register layer types of DNN model. +//! +//! @note In order to thread-safely access the factory, see getLayerFactoryMutex() function. LayerFactory_Impl& getLayerFactoryImpl(); +//! Get the mutex guarding @ref LayerFactory_Impl, see getLayerFactoryImpl() function. +Mutex& getLayerFactoryMutex(); + //! @} CV__DNN_INLINE_NS_END } diff --git a/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp b/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp new file mode 100644 index 0000000000..71dd3ab8d6 --- /dev/null +++ b/modules/dnn/include/opencv2/dnn/utils/debug_utils.hpp @@ -0,0 +1,24 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_DNN_UTILS_DEBUG_UTILS_HPP +#define OPENCV_DNN_UTILS_DEBUG_UTILS_HPP + +#include "../dnn.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +/** + * @brief Skip model import after diagnostic run in readNet() functions. + * @param[in] skip Indicates whether to skip the import. + * + * This is an internal OpenCV function not intended for users. + */ +CV_EXPORTS void skipModelImport(bool skip); + +CV__DNN_INLINE_NS_END +}} // namespace + +#endif // OPENCV_DNN_UTILS_DEBUG_UTILS_HPP diff --git a/modules/dnn/include/opencv2/dnn/version.hpp b/modules/dnn/include/opencv2/dnn/version.hpp index 98086a80ff..fe479d4bf1 100644 --- a/modules/dnn/include/opencv2/dnn/version.hpp +++ b/modules/dnn/include/opencv2/dnn/version.hpp @@ -6,7 +6,7 @@ #define OPENCV_DNN_VERSION_HPP /// Use with major OpenCV version only. -#define OPENCV_DNN_API_VERSION 20210608 +#define OPENCV_DNN_API_VERSION 20211015 #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_INLINE_NS #define CV__DNN_INLINE_NS __CV_CAT(dnn5_v, OPENCV_DNN_API_VERSION) diff --git a/modules/dnn/misc/java/gen_dict.json b/modules/dnn/misc/java/gen_dict.json index 5a397eac51..65ecfdc25e 100644 --- a/modules/dnn/misc/java/gen_dict.json +++ b/modules/dnn/misc/java/gen_dict.json @@ -54,7 +54,7 @@ ] ], - "jni_name": "(*(cv::dnn::DictValue*)%(n)s_nativeObj)", + "jni_name": "(*(*(Ptr*)%(n)s_nativeObj))", "jni_type": "jlong", "suffix": "J", "j_import": "org.opencv.dnn.DictValue" diff --git a/modules/dnn/misc/python/test/test_dnn.py b/modules/dnn/misc/python/test/test_dnn.py index d0687ca4bc..272121ba36 100644 --- a/modules/dnn/misc/python/test/test_dnn.py +++ b/modules/dnn/misc/python/test/test_dnn.py @@ -62,6 +62,12 @@ def printParams(backend, target): } print('%s/%s' % (backendNames[backend], targetNames[target])) +def getDefaultThreshold(target): + if target == cv.dnn.DNN_TARGET_OPENCL_FP16 or target == cv.dnn.DNN_TARGET_MYRIAD: + return 4e-3 + else: + return 1e-5 + testdata_required = bool(os.environ.get('OPENCV_DNN_TEST_REQUIRE_TESTDATA', False)) g_dnnBackendsAndTargets = None @@ -212,7 +218,7 @@ class dnn_test(NewOpenCVTests): model.setInputParams(scale, size, mean) out, _ = model.detect(frame) - self.assertTrue(type(out) == list) + self.assertTrue(type(out) == tuple, msg='actual type {}'.format(str(type(out)))) self.assertTrue(np.array(out).shape == (2, 4, 2)) @@ -373,5 +379,37 @@ class dnn_test(NewOpenCVTests): cv.dnn_unregisterLayer('CropCaffe') + # check that dnn module can work with 3D tensor as input for network + def test_input_3d(self): + model = self.find_dnn_file('dnn/onnx/models/hidden_lstm.onnx') + input_file = self.find_dnn_file('dnn/onnx/data/input_hidden_lstm.npy') + output_file = self.find_dnn_file('dnn/onnx/data/output_hidden_lstm.npy') + if model is None: + raise unittest.SkipTest("Missing DNN test files (dnn/onnx/models/hidden_lstm.onnx). " + "Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") + if input_file is None or output_file is None: + raise unittest.SkipTest("Missing DNN test files (dnn/onnx/data/{input/output}_hidden_lstm.npy). " + "Verify OPENCV_DNN_TEST_DATA_PATH configuration parameter.") + + input = np.load(input_file) + # we have to expand the shape of input tensor because Python bindings cut 3D tensors to 2D + # it should be fixed in future. see : https://github.com/opencv/opencv/issues/19091 + # please remove `expand_dims` after that + input = np.expand_dims(input, axis=3) + gold_output = np.load(output_file) + + for backend, target in self.dnnBackendsAndTargets: + printParams(backend, target) + + net = cv.dnn.readNet(model) + + net.setPreferableBackend(backend) + net.setPreferableTarget(target) + + net.setInput(input) + real_output = net.forward() + + normAssert(self, real_output, gold_output, "", getDefaultThreshold(target)) + if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/dnn/perf/perf_convolution.cpp b/modules/dnn/perf/perf_convolution.cpp index c2a3a66ab9..bb890c6a00 100644 --- a/modules/dnn/perf/perf_convolution.cpp +++ b/modules/dnn/perf/perf_convolution.cpp @@ -26,25 +26,90 @@ struct ConvParam_t { double declared_flops; }; // Details: #12142 +// Last update: 2021-09 static const ConvParam_t testConvolutionConfigs[] = { + /* GFLOPS 3.398 x 20 = 67.956 */ {{7, 7}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {3, 3}, {0, 0}, "", true, 3397788160.}, + /* GFLOPS 16.987 x 3 = 50.962 */ {{5, 5}, {{1, 1152, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 16987226112.}, + /* GFLOPS 23.122 x 2 = 46.244 */ {{5, 5}, {{1, 672, 32, 32}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 23121788928.}, + /* GFLOPS 9.987 x 3 = 29.960 */ {{3, 3}, {{1, 256, 92, 92}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9986707456.}, + /* GFLOPS 1.595 x 16 = 25.524 */ {{3, 3}, {{1, 256, 26, 26}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595230208.}, + /* GFLOPS 4.566 x 5 = 22.828 */ {{7, 7}, {{1, 172, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {3, 3}, {0, 0}, "", true, 4565684736.}, + /* GFLOPS 1.596 x 14 = 22.338 */ {{3, 3}, {{1, 128, 52, 52}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595576320.}, + /* GFLOPS 1.595 x 12 = 19.141 */ {{3, 3}, {{1, 512, 13, 13}}, 1024, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595057152.}, + /* GFLOPS 6.814 x 2 = 13.629 */ {{3, 3}, {{1, 512, 38, 38}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6814386176.}, + /* GFLOPS 6.637 x 2 = 13.274 */ {{3, 3}, {{1, 256, 75, 75}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6636960000.}, + /* GFLOPS 11.797 x 1 = 11.797 */ {{5, 5}, {{1, 240, 64, 64}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 11797463040.}, + /* GFLOPS 11.797 x 1 = 11.797 */ {{5, 5}, {{1, 480, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 11796971520.}, + /* GFLOPS 10.701 x 1 = 10.701 */ {{3, 3}, {{1, 512, 38, 38}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 10700715792.}, /* GFLOPS 10.087 x 1 = 10.087 */ {{3, 3}, {{1, 576, 38, 50}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 10086963200.}, + /* GFLOPS 9.993 x 1 = 9.993 */ {{3, 3}, {{1, 64, 368, 368}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9993207808.}, + /* GFLOPS 9.989 x 1 = 9.989 */ {{3, 3}, {{1, 128, 184, 184}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9988874240.}, + /* GFLOPS 9.986 x 1 = 9.986 */ {{3, 3}, {{1, 512, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 9985624064.}, /* GFLOPS 1.704 x 5 = 8.518 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 512, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703596544.}, /* GFLOPS 1.704 x 5 = 8.518 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1703596544.}, + /* GFLOPS 4.247 x 2 = 8.494 */ {{3, 3}, {{1, 480, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4247224320.}, + /* GFLOPS 8.025 x 1 = 8.025 */ {{3, 3}, {{1, 1024, 19, 19}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 8025101478.}, + /* GFLOPS 0.798 x 9 = 7.180 */ {{3, 3}, {{1, 128, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797788160.}, + /* GFLOPS 0.798 x 9 = 7.179 */ {{3, 3}, {{1, 256, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797615104.}, + /* GFLOPS 6.641 x 1 = 6.641 */ {{3, 3}, {{1, 64, 300, 300}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6641280000.}, /* GFLOPS 6.641 x 1 = 6.641 */ {{3, 3}, {{1, 64, 150, 200}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6641280000.}, + /* GFLOPS 6.638 x 1 = 6.638 */ {{3, 3}, {{1, 128, 150, 150}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 6638400000.}, + /* GFLOPS 6.118 x 1 = 6.118 */ {{3, 3}, {{1, 144, 128, 128}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 6117654528.}, + /* GFLOPS 6.116 x 1 = 6.116 */ {{3, 3}, {{1, 1152, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 6115590144.}, + /* GFLOPS 5.780 x 1 = 5.780 */ {{5, 5}, {{1, 672, 32, 32}}, 672, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 5780447232.}, + /* GFLOPS 1.704 x 3 = 5.111 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703596544.}, + /* GFLOPS 4.997 x 1 = 4.997 */ {{3, 3}, {{1, 64, 184, 184}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4996603904.}, + /* GFLOPS 4.994 x 1 = 4.994 */ {{3, 3}, {{1, 128, 92, 92}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4994437120.}, + /* GFLOPS 4.993 x 1 = 4.993 */ {{3, 3}, {{1, 256, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4993353728.}, + /* GFLOPS 4.993 x 1 = 4.993 */ {{3, 3}, {{1, 512, 46, 46}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 4992812032.}, /* GFLOPS 1.659 x 3 = 4.977 */ {{3, 3}, {{1, 960, 10, 10}}, 960, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1658976000.}, /* GFLOPS 2.156 x 2 = 4.312 */ {{3, 3}, {{1, 576, 19, 19}}, 576, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2156088384.}, + /* GFLOPS 4.247 x 1 = 4.247 */ {{5, 5}, {{1, 144, 128, 128}}, 144, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4247322624.}, + /* GFLOPS 0.798 x 5 = 3.988 */ {{3, 3}, {{1, 512, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 797528576.}, /* GFLOPS 0.958 x 4 = 3.833 */ {{3, 3}, {{1, 384, 19, 19}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 958307712.}, + /* GFLOPS 0.624 x 6 = 3.746 */ {{3, 3}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 624304640.}, + /* GFLOPS 3.408 x 1 = 3.408 */ {{3, 3}, {{1, 256, 38, 38}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3407562752.}, + /* GFLOPS 3.407 x 1 = 3.407 */ {{3, 3}, {{1, 512, 19, 19}}, 1024, 1, {1, 1}, {6, 6}, {6, 6}, {0, 0}, "", true, 3407193088.}, + /* GFLOPS 0.177 x 19 = 3.370 */ {{1, 1}, {{1, 512, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177382400.}, + /* GFLOPS 0.302 x 11 = 3.325 */ {{3, 3}, {{1, 64, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 302252032.}, + /* GFLOPS 3.321 x 1 = 3.321 */ {{3, 3}, {{1, 64, 150, 150}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3320640000.}, /* GFLOPS 0.830 x 4 = 3.321 */ {{3, 3}, {{1, 64, 75, 100}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 830160000.}, + /* GFLOPS 3.319 x 1 = 3.319 */ {{3, 3}, {{1, 128, 75, 75}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3319200000.}, + /* GFLOPS 1.598 x 2 = 3.195 */ {{3, 3}, {{1, 32, 416, 416}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1597652992.}, + /* GFLOPS 1.598 x 2 = 3.195 */ {{3, 3}, {{1, 32, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1597652992.}, + /* GFLOPS 1.596 x 2 = 3.193 */ {{3, 3}, {{1, 64, 208, 208}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1596268544.}, + /* GFLOPS 1.596 x 2 = 3.193 */ {{3, 3}, {{1, 64, 104, 104}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1596268544.}, + /* GFLOPS 1.596 x 2 = 3.191 */ {{3, 3}, {{1, 128, 104, 104}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595576320.}, + /* GFLOPS 1.595 x 2 = 3.190 */ {{3, 3}, {{1, 256, 52, 52}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595230208.}, + /* GFLOPS 1.595 x 2 = 3.190 */ {{3, 3}, {{1, 512, 26, 26}}, 1024, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 1595057152.}, + /* GFLOPS 0.178 x 16 = 2.841 */ {{1, 1}, {{1, 256, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177555456.}, + /* GFLOPS 2.719 x 1 = 2.719 */ {{3, 3}, {{1, 96, 256, 256}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2719481856.}, + /* GFLOPS 0.177 x 15 = 2.659 */ {{1, 1}, {{1, 1024, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177295872.}, /* GFLOPS 1.245 x 2 = 2.490 */ {{3, 3}, {{1, 96, 75, 100}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1244880000.}, + /* GFLOPS 0.798 x 3 = 2.394 */ {{3, 3}, {{1, 64, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 798134272.}, + /* GFLOPS 0.472 x 5 = 2.360 */ {{3, 3}, {{1, 256, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471961600.}, + /* GFLOPS 2.255 x 1 = 2.255 */ {{3, 3}, {{1, 128, 80, 100}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2255285760.}, + /* GFLOPS 2.153 x 1 = 2.153 */ {{3, 3}, {{1, 128, 78, 98}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2152611840.}, /* GFLOPS 2.100 x 1 = 2.100 */ {{3, 3}, {{1, 144, 75, 75}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2100330000.}, + /* GFLOPS 2.052 x 1 = 2.052 */ {{3, 3}, {{1, 128, 76, 96}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2052298240.}, /* GFLOPS 1.022 x 2 = 2.044 */ {{3, 3}, {{1, 576, 19, 19}}, 273, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1021896057.}, + /* GFLOPS 1.995 x 1 = 1.995 */ {{9, 9}, {{1, 3, 320, 400}}, 32, 1, {1, 1}, {1, 1}, {4, 4}, {0, 0}, "", true, 1994752000.}, + /* GFLOPS 1.954 x 1 = 1.954 */ {{3, 3}, {{1, 128, 74, 94}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1954344960.}, /* GFLOPS 0.958 x 2 = 1.917 */ {{3, 3}, {{1, 192, 38, 38}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 958446336.}, /* GFLOPS 1.888 x 1 = 1.888 */ {{3, 3}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1887539200.}, /* GFLOPS 1.888 x 1 = 1.888 */ {{3, 3}, {{1, 1024, 10, 10}}, 1024, 1024, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1887539200.}, + /* GFLOPS 1.859 x 1 = 1.859 */ {{3, 3}, {{1, 128, 72, 92}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1858752000.}, + /* GFLOPS 1.766 x 1 = 1.766 */ {{3, 3}, {{1, 128, 70, 90}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1765519360.}, /* GFLOPS 1.704 x 1 = 1.704 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 256, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1703781376.}, /* GFLOPS 1.704 x 1 = 1.704 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1703781376.}, + /* GFLOPS 1.675 x 1 = 1.675 */ {{3, 3}, {{1, 128, 68, 88}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1674647040.}, /* GFLOPS 1.660 x 1 = 1.660 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 128, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1659600000.}, /* GFLOPS 1.660 x 1 = 1.660 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1659600000.}, + /* GFLOPS 1.586 x 1 = 1.586 */ {{3, 3}, {{1, 128, 66, 86}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1586135040.}, + /* GFLOPS 1.500 x 1 = 1.500 */ {{3, 3}, {{1, 128, 64, 84}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1499983360.}, + /* GFLOPS 1.416 x 1 = 1.416 */ {{3, 3}, {{1, 128, 62, 82}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1416192000.}, + /* GFLOPS 0.472 x 3 = 1.416 */ {{3, 3}, {{1, 128, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 472064000.}, + /* GFLOPS 0.472 x 3 = 1.416 */ {{3, 3}, {{1, 512, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471910400.}, /* GFLOPS 0.280 x 5 = 1.402 */ {{1, 1}, {{1, 576, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 280409600.}, /* GFLOPS 0.701 x 2 = 1.401 */ {{3, 3}, {{1, 128, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 700720000.}, /* GFLOPS 0.231 x 6 = 1.388 */ {{3, 3}, {{1, 128, 56, 56}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231311360.}, @@ -53,20 +118,39 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.420 x 3 = 1.261 */ {{3, 3}, {{1, 96, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 420492800.}, /* GFLOPS 1.261 x 1 = 1.261 */ {{3, 3}, {{1, 192, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1261113600.}, /* GFLOPS 1.258 x 1 = 1.258 */ {{3, 3}, {{1, 1280, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1258038600.}, + /* GFLOPS 1.248 x 1 = 1.248 */ {{3, 3}, {{1, 256, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1248338432.}, /* GFLOPS 1.245 x 1 = 1.245 */ {{3, 3}, {{1, 64, 75, 75}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1245240000.}, + /* GFLOPS 1.210 x 1 = 1.210 */ {{3, 3}, {{1, 32, 256, 256}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1210056704.}, + /* GFLOPS 1.196 x 1 = 1.196 */ {{3, 3}, {{1, 384, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 1196336128.}, + /* GFLOPS 1.195 x 1 = 1.195 */ {{9, 9}, {{1, 32, 240, 320}}, 3, 1, {1, 1}, {1, 1}, {4, 4}, {0, 0}, "", true, 1194624000.}, + /* GFLOPS 1.182 x 1 = 1.182 */ {{3, 3}, {{1, 32, 320, 400}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 1181696000.}, + /* GFLOPS 1.181 x 1 = 1.181 */ {{3, 3}, {{1, 64, 160, 200}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 1180672000.}, /* GFLOPS 0.561 x 2 = 1.121 */ {{3, 3}, {{1, 128, 38, 50}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 560576000.}, + /* GFLOPS 1.112 x 1 = 1.112 */ {{3, 3}, {{1, 512, 10, 10}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1111570200.}, + /* GFLOPS 0.357 x 3 = 1.072 */ {{1, 1}, {{1, 64, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 357187584.}, + /* GFLOPS 1.062 x 1 = 1.062 */ {{3, 3}, {{1, 240, 64, 64}}, 240, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1061928960.}, + /* GFLOPS 0.076 x 14 = 1.058 */ {{3, 3}, {{1, 64, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 75563008.}, /* GFLOPS 1.051 x 1 = 1.051 */ {{3, 3}, {{1, 160, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1050988800.}, + /* GFLOPS 0.210 x 5 = 1.051 */ {{1, 1}, {{1, 256, 20, 20}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210124800.}, + /* GFLOPS 0.210 x 5 = 1.049 */ {{1, 1}, {{1, 1024, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209817600.}, /* GFLOPS 1.006 x 1 = 1.006 */ {{3, 3}, {{1, 1024, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1006441800.}, /* GFLOPS 0.246 x 4 = 0.985 */ {{1, 1}, {{1, 256, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 246240000.}, /* GFLOPS 0.189 x 5 = 0.947 */ {{1, 1}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189452800.}, /* GFLOPS 0.189 x 5 = 0.947 */ {{1, 1}, {{1, 512, 19, 19}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 189452800.}, + /* GFLOPS 0.472 x 2 = 0.945 */ {{3, 3}, {{1, 64, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 472268800.}, /* GFLOPS 0.934 x 1 = 0.934 */ {{3, 3}, {{1, 96, 150, 150}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 933660000.}, /* GFLOPS 0.231 x 4 = 0.925 */ {{3, 3}, {{1, 128, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231311360.}, /* GFLOPS 0.896 x 1 = 0.896 */ {{5, 5}, {{1, 96, 27, 27}}, 256, 2, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 895981824.}, + /* GFLOPS 0.089 x 10 = 0.890 */ {{1, 1}, {{1, 128, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88950784.}, + /* GFLOPS 0.089 x 10 = 0.888 */ {{1, 1}, {{1, 256, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88777728.}, /* GFLOPS 0.876 x 1 = 0.876 */ {{3, 3}, {{1, 160, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 875824000.}, /* GFLOPS 0.850 x 1 = 0.850 */ {{7, 7}, {{1, 3, 600, 800}}, 24, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 849600000.}, /* GFLOPS 0.841 x 1 = 0.841 */ {{3, 3}, {{1, 128, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 840864000.}, /* GFLOPS 0.415 x 2 = 0.831 */ {{3, 3}, {{1, 32, 150, 150}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 415440000.}, + /* GFLOPS 0.757 x 1 = 0.757 */ {{1, 1}, {{1, 1024, 19, 19}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 757441536.}, + /* GFLOPS 0.712 x 1 = 0.712 */ {{1, 1}, {{1, 128, 208, 208}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 711606272.}, + /* GFLOPS 0.178 x 4 = 0.712 */ {{1, 1}, {{1, 128, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 177901568.}, + /* GFLOPS 0.354 x 2 = 0.707 */ {{1, 1}, {{1, 256, 52, 52}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 353723760.}, /* GFLOPS 0.351 x 2 = 0.701 */ {{1, 1}, {{1, 576, 38, 50}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 350512000.}, /* GFLOPS 0.701 x 1 = 0.701 */ {{3, 3}, {{1, 128, 75, 100}}, 160, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 700720000.}, /* GFLOPS 0.694 x 1 = 0.694 */ {{3, 3}, {{1, 64, 56, 56}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 694235136.}, @@ -75,19 +159,31 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.058 x 12 = 0.694 */ {{3, 3}, {{1, 128, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 57827840.}, /* GFLOPS 0.231 x 3 = 0.694 */ {{3, 3}, {{1, 512, 7, 7}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 231236096.}, /* GFLOPS 0.160 x 4 = 0.639 */ {{3, 3}, {{1, 64, 38, 38}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 159833472.}, + /* GFLOPS 0.211 x 3 = 0.634 */ {{1, 1}, {{1, 64, 80, 80}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 211353600.}, + /* GFLOPS 0.211 x 3 = 0.632 */ {{1, 1}, {{1, 128, 40, 40}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210534400.}, + /* GFLOPS 0.210 x 3 = 0.630 */ {{1, 1}, {{1, 512, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209920000.}, + /* GFLOPS 0.210 x 3 = 0.630 */ {{1, 1}, {{1, 512, 10, 10}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209920000.}, /* GFLOPS 0.103 x 6 = 0.618 */ {{1, 1}, {{1, 256, 14, 14}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102961152.}, /* GFLOPS 0.615 x 1 = 0.615 */ {{1, 1}, {{1, 320, 75, 100}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 615360000.}, + /* GFLOPS 0.305 x 2 = 0.609 */ {{3, 3}, {{1, 3, 416, 416}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 304578560.}, /* GFLOPS 0.597 x 1 = 0.597 */ {{3, 3}, {{1, 576, 19, 19}}, 576, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 597254400.}, + /* GFLOPS 0.278 x 2 = 0.557 */ {{1, 1}, {{1, 128, 46, 46}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 278431744.}, /* GFLOPS 0.185 x 3 = 0.554 */ {{1, 1}, {{1, 192, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 184800000.}, /* GFLOPS 0.553 x 1 = 0.553 */ {{3, 3}, {{1, 64, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 553440000.}, /* GFLOPS 0.539 x 1 = 0.539 */ {{3, 3}, {{1, 144, 75, 75}}, 144, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 539178048.}, /* GFLOPS 0.103 x 5 = 0.514 */ {{1, 1}, {{1, 1024, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102810624.}, /* GFLOPS 0.491 x 1 = 0.491 */ {{1, 1}, {{1, 576, 38, 50}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 490716800.}, + /* GFLOPS 0.483 x 1 = 0.483 */ {{7, 7}, {{1, 3, 320, 320}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", false, 483328000.}, /* GFLOPS 0.240 x 2 = 0.479 */ {{3, 3}, {{1, 96, 38, 38}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 239680896.}, + /* GFLOPS 0.477 x 1 = 0.477 */ {{3, 3}, {{1, 3, 368, 368}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 476692480.}, /* GFLOPS 0.237 x 2 = 0.474 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 236830720.}, /* GFLOPS 0.472 x 1 = 0.472 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 512, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 471910400.}, /* GFLOPS 0.472 x 1 = 0.472 */ {{3, 3}, {{1, 512, 19, 19}}, 512, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 471910400.}, + /* GFLOPS 0.155 x 3 = 0.464 */ {{1, 1}, {{1, 112, 32, 32}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 154828800.}, + /* GFLOPS 0.114 x 4 = 0.454 */ {{1, 1}, {{1, 192, 16, 16}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113541120.}, /* GFLOPS 0.449 x 1 = 0.449 */ {{3, 3}, {{1, 384, 13, 13}}, 384, 2, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 448626048.}, + /* GFLOPS 0.089 x 5 = 0.443 */ {{1, 1}, {{1, 512, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 88691200.}, + /* GFLOPS 0.428 x 1 = 0.428 */ {{1, 1}, {{1, 64, 64, 64}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 427991040.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 128, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 426037760.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 75, 75}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 426037760.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 128, 38, 38}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 426037760.}, @@ -95,46 +191,81 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 256, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 425945344.}, /* GFLOPS 0.426 x 1 = 0.426 */ {{3, 3}, {{1, 256, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 425945344.}, /* GFLOPS 0.421 x 1 = 0.421 */ {{1, 1}, {{1, 576, 38, 50}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 420614400.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 256, 40, 40}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 420249600.}, + /* GFLOPS 0.210 x 2 = 0.420 */ {{1, 1}, {{1, 256, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 210124800.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 512, 20, 20}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 419840000.}, + /* GFLOPS 0.420 x 1 = 0.420 */ {{1, 1}, {{1, 1024, 10, 10}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 419635200.}, + /* GFLOPS 0.210 x 2 = 0.420 */ {{1, 1}, {{1, 2048, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209766400.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 32, 150, 150}}, 32, 32, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 415440000.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 64, 150, 150}}, 64, 64, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 415080000.}, /* GFLOPS 0.415 x 1 = 0.415 */ {{3, 3}, {{1, 64, 150, 150}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 415080000.}, /* GFLOPS 0.104 x 4 = 0.414 */ {{1, 1}, {{1, 64, 56, 56}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103563264.}, /* GFLOPS 0.103 x 4 = 0.413 */ {{1, 1}, {{1, 128, 28, 28}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103161856.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 32, 208, 208}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 399413248.}, + /* GFLOPS 0.200 x 2 = 0.399 */ {{3, 3}, {{1, 32, 104, 104}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199706624.}, + /* GFLOPS 0.200 x 2 = 0.399 */ {{3, 3}, {{1, 64, 52, 52}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199533568.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 128, 52, 52}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 398894080.}, + /* GFLOPS 0.199 x 2 = 0.399 */ {{3, 3}, {{1, 128, 26, 26}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 199447040.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 256, 26, 26}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 398807552.}, + /* GFLOPS 0.399 x 1 = 0.399 */ {{3, 3}, {{1, 256, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 398807552.}, /* GFLOPS 0.376 x 1 = 0.376 */ {{1, 1}, {{1, 24, 300, 400}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 376320000.}, + /* GFLOPS 0.179 x 2 = 0.357 */ {{1, 1}, {{1, 64, 208, 208}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 178593792.}, + /* GFLOPS 0.089 x 4 = 0.357 */ {{1, 1}, {{1, 64, 104, 104}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 89296896.}, + /* GFLOPS 0.356 x 1 = 0.356 */ {{1, 1}, {{1, 128, 104, 104}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 355803136.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 256, 52, 52}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 355110912.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 512, 26, 26}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354764800.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 1024, 13, 13}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354591744.}, + /* GFLOPS 0.355 x 1 = 0.355 */ {{1, 1}, {{1, 2048, 13, 13}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 354505216.}, + /* GFLOPS 0.177 x 2 = 0.353 */ {{1, 1}, {{1, 512, 26, 26}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 176689500.}, + /* GFLOPS 0.070 x 5 = 0.348 */ {{1, 1}, {{1, 128, 46, 46}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 69607936.}, /* GFLOPS 0.347 x 1 = 0.347 */ {{3, 3}, {{1, 128, 28, 28}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 346967040.}, /* GFLOPS 0.347 x 1 = 0.347 */ {{3, 3}, {{1, 128, 28, 28}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 346967040.}, /* GFLOPS 0.014 x 24 = 0.347 */ {{3, 3}, {{1, 128, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 14456960.}, + /* GFLOPS 0.113 x 3 = 0.340 */ {{1, 1}, {{1, 1152, 16, 16}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113295360.}, /* GFLOPS 0.053 x 6 = 0.320 */ {{1, 1}, {{1, 576, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 53277824.}, /* GFLOPS 0.319 x 1 = 0.319 */ {{3, 3}, {{1, 192, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 319482112.}, + /* GFLOPS 0.317 x 1 = 0.317 */ {{3, 3}, {{1, 3, 300, 300}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 316800000.}, /* GFLOPS 0.315 x 1 = 0.315 */ {{3, 3}, {{1, 96, 75, 100}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 315369600.}, /* GFLOPS 0.103 x 3 = 0.309 */ {{1, 1}, {{1, 512, 7, 7}}, 2048, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102860800.}, /* GFLOPS 0.103 x 3 = 0.309 */ {{1, 1}, {{1, 512, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102860800.}, + /* GFLOPS 0.154 x 2 = 0.309 */ {{1, 1}, {{1, 672, 32, 32}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 154255360.}, /* GFLOPS 0.308 x 1 = 0.308 */ {{1, 1}, {{1, 320, 75, 100}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 307680000.}, + /* GFLOPS 0.034 x 9 = 0.304 */ {{1, 1}, {{1, 64, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 33816576.}, /* GFLOPS 0.299 x 1 = 0.299 */ {{3, 3}, {{1, 256, 13, 13}}, 384, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 299105664.}, /* GFLOPS 0.299 x 1 = 0.299 */ {{3, 3}, {{1, 384, 13, 13}}, 256, 2, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 299084032.}, /* GFLOPS 0.017 x 17 = 0.290 */ {{1, 1}, {{1, 32, 32, 64}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 17039360.}, /* GFLOPS 0.017 x 16 = 0.269 */ {{1, 1}, {{1, 128, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 16842752.}, /* GFLOPS 0.133 x 2 = 0.266 */ {{3, 3}, {{1, 128, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 133136800.}, + /* GFLOPS 0.266 x 1 = 0.266 */ {{1, 1}, {{1, 384, 52, 52}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 266160128.}, + /* GFLOPS 0.266 x 1 = 0.266 */ {{1, 1}, {{1, 768, 26, 26}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 265987072.}, /* GFLOPS 0.038 x 7 = 0.265 */ {{3, 3}, {{1, 16, 64, 128}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 37879808.}, + /* GFLOPS 0.019 x 14 = 0.264 */ {{3, 3}, {{1, 64, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 18890752.}, + /* GFLOPS 0.262 x 1 = 0.262 */ {{1, 1}, {{1, 2560, 20, 20}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 262195200.}, /* GFLOPS 0.126 x 2 = 0.252 */ {{3, 3}, {{1, 512, 5, 5}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 125812050.}, /* GFLOPS 0.248 x 1 = 0.248 */ {{1, 1}, {{1, 64, 150, 200}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 247680000.}, /* GFLOPS 0.040 x 6 = 0.240 */ {{1, 1}, {{1, 576, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39958368.}, /* GFLOPS 0.080 x 3 = 0.240 */ {{3, 3}, {{1, 96, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 79893632.}, /* GFLOPS 0.240 x 1 = 0.240 */ {{3, 3}, {{1, 192, 38, 38}}, 192, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 239611584.}, /* GFLOPS 0.240 x 1 = 0.240 */ {{3, 3}, {{1, 192, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 239611584.}, + /* GFLOPS 0.079 x 3 = 0.237 */ {{1, 1}, {{1, 80, 32, 32}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 79134720.}, /* GFLOPS 0.237 x 1 = 0.237 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", false, 236830720.}, /* GFLOPS 0.237 x 1 = 0.237 */ {{7, 7}, {{1, 3, 224, 224}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 236830720.}, + /* GFLOPS 0.118 x 2 = 0.236 */ {{3, 3}, {{1, 32, 80, 80}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118169600.}, + /* GFLOPS 0.236 x 1 = 0.236 */ {{3, 3}, {{1, 256, 19, 19}}, 512, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 235980800.}, + /* GFLOPS 0.116 x 2 = 0.231 */ {{1, 1}, {{1, 24, 128, 128}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 115605504.}, /* GFLOPS 0.111 x 2 = 0.221 */ {{3, 3}, {{1, 192, 10, 10}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110624000.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 128, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 213018880.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 128, 19, 19}}, 256, 1, {1, 1}, {2, 2}, {2, 2}, {0, 0}, "", false, 213018880.}, /* GFLOPS 0.107 x 2 = 0.213 */ {{3, 3}, {{1, 128, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 106509440.}, /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 256, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 212972672.}, + /* GFLOPS 0.213 x 1 = 0.213 */ {{3, 3}, {{1, 512, 38, 38}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 212949568.}, /* GFLOPS 0.212 x 1 = 0.212 */ {{7, 7}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 212400000.}, /* GFLOPS 0.211 x 1 = 0.211 */ {{11, 11}, {{1, 3, 227, 227}}, 96, 1, {4, 4}, {1, 1}, {0, 0}, {0, 0}, "", true, 211120800.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{3, 3}, {{1, 64, 38, 50}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 210307200.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{1, 1}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 209817600.}, /* GFLOPS 0.210 x 1 = 0.210 */ {{1, 1}, {{1, 1024, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 209817600.}, /* GFLOPS 0.104 x 2 = 0.208 */ {{3, 3}, {{1, 32, 75, 75}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", false, 103860000.}, + /* GFLOPS 0.208 x 1 = 0.208 */ {{1, 1}, {{1, 16, 256, 256}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 207618048.}, /* GFLOPS 0.206 x 1 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 512, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 205922304.}, /* GFLOPS 0.206 x 1 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 205922304.}, /* GFLOPS 0.103 x 2 = 0.206 */ {{1, 1}, {{1, 256, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 102961152.}, @@ -148,27 +279,35 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.190 x 1 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189637632.}, /* GFLOPS 0.190 x 1 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 189637632.}, /* GFLOPS 0.047 x 4 = 0.190 */ {{1, 1}, {{1, 256, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 47409408.}, + /* GFLOPS 0.189 x 1 = 0.189 */ {{1, 1}, {{1, 1024, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 189360384.}, /* GFLOPS 0.038 x 5 = 0.189 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 37814272.}, + /* GFLOPS 0.189 x 1 = 0.189 */ {{1, 1}, {{1, 1152, 16, 16}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 188825600.}, /* GFLOPS 0.185 x 1 = 0.185 */ {{1, 1}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 185040000.}, /* GFLOPS 0.185 x 1 = 0.185 */ {{1, 1}, {{1, 128, 75, 75}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 185040000.}, /* GFLOPS 0.181 x 1 = 0.181 */ {{3, 3}, {{1, 160, 14, 14}}, 320, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 180696320.}, /* GFLOPS 0.181 x 1 = 0.181 */ {{3, 3}, {{1, 160, 14, 14}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 180696320.}, /* GFLOPS 0.090 x 2 = 0.181 */ {{3, 3}, {{1, 224, 10, 10}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 90339200.}, /* GFLOPS 0.180 x 1 = 0.180 */ {{1, 1}, {{1, 224, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 180232192.}, + /* GFLOPS 0.088 x 2 = 0.177 */ {{1, 1}, {{1, 1024, 13, 13}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 88301655.}, /* GFLOPS 0.174 x 1 = 0.174 */ {{3, 3}, {{1, 96, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 173508608.}, /* GFLOPS 0.174 x 1 = 0.174 */ {{3, 3}, {{1, 96, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 173508608.}, /* GFLOPS 0.166 x 1 = 0.166 */ {{3, 3}, {{1, 160, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 166406560.}, /* GFLOPS 0.080 x 2 = 0.160 */ {{1, 1}, {{1, 576, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 79916736.}, /* GFLOPS 0.160 x 1 = 0.160 */ {{3, 3}, {{1, 128, 19, 19}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 159764160.}, + /* GFLOPS 0.160 x 1 = 0.160 */ {{3, 3}, {{1, 1024, 19, 19}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 159703512.}, /* GFLOPS 0.159 x 1 = 0.159 */ {{7, 7}, {{1, 3, 300, 300}}, 24, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 159300000.}, + /* GFLOPS 0.080 x 2 = 0.159 */ {{1, 1}, {{1, 40, 64, 64}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 79626240.}, + /* GFLOPS 0.079 x 2 = 0.157 */ {{1, 1}, {{1, 480, 32, 32}}, 80, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78725120.}, /* GFLOPS 0.155 x 1 = 0.155 */ {{1, 1}, {{1, 192, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 154542080.}, /* GFLOPS 0.146 x 1 = 0.146 */ {{3, 3}, {{1, 144, 14, 14}}, 288, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 146369664.}, /* GFLOPS 0.146 x 1 = 0.146 */ {{3, 3}, {{1, 144, 14, 14}}, 288, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 146369664.}, /* GFLOPS 0.072 x 2 = 0.144 */ {{1, 1}, {{1, 1024, 10, 10}}, 352, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 72124800.}, /* GFLOPS 0.140 x 1 = 0.140 */ {{1, 1}, {{1, 576, 38, 50}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 140204800.}, + /* GFLOPS 0.139 x 1 = 0.139 */ {{3, 3}, {{1, 256, 5, 5}}, 1206, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 138961350.}, /* GFLOPS 0.017 x 8 = 0.138 */ {{1, 1}, {{1, 16, 64, 128}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 17301504.}, /* GFLOPS 0.067 x 2 = 0.133 */ {{1, 1}, {{1, 576, 19, 19}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 66597280.}, /* GFLOPS 0.133 x 1 = 0.133 */ {{3, 3}, {{1, 128, 38, 38}}, 160, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 133136800.}, + /* GFLOPS 0.044 x 3 = 0.133 */ {{1, 1}, {{1, 512, 13, 13}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 44345600.}, /* GFLOPS 0.129 x 1 = 0.129 */ {{1, 1}, {{1, 160, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 128851968.}, /* GFLOPS 0.128 x 1 = 0.128 */ {{3, 3}, {{1, 64, 24, 24}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 127512576.}, /* GFLOPS 0.120 x 1 = 0.120 */ {{5, 5}, {{1, 32, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 120497664.}, @@ -176,22 +315,35 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 3 = 0.120 */ {{1, 1}, {{1, 96, 19, 19}}, 576, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 40131648.}, /* GFLOPS 0.118 x 1 = 0.118 */ {{1, 1}, {{1, 320, 38, 38}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 118477312.}, /* GFLOPS 0.017 x 7 = 0.118 */ {{1, 1}, {{1, 64, 64, 128}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 16908288.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 64, 80, 80}}, 64, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 118067200.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 64, 40, 40}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118067200.}, /* GFLOPS 0.039 x 3 = 0.118 */ {{1, 1}, {{1, 1024, 10, 10}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39340800.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 128, 40, 40}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 118016000.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 128, 20, 20}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 118016000.}, + /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 256, 20, 20}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 117990400.}, /* GFLOPS 0.118 x 1 = 0.118 */ {{3, 3}, {{1, 256, 19, 19}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 117990400.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 16, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 58003456.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 32, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 57903104.}, /* GFLOPS 0.058 x 2 = 0.116 */ {{3, 3}, {{1, 64, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 57852928.}, /* GFLOPS 0.116 x 1 = 0.116 */ {{3, 3}, {{1, 128, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 115655680.}, /* GFLOPS 0.116 x 1 = 0.116 */ {{3, 3}, {{1, 128, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 115655680.}, + /* GFLOPS 0.115 x 1 = 0.115 */ {{3, 3}, {{1, 3, 512, 512}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 115343360.}, + /* GFLOPS 0.114 x 1 = 0.114 */ {{1, 1}, {{1, 144, 128, 128}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 113639424.}, /* GFLOPS 0.112 x 1 = 0.112 */ {{1, 1}, {{1, 1024, 10, 10}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 111875400.}, + /* GFLOPS 0.110 x 1 = 0.110 */ {{1, 1}, {{1, 480, 32, 32}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 110215168.}, + /* GFLOPS 0.107 x 1 = 0.107 */ {{1, 1}, {{1, 64, 32, 32}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 106997760.}, /* GFLOPS 0.036 x 3 = 0.107 */ {{1, 1}, {{1, 192, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 35580160.}, /* GFLOPS 0.107 x 1 = 0.107 */ {{3, 3}, {{1, 32, 75, 75}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 106648064.}, /* GFLOPS 0.107 x 1 = 0.107 */ {{3, 3}, {{1, 64, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 106555648.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 256, 40, 40}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 105062400.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 20, 20}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104960000.}, /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 104960000.}, /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 512, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104960000.}, + /* GFLOPS 0.105 x 1 = 0.105 */ {{1, 1}, {{1, 1024, 10, 10}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 104908800.}, /* GFLOPS 0.103 x 1 = 0.103 */ {{1, 1}, {{1, 128, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 103161856.}, /* GFLOPS 0.051 x 2 = 0.103 */ {{1, 1}, {{1, 256, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 51480576.}, /* GFLOPS 0.051 x 2 = 0.103 */ {{1, 1}, {{1, 256, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 51480576.}, + /* GFLOPS 0.008 x 12 = 0.101 */ {{1, 1}, {{1, 64, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 8454144.}, /* GFLOPS 0.101 x 1 = 0.101 */ {{1, 1}, {{1, 512, 19, 19}}, 273, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 101016825.}, /* GFLOPS 0.096 x 1 = 0.096 */ {{1, 1}, {{1, 480, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 96438272.}, /* GFLOPS 0.095 x 1 = 0.095 */ {{1, 1}, {{1, 128, 38, 38}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 95003648.}, @@ -208,8 +360,10 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.092 x 1 = 0.092 */ {{1, 1}, {{1, 192, 75, 100}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 92400000.}, /* GFLOPS 0.090 x 1 = 0.090 */ {{1, 1}, {{1, 448, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 90015744.}, /* GFLOPS 0.045 x 2 = 0.090 */ {{3, 3}, {{1, 576, 19, 19}}, 12, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44918508.}, + /* GFLOPS 0.044 x 2 = 0.089 */ {{1, 1}, {{1, 256, 26, 26}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 44388864.}, /* GFLOPS 0.089 x 1 = 0.089 */ {{3, 3}, {{1, 112, 14, 14}}, 224, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 88554368.}, /* GFLOPS 0.089 x 1 = 0.089 */ {{3, 3}, {{1, 112, 14, 14}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 88554368.}, + /* GFLOPS 0.088 x 1 = 0.088 */ {{1, 1}, {{1, 256, 26, 26}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 88430940.}, /* GFLOPS 0.021 x 4 = 0.084 */ {{5, 1}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {2, 0}, {0, 0}, "", false, 21037056.}, /* GFLOPS 0.021 x 4 = 0.084 */ {{1, 5}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {1, 1}, {0, 2}, {0, 0}, "", true, 21037056.}, /* GFLOPS 0.084 x 1 = 0.084 */ {{1, 1}, {{1, 416, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 83593216.}, @@ -217,9 +371,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 2 = 0.080 */ {{1, 1}, {{1, 576, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39958368.}, /* GFLOPS 0.040 x 2 = 0.079 */ {{1, 1}, {{1, 24, 75, 75}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39690000.}, /* GFLOPS 0.040 x 2 = 0.079 */ {{3, 3}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39600000.}, + /* GFLOPS 0.079 x 1 = 0.079 */ {{1, 1}, {{1, 240, 64, 64}}, 40, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78807040.}, + /* GFLOPS 0.079 x 1 = 0.079 */ {{1, 1}, {{1, 384, 40, 40}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 78745600.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{1, 1}, {{1, 96, 56, 56}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 77471744.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{3, 3}, {{1, 192, 10, 10}}, 224, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 77436800.}, /* GFLOPS 0.077 x 1 = 0.077 */ {{1, 1}, {{1, 384, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 77170688.}, + /* GFLOPS 0.076 x 1 = 0.076 */ {{3, 3}, {{1, 3, 416, 416}}, 32, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", false, 76144640.}, + /* GFLOPS 0.076 x 1 = 0.076 */ {{1, 1}, {{1, 96, 128, 128}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 75890688.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {8, 8}, {8, 8}, {0, 0}, "", true, 37814272.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {4, 4}, {4, 4}, {0, 0}, "", true, 37814272.}, /* GFLOPS 0.038 x 2 = 0.076 */ {{3, 3}, {{1, 32, 32, 64}}, 32, 1, {1, 1}, {2, 2}, {2, 2}, {0, 0}, "", true, 37814272.}, @@ -230,6 +388,9 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.071 x 1 = 0.071 */ {{1, 1}, {{1, 24, 150, 150}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 70560000.}, /* GFLOPS 0.070 x 1 = 0.070 */ {{3, 3}, {{1, 96, 14, 14}}, 208, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 70487872.}, /* GFLOPS 0.069 x 1 = 0.069 */ {{3, 3}, {{1, 96, 14, 14}}, 204, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 69132336.}, + /* GFLOPS 0.068 x 1 = 0.068 */ {{1, 1}, {{1, 32, 256, 256}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 68157440.}, + /* GFLOPS 0.005 x 14 = 0.066 */ {{3, 3}, {{1, 64, 8, 8}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 4722688.}, + /* GFLOPS 0.066 x 1 = 0.066 */ {{1, 1}, {{1, 672, 16, 16}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 66109440.}, /* GFLOPS 0.066 x 1 = 0.066 */ {{1, 1}, {{1, 1280, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 65561600.}, /* GFLOPS 0.033 x 2 = 0.065 */ {{3, 3}, {{1, 48, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 32551680.}, /* GFLOPS 0.065 x 1 = 0.065 */ {{3, 3}, {{1, 192, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 65046912.}, @@ -239,6 +400,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.032 x 2 = 0.064 */ {{3, 3}, {{1, 96, 12, 12}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 31868928.}, /* GFLOPS 0.061 x 1 = 0.061 */ {{1, 1}, {{1, 960, 10, 10}}, 320, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 61472000.}, /* GFLOPS 0.031 x 2 = 0.061 */ {{1, 1}, {{1, 960, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 30736000.}, + /* GFLOPS 0.061 x 1 = 0.061 */ {{1, 1}, {{1, 512, 46, 46}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 60729200.}, /* GFLOPS 0.060 x 1 = 0.060 */ {{3, 3}, {{1, 96, 38, 38}}, 96, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 59920224.}, /* GFLOPS 0.059 x 1 = 0.059 */ {{1, 1}, {{1, 320, 38, 38}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 59238656.}, /* GFLOPS 0.059 x 1 = 0.059 */ {{3, 3}, {{1, 128, 19, 19}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 59008000.}, @@ -253,6 +415,11 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.053 x 1 = 0.053 */ {{3, 3}, {{1, 128, 38, 38}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 53254720.}, /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 528, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 53036032.}, /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 528, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 53036032.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 64, 80, 80}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52838400.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 64, 40, 40}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52838400.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 128, 80, 80}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52633600.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 128, 20, 20}}, 512, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52633600.}, + /* GFLOPS 0.053 x 1 = 0.053 */ {{1, 1}, {{1, 256, 10, 10}}, 1024, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52531200.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 52454400.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 52454400.}, /* GFLOPS 0.052 x 1 = 0.052 */ {{1, 1}, {{1, 1024, 10, 10}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 52454400.}, @@ -268,6 +435,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.050 x 1 = 0.050 */ {{1, 1}, {{1, 992, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 49799680.}, /* GFLOPS 0.048 x 1 = 0.048 */ {{1, 1}, {{1, 960, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 48194048.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 256, 19, 19}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 47409408.}, + /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 144, 64, 64}}, 40, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 47349760.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 512, 38, 50}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 46740000.}, /* GFLOPS 0.047 x 1 = 0.047 */ {{1, 1}, {{1, 928, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 46588416.}, /* GFLOPS 0.046 x 1 = 0.046 */ {{1, 1}, {{1, 64, 75, 75}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 46440000.}, @@ -280,6 +448,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.045 x 1 = 0.045 */ {{3, 3}, {{1, 3, 227, 227}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", true, 44946880.}, /* GFLOPS 0.044 x 1 = 0.044 */ {{3, 3}, {{1, 128, 19, 19}}, 192, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44256000.}, /* GFLOPS 0.044 x 1 = 0.044 */ {{3, 3}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 44239200.}, + /* GFLOPS 0.044 x 1 = 0.044 */ {{1, 1}, {{1, 512, 13, 13}}, 255, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 44172375.}, /* GFLOPS 0.043 x 1 = 0.043 */ {{7, 7}, {{1, 3, 96, 96}}, 64, 1, {2, 2}, {1, 1}, {3, 3}, {0, 0}, "", true, 43499520.}, /* GFLOPS 0.043 x 1 = 0.043 */ {{1, 1}, {{1, 864, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 43377152.}, /* GFLOPS 0.042 x 1 = 0.042 */ {{1, 1}, {{1, 832, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 41771520.}, @@ -289,6 +458,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 64, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 39958368.}, /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 256, 19, 19}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 39932376.}, /* GFLOPS 0.040 x 1 = 0.040 */ {{3, 3}, {{1, 3, 300, 300}}, 32, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 39600000.}, + /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 240, 32, 32}}, 80, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39403520.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 144, 75, 75}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 39015000.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 192, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 38635520.}, /* GFLOPS 0.039 x 1 = 0.039 */ {{1, 1}, {{1, 768, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 38560256.}, @@ -297,9 +467,11 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.036 x 1 = 0.036 */ {{1, 1}, {{1, 480, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 36164352.}, /* GFLOPS 0.018 x 2 = 0.036 */ {{1, 1}, {{1, 192, 38, 38}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 17790080.}, /* GFLOPS 0.035 x 1 = 0.035 */ {{1, 1}, {{1, 704, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 35348992.}, + /* GFLOPS 0.035 x 1 = 0.035 */ {{1, 1}, {{1, 512, 46, 46}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 34702400.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{1, 1}, {{1, 672, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 33743360.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{1, 1}, {{1, 128, 32, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 33685504.}, /* GFLOPS 0.034 x 1 = 0.034 */ {{2, 2}, {{1, 64, 64, 128}}, 32, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 33619968.}, + /* GFLOPS 0.033 x 1 = 0.033 */ {{3, 3}, {{1, 256, 3, 3}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 33350724.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 528, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 33147520.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 528, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 33147520.}, /* GFLOPS 0.033 x 1 = 0.033 */ {{1, 1}, {{1, 1024, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 32784000.}, @@ -307,24 +479,29 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 512, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 32144000.}, /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 640, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 32137728.}, /* GFLOPS 0.032 x 1 = 0.032 */ {{1, 1}, {{1, 508, 14, 14}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 31893120.}, + /* GFLOPS 0.011 x 3 = 0.032 */ {{1, 1}, {{1, 320, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 10502144.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 832, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 31328640.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 832, 7, 7}}, 384, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 31328640.}, /* GFLOPS 0.031 x 1 = 0.031 */ {{1, 1}, {{1, 608, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 30532096.}, + /* GFLOPS 0.015 x 2 = 0.030 */ {{1, 1}, {{1, 128, 46, 46}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 15226736.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 24, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 15065344.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 24, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15065344.}, /* GFLOPS 0.015 x 2 = 0.030 */ {{5, 5}, {{1, 48, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15059072.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{3, 3}, {{1, 256, 10, 10}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 29497600.}, + /* GFLOPS 0.015 x 2 = 0.029 */ {{1, 1}, {{1, 112, 32, 32}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 14745600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 192, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 28976640.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 192, 28, 28}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 28976640.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 512, 14, 14}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 28929600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 512, 14, 14}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 28929600.}, /* GFLOPS 0.029 x 1 = 0.029 */ {{1, 1}, {{1, 576, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 28926464.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 544, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 27320832.}, + /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 64, 16, 16}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 26749440.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 384, 19, 19}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 26650464.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 576, 19, 19}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 26638912.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{3, 3}, {{1, 128, 38, 38}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 26627360.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 528, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 26518016.}, /* GFLOPS 0.027 x 1 = 0.027 */ {{1, 1}, {{1, 528, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 26518016.}, + /* GFLOPS 0.009 x 3 = 0.026 */ {{1, 1}, {{1, 128, 46, 46}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 8700992.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 96, 75, 75}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 26055000.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 64, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 25890816.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 64, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 25890816.}, @@ -336,6 +513,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.013 x 2 = 0.026 */ {{1, 1}, {{1, 256, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 12870144.}, /* GFLOPS 0.026 x 1 = 0.026 */ {{1, 1}, {{1, 512, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 25715200.}, /* GFLOPS 0.013 x 2 = 0.026 */ {{1, 1}, {{1, 512, 14, 14}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 12857600.}, + /* GFLOPS 0.002 x 12 = 0.025 */ {{1, 1}, {{1, 64, 16, 16}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 2113536.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 480, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 24109568.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 128, 38, 38}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "", false, 23750912.}, /* GFLOPS 0.024 x 1 = 0.024 */ {{1, 1}, {{1, 256, 19, 19}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 23704704.}, @@ -345,7 +523,9 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.023 x 1 = 0.023 */ {{1, 1}, {{1, 448, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 22503936.}, /* GFLOPS 0.023 x 1 = 0.023 */ {{1, 1}, {{1, 512, 14, 14}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 22500800.}, /* GFLOPS 0.022 x 1 = 0.022 */ {{1, 1}, {{1, 508, 14, 14}}, 112, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 22325184.}, + /* GFLOPS 0.022 x 1 = 0.022 */ {{3, 3}, {{1, 512, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 22120800.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{3, 3}, {{1, 128, 12, 12}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 21242880.}, + /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 40, 64, 64}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 21233664.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 416, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 20898304.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 832, 7, 7}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 20885760.}, /* GFLOPS 0.021 x 1 = 0.021 */ {{1, 1}, {{1, 832, 7, 7}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 20885760.}, @@ -360,6 +540,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 192, 28, 28}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 19317760.}, /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 192, 28, 28}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19317760.}, /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 384, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 19292672.}, + /* GFLOPS 0.019 x 1 = 0.019 */ {{1, 1}, {{1, 64, 64, 64}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 19021824.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 576, 10, 10}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 18448000.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 480, 14, 14}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 18082176.}, /* GFLOPS 0.018 x 1 = 0.018 */ {{1, 1}, {{1, 480, 14, 14}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 18082176.}, @@ -371,13 +552,16 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.016 x 1 = 0.016 */ {{1, 1}, {{1, 832, 7, 7}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 15664320.}, /* GFLOPS 0.015 x 1 = 0.015 */ {{5, 5}, {{1, 48, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 15059072.}, /* GFLOPS 0.015 x 1 = 0.015 */ {{5, 5}, {{1, 32, 12, 12}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 14754816.}, + /* GFLOPS 0.015 x 1 = 0.015 */ {{3, 3}, {{1, 128, 10, 10}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 14752000.}, /* GFLOPS 0.014 x 1 = 0.014 */ {{1, 1}, {{1, 288, 14, 14}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 14475776.}, /* GFLOPS 0.014 x 1 = 0.014 */ {{1, 1}, {{1, 512, 5, 5}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 13991250.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 144, 38, 38}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 13354112.}, /* GFLOPS 0.007 x 2 = 0.013 */ {{1, 1}, {{1, 16, 56, 56}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6623232.}, + /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 512, 10, 10}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 13120000.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 832, 7, 7}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 13053600.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 832, 7, 7}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 13053600.}, /* GFLOPS 0.007 x 2 = 0.013 */ {{1, 1}, {{1, 32, 28, 28}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6522880.}, + /* GFLOPS 0.001 x 11 = 0.013 */ {{3, 3}, {{1, 64, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1180672.}, /* GFLOPS 0.006 x 2 = 0.013 */ {{1, 1}, {{1, 64, 14, 14}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6472704.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 128, 56, 56}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 12895232.}, /* GFLOPS 0.013 x 1 = 0.013 */ {{1, 1}, {{1, 256, 28, 28}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 12870144.}, @@ -394,6 +578,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.012 x 1 = 0.012 */ {{1, 1}, {{1, 640, 6, 6}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 11805696.}, /* GFLOPS 0.012 x 1 = 0.012 */ {{1, 1}, {{1, 928, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11647104.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 896, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11245696.}, + /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 256, 13, 13}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 11097216.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{3, 3}, {{1, 256, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 11061600.}, /* GFLOPS 0.006 x 2 = 0.011 */ {{3, 3}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5530200.}, /* GFLOPS 0.011 x 1 = 0.011 */ {{1, 1}, {{1, 864, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 10844288.}, @@ -417,13 +602,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.008 x 1 = 0.008 */ {{1, 1}, {{1, 608, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 7633024.}, /* GFLOPS 0.008 x 1 = 0.008 */ {{5, 5}, {{1, 16, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 7535808.}, /* GFLOPS 0.008 x 1 = 0.008 */ {{5, 5}, {{1, 16, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 7535808.}, - /* GFLOPS 0.004 x 2 = 0.007 */ {{3, 3}, {{1, 64, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3689600.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 640, 6, 6}}, 160, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7378560.}, /* GFLOPS 0.004 x 2 = 0.007 */ {{1, 1}, {{1, 48, 14, 14}}, 192, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 3650304.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 384, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7234752.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 576, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 7231616.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 256, 12, 12}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 7091712.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 544, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 6830208.}, + /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 64, 8, 8}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 6687360.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{3, 3}, {{1, 160, 6, 6}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 6637824.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 528, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6629504.}, /* GFLOPS 0.007 x 1 = 0.007 */ {{1, 1}, {{1, 528, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6629504.}, @@ -434,11 +619,13 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 7, 7}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 6428800.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6428800.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{1, 1}, {{1, 512, 14, 14}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6428800.}, + /* GFLOPS 0.001 x 12 = 0.006 */ {{1, 1}, {{1, 64, 8, 8}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 528384.}, /* GFLOPS 0.006 x 1 = 0.006 */ {{3, 3}, {{1, 256, 10, 10}}, 12, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 5530800.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 192, 12, 12}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 5322240.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 5310720.}, + /* GFLOPS 0.005 x 1 = 0.005 */ {{3, 3}, {{1, 128, 5, 5}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 5310720.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4917600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 10, 10}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4917600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 192, 28, 28}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4829440.}, @@ -446,6 +633,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 256, 14, 14}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4826304.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 512, 14, 14}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4821600.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 508, 14, 14}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4783968.}, + /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 64, 32, 32}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 4755456.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 64, 24, 24}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4755456.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 256, 12, 12}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4727808.}, /* GFLOPS 0.005 x 1 = 0.005 */ {{1, 1}, {{1, 1024, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 4720896.}, @@ -455,6 +643,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 16, 128, 256}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 4325376.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 64, 64, 128}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", false, 4227072.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{1, 1}, {{1, 832, 7, 7}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 3916080.}, + /* GFLOPS 0.004 x 1 = 0.004 */ {{3, 3}, {{1, 256, 1, 1}}, 804, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 3705636.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{5, 5}, {{1, 16, 12, 12}}, 32, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 3691008.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{3, 3}, {{1, 64, 10, 10}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 3689600.}, /* GFLOPS 0.004 x 1 = 0.004 */ {{5, 5}, {{1, 32, 6, 6}}, 64, 1, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 3688704.}, @@ -470,6 +659,7 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 480, 14, 14}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 3013696.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 320, 12, 12}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2953728.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 640, 6, 6}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2951424.}, + /* GFLOPS 0.003 x 1 = 0.003 */ {{3, 3}, {{1, 256, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 2765400.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{3, 3}, {{1, 128, 5, 5}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 2655360.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 832, 7, 7}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2610720.}, /* GFLOPS 0.003 x 1 = 0.003 */ {{1, 1}, {{1, 256, 3, 3}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 2520882.}, @@ -482,32 +672,46 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 508, 4, 4}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 2082816.}, /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 1024, 1, 1}}, 1000, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 2049000.}, /* GFLOPS 0.001 x 2 = 0.002 */ {{3, 3}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 995544.}, - /* GFLOPS 0.001 x 2 = 0.002 */ {{3, 3}, {{1, 128, 5, 5}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 922000.}, /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 1024, 3, 3}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1770336.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 64, 4, 4}}, 810, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 1671840.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 32, 80, 80}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1664000.}, + /* GFLOPS 0.002 x 1 = 0.002 */ {{1, 1}, {{1, 256, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1641600.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 640, 6, 6}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1475712.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 1383000.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 64, 5, 5}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1328256.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 736, 3, 3}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 1272672.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 64, 16, 16}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 1188864.}, + /* GFLOPS 0.000 x 9 = 0.001 */ {{1, 1}, {{1, 64, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 132096.}, + /* GFLOPS 0.001 x 2 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590976.}, /* GFLOPS 0.001 x 2 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 590976.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 3, 3}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 1180160.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 2, 2}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1120392.}, - /* GFLOPS 0.000 x 2 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 461000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 192, 12, 12}}, 16, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 887040.}, /* GFLOPS 0.000 x 2 = 0.001 */ {{3, 3}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 442464.}, - /* GFLOPS 0.000 x 2 = 0.001 */ {{1, 1}, {{1, 128, 5, 5}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 411200.}, + /* GFLOPS 0.000 x 2 = 0.001 */ {{1, 1}, {{1, 32, 80, 80}}, 1, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 416000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 5, 5}}, 12, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 691500.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 256, 3, 3}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 663696.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 640, 2, 2}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 655872.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 615000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 512, 5, 5}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 615000.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 128, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 592128.}, - /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590976.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 590976.}, + /* GFLOPS 0.001 x 1 = 0.001 */ {{3, 3}, {{1, 128, 3, 3}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 590080.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 3, 3}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 581742.}, /* GFLOPS 0.001 x 1 = 0.001 */ {{1, 1}, {{1, 256, 4, 4}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 525312.}, + /* GFLOPS 0.000 x 4 = 0.000 */ {{1, 1}, {{1, 48, 1, 1}}, 1152, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 111744.}, + /* GFLOPS 0.000 x 4 = 0.000 */ {{1, 1}, {{1, 1152, 1, 1}}, 48, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110640.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 5, 5}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 411200.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 3, 3}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 331920.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 192, 5, 5}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 308000.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 8, 8}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 297216.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 2, 2}}, 256, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 263168.}, /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 131328.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 258552.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 1024, 1, 1}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 196704.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 3, 3}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 165960.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 3, 3}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 148032.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 3, 3}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {1, 1}, {0, 0}, "", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 147584.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 128, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 147584.}, @@ -515,16 +719,32 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 546, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 140322.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 131328.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 64, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 131328.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 28, 1, 1}}, 672, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 38304.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 672, 1, 1}}, 28, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 37660.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 110808.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 3, 3}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 110808.}, /* GFLOPS 0.000 x 2 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 55320.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 4, 4}}, 36, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "VALID", true, 74304.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 64, 2, 2}}, 64, 1, {2, 2}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 73792.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 256, 1, 1}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 73744.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 20, 1, 1}}, 480, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19680.}, + /* GFLOPS 0.000 x 3 = 0.000 */ {{1, 1}, {{1, 480, 1, 1}}, 20, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 19220.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 49248.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 256, 2, 2}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 49248.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 16, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 36880.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 126, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 32382.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{3, 3}, {{1, 128, 1, 1}}, 8, 1, {1, 1}, {1, 1}, {1, 1}, {0, 0}, "", true, 18440.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 64, 1, 1}}, 128, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", false, 16512.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 10, 1, 1}}, 240, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 5040.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 240, 1, 1}}, 10, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 4810.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "", true, 6168.}, - /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6168.} + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 128, 1, 1}}, 24, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 6168.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 6, 1, 1}}, 144, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1872.}, + /* GFLOPS 0.000 x 2 = 0.000 */ {{1, 1}, {{1, 144, 1, 1}}, 6, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 1734.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 4, 1, 1}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 864.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 96, 1, 1}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 772.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 8, 1, 1}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 544.}, + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 32, 1, 1}}, 8, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 520.} }; struct ConvParamID { diff --git a/modules/dnn/src/caffe/caffe_importer.cpp b/modules/dnn/src/caffe/caffe_importer.cpp index 05f94e6dc9..9cf989f2fc 100644 --- a/modules/dnn/src/caffe/caffe_importer.cpp +++ b/modules/dnn/src/caffe/caffe_importer.cpp @@ -306,16 +306,13 @@ public: caffe::LayerParameter* binLayer = netBinary.mutable_layer(li); const int numBlobs = binLayer->blobs_size(); + std::vector blobs(numBlobs); + binLayer->mutable_blobs()->ExtractSubrange(0, numBlobs, blobs.data()); layerParams.blobs.resize(numBlobs); for (int bi = 0; bi < numBlobs; bi++) { - blobFromProto(binLayer->blobs(bi), layerParams.blobs[bi]); - } - binLayer->clear_blobs(); - CV_Assert(numBlobs == binLayer->blobs().ClearedCount()); - for (int bi = 0; bi < numBlobs; bi++) - { - delete binLayer->mutable_blobs()->ReleaseCleared(); + blobFromProto(*blobs[bi], layerParams.blobs[bi]); + delete blobs[bi]; } } diff --git a/modules/dnn/src/caffe/caffe_io.cpp b/modules/dnn/src/caffe/caffe_io.cpp index 2fc4d84f46..ebecf95eea 100644 --- a/modules/dnn/src/caffe/caffe_io.cpp +++ b/modules/dnn/src/caffe/caffe_io.cpp @@ -92,6 +92,7 @@ #ifdef HAVE_PROTOBUF #include #include +#include #include #include @@ -1111,7 +1112,11 @@ static const int kProtoReadBytesLimit = INT_MAX; // Max size of 2 GB minus 1 by bool ReadProtoFromBinary(ZeroCopyInputStream* input, Message *proto) { CodedInputStream coded_input(input); +#if GOOGLE_PROTOBUF_VERSION >= 3006000 + coded_input.SetTotalBytesLimit(kProtoReadBytesLimit); +#else coded_input.SetTotalBytesLimit(kProtoReadBytesLimit, 536870912); +#endif return proto->ParseFromCodedStream(&coded_input); } diff --git a/modules/dnn/src/cuda/eltwise_ops.cu b/modules/dnn/src/cuda/eltwise_ops.cu index b24801531f..db4daa4718 100644 --- a/modules/dnn/src/cuda/eltwise_ops.cu +++ b/modules/dnn/src/cuda/eltwise_ops.cu @@ -5,13 +5,16 @@ #include #include +#include "array.hpp" #include "functors.hpp" #include "grid_stride_range.hpp" #include "execution.hpp" #include "vector_traits.hpp" +#include "kernel_dispatcher.hpp" #include "../cuda4dnn/csl/stream.hpp" #include "../cuda4dnn/csl/span.hpp" +#include "../cuda4dnn/csl/tensor.hpp" #include @@ -40,6 +43,32 @@ namespace raw { v_store(output_vPtr[i], vec_x); } } + + template + __global__ void eltwise_op_bcast( + Span output, array out_strides, + View x, array x_strides, array x_bcast, + View y, array y_strides, array y_bcast, + const typename EltwiseOp::Params params) { + EltwiseOp eltwise_op(params); + + for (auto i : grid_stride_range(output.size())) { + index_type out_index = i / out_strides[0]; + index_type x_index = x_bcast[0] ? 0 : out_index * x_strides[0]; + index_type y_index = y_bcast[0] ? 0 : out_index * y_strides[0]; + + for (int j = 1; j < Rank; j++) + { + out_index = (i % out_strides[j - 1]) / out_strides[j]; + if (!x_bcast[j]) + x_index += out_index * x_strides[j]; + if (!y_bcast[j]) + y_index += out_index * y_strides[j]; + } + + output[i] = eltwise_op(x[x_index], y[y_index]); + } + } } template static @@ -55,56 +84,251 @@ void launch_vectorized_eltwise_op(const Stream& stream, Span output, View launch_kernel(kernel, policy, output, x, y, params); } -template static -void eltwise_op(const Stream& stream, Span output, View x, View y, const typename EltwiseOp::Params& params = {}) { - CV_Assert(x.size() == y.size()); - CV_Assert(x.size() == output.size()); +template static +void launch_eltwise_op_bcast( + const Stream& stream, + Span output, const std::vector& outStride, + View x, const std::vector& inStride1, const std::vector& inBcast1, + View y, const std::vector& inStride2, const std::vector& inBcast2, + const typename EltwiseOp::Params& params) +{ + CV_Assert(outStride.size() == Rank); + CV_Assert(inStride1.size() == Rank); + CV_Assert(inStride2.size() == Rank); + CV_Assert(inBcast1.size() == Rank); + CV_Assert(inBcast2.size() == Rank); - if (is_fully_aligned(output, 4) && is_fully_aligned(x, 4) && is_fully_aligned(y, 4)) { - launch_vectorized_eltwise_op(stream, output, x, y, params); - } else if (is_fully_aligned(output, 2) && is_fully_aligned(x, 2) && is_fully_aligned(y, 2)) { - launch_vectorized_eltwise_op(stream, output, x, y, params); - } else { - launch_vectorized_eltwise_op(stream, output, x, y, params); + array outStride_k, inStride1_k, inStride2_k; + outStride_k.assign(std::begin(outStride), std::end(outStride)); + inStride1_k.assign(std::begin(inStride1), std::end(inStride1)); + inStride2_k.assign(std::begin(inStride2), std::end(inStride2)); + + array inBcast1_k, inBcast2_k; + inBcast1_k.assign(std::begin(inBcast1), std::end(inBcast1)); + inBcast2_k.assign(std::begin(inBcast2), std::end(inBcast2)); + + auto kernel = raw::eltwise_op_bcast; + auto policy = make_policy(kernel, output.size(), 0, stream); + launch_kernel(kernel, policy, output, outStride_k, x, inStride1_k, inBcast1_k, y, inStride2_k, inBcast2_k, params); +} + +GENERATE_KERNEL_DISPATCHER_2TP(eltwise_op_bcast_dispatcher, launch_eltwise_op_bcast); + +template static +void eltwise_op(const Stream& stream, TensorSpan output, TensorView x, TensorView y, const typename EltwiseOp::Params& params = {}) { + if (is_shape_same(output, x) && is_shape_same(output, y)) + { + /* no broadcasting; use fast path */ + CV_Assert(x.size() == y.size()); + CV_Assert(x.size() == output.size()); + + if (is_fully_aligned(output, 4) && is_fully_aligned(x, 4) && is_fully_aligned(y, 4)) { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } else if (is_fully_aligned(output, 2) && is_fully_aligned(x, 2) && is_fully_aligned(y, 2)) { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } else { + launch_vectorized_eltwise_op(stream, output, x, y, params); + } + } + else + { + CV_Assert(is_shape_compatible(output, x)); + CV_Assert(is_shape_compatible(output, y)); + + /* matching singleton axes in both input tensors can be eliminated + * + * Reasoning: + * ---------- + * Singleton axes do not contribute towards address calculation. They are redundant + * unless there is broadcasting. If both input tensors have singleton axis at a + * specified position, there is no broadcasting on that axis. + * + * Example: + * --------- + * x: [1, 256, 32, 32] -> [256, 32, 32] + * y: [1, 256, 1, 1] -> [256, 1, 1] + */ + for (int r = 0; r < output.rank(); r++) + { + while (x.get_axis_size(r) == 1 && y.get_axis_size(r) == 1) { + CV_Assert(output.get_axis_size(r) == 1); + + x.squeeze(r); + y.squeeze(r); + output.squeeze(r); + } + } + + auto inShape1 = x.shape_as_vector(); + auto inShape2 = y.shape_as_vector(); + auto outShape = output.shape_as_vector(); + + /* contiguous axes that do not broadcast can be merged into one axis + * + * Example: + * --------- + * x: [32, 8, 8] -> [32, 64] + * y: [1, 8, 8] -> [1, 64] + */ + for (int i = 0; i < inShape1.size(); i++) { + /* check if axis `i` requires any broadcasting */ + if (inShape1[i] == inShape2[i]) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] == inShape2[j]) { + CV_Assert(outShape[j] == inShape1[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + auto new_size = inShape1[i] * inShape1[j]; + inShape1[i] = new_size; + inShape2[i] = new_size; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == outShape[i]); + CV_Assert(inShape2[i] == outShape[i]); + } + } + } + + /* contiguous broadcasting axes on the same tensor can be merged into one axis + * + * Example: + * --------- + * x: [256, 8, 8] -> [256, 64] + * y: [256, 1, 1] -> [256, 1] + */ + for (int i = 0; i < inShape1.size(); i++) { + /* check if axis `i` requires any broadcasting in tensor 1 */ + if (inShape1[i] == 1 && inShape2[i] != 1) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] == 1 && inShape2[j] != 1) { + CV_Assert(outShape[j] == inShape2[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + inShape1[i] = 1; + inShape2[i] = inShape2[i] * inShape2[j]; + outShape[i] = inShape2[i]; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == 1); + CV_Assert(inShape2[i] == outShape[i]); + } + } + + /* check if axis `i` requires any broadcasting in tensor 2 */ + if (inShape1[i] != 1 && inShape2[i] == 1) { + /* loop invariant: `i` is the first axis in the contiguous axis sequence */ + + int j = i + 1; /* `j` is the axis which we will attempt to merge */ + while (j < inShape1.size() && inShape1[j] != 1 && inShape2[j] == 1) { + CV_Assert(outShape[j] == inShape1[j]); + + /* `j` axis is also used fully; merge `i` and `j` */ + inShape1[i] = inShape1[i] * inShape1[j]; + inShape2[i] = 1; + outShape[i] = inShape1[i]; + + /* delete axis `j` */ + inShape1.erase(std::begin(inShape1) + j); + inShape2.erase(std::begin(inShape2) + j); + outShape.erase(std::begin(outShape) + j); + + /* optimizations should not break the invariants */ + CV_Assert(inShape1.size() == outShape.size()); + CV_Assert(inShape2.size() == outShape.size()); + CV_Assert(inShape1[i] == outShape[i]); + CV_Assert(inShape2[i] == 1); + } + } + } + + auto rank = outShape.size(); + + std::vector inStride1(rank), inStride2(rank), outStride(rank); + inStride1.back() = 1; + inStride2.back() = 1; + outStride.back() = 1; + /* garbage, ..., garbage, 1 */ + + std::copy(std::begin(inShape1) + 1, std::end(inShape1), std::begin(inStride1)); + std::copy(std::begin(inShape2) + 1, std::end(inShape2), std::begin(inStride2)); + std::copy(std::begin(outShape) + 1, std::end(outShape), std::begin(outStride)); + /* dim[0], dim[1], ..., dim[-1], 1 */ + + std::partial_sum(inStride1.rbegin(), inStride1.rend(), inStride1.rbegin(), std::multiplies()); + std::partial_sum(inStride2.rbegin(), inStride2.rend(), inStride2.rbegin(), std::multiplies()); + std::partial_sum(outStride.rbegin(), outStride.rend(), outStride.rbegin(), std::multiplies()); + /* stride[0], stride[1], ..., stride[-2], 1 */ + + std::vector inBcast1(rank), inBcast2(rank); + std::transform(std::begin(inShape1), std::end(inShape1), std::begin(inBcast1), [](std::size_t sz) { return sz == 1; }); + std::transform(std::begin(inShape2), std::end(inShape2), std::begin(inBcast2), [](std::size_t sz) { return sz == 1; }); + + CV_Assert(1 <= rank && rank <= CSL_MAX_TENSOR_RANK); + eltwise_op_bcast_dispatcher(rank, stream, output, outStride, x, inStride1, inBcast1, y, inStride2, inBcast2, params); } } template -void eltwise_max_2(const Stream& stream, Span output, View x, View y) { +void eltwise_max_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_sum_2(const Stream& stream, Span output, View x, View y) { +void eltwise_min_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { + eltwise_op>(stream, output, x, y); +} + +template +void eltwise_sum_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_sum_coeff_2(const Stream& stream, Span output, T coeff_x, View x, T coeff_y, View y) { +void eltwise_sum_coeff_2(const Stream& stream, TensorSpan output, T coeff_x, TensorView x, T coeff_y, TensorView y) { eltwise_op>(stream, output, x, y, {coeff_x, coeff_y}); } template -void eltwise_prod_2(const Stream& stream, Span output, View x, View y) { +void eltwise_prod_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } template -void eltwise_div_2(const Stream& stream, Span output, View x, View y) { +void eltwise_div_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y) { eltwise_op>(stream, output, x, y); } #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) - template void eltwise_div_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_prod_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_sum_coeff_2(const Stream&, Span<__half>, __half, View<__half>, __half, View<__half>); - template void eltwise_sum_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); - template void eltwise_max_2(const Stream& stream, Span<__half> output, View<__half> x, View<__half> y); + template void eltwise_div_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_prod_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_sum_coeff_2(const Stream&, TensorSpan<__half>, __half, TensorView<__half>, __half, TensorView<__half>); + template void eltwise_sum_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_max_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); + template void eltwise_min_2(const Stream& stream, TensorSpan<__half> output, TensorView<__half> x, TensorView<__half> y); #endif - template void eltwise_div_2(const Stream& stream, Span output, View x, View y); - template void eltwise_prod_2(const Stream& stream, Span output, View x, View y); - template void eltwise_sum_coeff_2(const Stream&, Span, float, View, float, View); - template void eltwise_sum_2(const Stream& stream, Span output, View x, View y); - template void eltwise_max_2(const Stream& stream, Span output, View x, View y); + template void eltwise_div_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_prod_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_sum_coeff_2(const Stream&, TensorSpan, float, TensorView, float, TensorView); + template void eltwise_sum_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_max_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); + template void eltwise_min_2(const Stream& stream, TensorSpan output, TensorView x, TensorView y); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda/functors.hpp b/modules/dnn/src/cuda/functors.hpp index 1c29de0426..f01a07c77e 100644 --- a/modules/dnn/src/cuda/functors.hpp +++ b/modules/dnn/src/cuda/functors.hpp @@ -262,6 +262,21 @@ struct MaxFunctor { } }; +template +struct MinFunctor { + struct Params { + CUDA4DNN_HOST_DEVICE Params() { } + }; + + CUDA4DNN_DEVICE MinFunctor() { } + CUDA4DNN_DEVICE MinFunctor(const Params& params) { } + + CUDA4DNN_DEVICE T operator()(T x, T y) { + using csl::device::min; + return min(x, y); + } +}; + template struct SumFunctor { struct Params { diff --git a/modules/dnn/src/cuda/kernel_dispatcher.hpp b/modules/dnn/src/cuda/kernel_dispatcher.hpp index 6eff8344d3..b0fc658850 100644 --- a/modules/dnn/src/cuda/kernel_dispatcher.hpp +++ b/modules/dnn/src/cuda/kernel_dispatcher.hpp @@ -73,4 +73,22 @@ name(selector, std::forward(args)...); \ } +// Same as GENERATE_KERNEL_DISPATCHER but takes two class template parameters T and TP1 instead of just T +#define GENERATE_KERNEL_DISPATCHER_2TP(name,func); \ + template static \ + typename std::enable_if \ + ::type name(int selector, Args&& ...args) { \ + if(selector == start) \ + func(std::forward(args)...); \ + } \ + \ + template static \ + typename std::enable_if \ + ::type name(int selector, Args&& ...args) { \ + if(selector == start) \ + func(std::forward(args)...); \ + else \ + name(selector, std::forward(args)...); \ + } + #endif /* OPENCV_DNN_SRC_CUDA_KERNEL_DISPATCHER_HPP */ diff --git a/modules/dnn/src/cuda/region.cu b/modules/dnn/src/cuda/region.cu index 06b44abe9c..3700a93d99 100644 --- a/modules/dnn/src/cuda/region.cu +++ b/modules/dnn/src/cuda/region.cu @@ -31,7 +31,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { size_type boxes_per_cell, size_type box_size, size_type rows, size_type cols, T scale_x_y, size_type height_norm, size_type width_norm, - T object_prob_cutoff) + T object_prob_cutoff, bool new_coords) { using vector2_type = get_vector_type_t; auto bias_vPtr = vector2_type::get_pointer(bias.data()); @@ -47,22 +47,43 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { const auto y = (box_index % batch_inner_size) / row_inner_size; const auto x = (box_index % row_inner_size) / col_inner_size; - using device::fast_sigmoid; - const auto tmp_x = (fast_sigmoid(input[box_offset + 0]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); - const auto tmp_y = (fast_sigmoid(input[box_offset + 1]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); - output[box_offset + 0] = (T(x) + tmp_x) / T(cols); - output[box_offset + 1] = (T(y) + tmp_y) / T(rows); + /* When new_coords is true, we shouldn't use logistic activation again */ + T objectness_prob; + if (new_coords) + { + const auto tmp_x = (input[box_offset + 0] - static_cast(0.5)) * scale_x_y + static_cast(0.5); + const auto tmp_y = (input[box_offset + 1] - static_cast(0.5)) * scale_x_y + static_cast(0.5); - vector2_type bias_xy; - v_load(bias_xy, bias_vPtr[box_of_the_cell]); + output[box_offset + 0] = fast_divide_ftz(static_cast(x) + tmp_x, static_cast(cols)); + output[box_offset + 1] = fast_divide_ftz(static_cast(y) + tmp_y, static_cast(rows)); - using device::fast_exp; - output[box_offset + 2] = fast_exp(input[box_offset + 2]) * bias_xy.data[0] / T(width_norm); - output[box_offset + 3] = fast_exp(input[box_offset + 3]) * bias_xy.data[1] / T(height_norm); + vector2_type bias_xy; + v_load(bias_xy, bias_vPtr[box_of_the_cell]); - /* squash objectness score into a probability */ - using device::fast_sigmoid; - T objectness_prob = fast_sigmoid(input[box_offset + 4]); + output[box_offset + 2] = input[box_offset + 2] * input[box_offset + 2] * + static_cast(4) * bias_xy.data[0] / static_cast(width_norm); + output[box_offset + 3] = input[box_offset + 3] * input[box_offset + 3] * + static_cast(4) * bias_xy.data[1] / static_cast(height_norm); + + objectness_prob = input[box_offset + 4]; + } + else + { + const auto tmp_x = (fast_sigmoid(input[box_offset + 0]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); + const auto tmp_y = (fast_sigmoid(input[box_offset + 1]) - static_cast(0.5)) * scale_x_y + static_cast(0.5); + + output[box_offset + 0] = fast_divide_ftz(static_cast(x) + tmp_x, static_cast(cols)); + output[box_offset + 1] = fast_divide_ftz(static_cast(y) + tmp_y, static_cast(rows)); + + vector2_type bias_xy; + v_load(bias_xy, bias_vPtr[box_of_the_cell]); + + output[box_offset + 2] = fast_exp(input[box_offset + 2]) * bias_xy.data[0] / static_cast(width_norm); + output[box_offset + 3] = fast_exp(input[box_offset + 3]) * bias_xy.data[1] / static_cast(height_norm); + + /* squash objectness score into a probability */ + objectness_prob = fast_sigmoid(input[box_offset + 4]); + } /* ignore prediction if the objectness probability is less than the cutoff */ if (objectness_prob < object_prob_cutoff) @@ -73,7 +94,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { } template - __global__ void region_sigmoid_class_score(Span output, View input, T class_prob_cutoff, size_type box_size) + __global__ void region_sigmoid_class_score(Span output, View input, T class_prob_cutoff, + size_type box_size, bool new_coords) { for (auto idx : grid_stride_range(output.size())) { const index_type box_no = idx / box_size; @@ -92,9 +114,20 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { * * to obtain the actual class probability, we multiply the conditional probability * with the object probability + * + * when new_coords is true, we shouldn't use logistic activation again. */ - using device::fast_sigmoid; - auto actual_class_prob = objectness_prob * fast_sigmoid(input[idx]); + + T actual_class_prob; + if (new_coords) + { + actual_class_prob = objectness_prob * input[idx]; + } + else + { + actual_class_prob = objectness_prob * fast_sigmoid(input[idx]); + } + if (actual_class_prob <= class_prob_cutoff) actual_class_prob = T(0); output[idx] = actual_class_prob; @@ -147,7 +180,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { std::size_t boxes_per_cell, std::size_t box_size, std::size_t rows, std::size_t cols, T scale_x_y, std::size_t height_norm, std::size_t width_norm, - bool if_true_sigmoid_else_softmax /* true = sigmoid, false = softmax */) + bool if_true_sigmoid_else_softmax, /* true = sigmoid, false = softmax */ + bool new_coords) { CV_Assert(output.size() == input.size()); CV_Assert(output.size() % box_size == 0); @@ -158,12 +192,12 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { launch_kernel(box_kernel, box_policy, output, input, bias, boxes_per_cell, box_size, rows, cols, scale_x_y, height_norm, width_norm, - object_prob_cutoff); + object_prob_cutoff, new_coords); if (if_true_sigmoid_else_softmax) { auto kernel_score = raw::region_sigmoid_class_score; auto policy_score = make_policy(kernel_score, output.size(), 0, stream); - launch_kernel(kernel_score, policy_score, output, input, class_prob_cutoff, box_size); + launch_kernel(kernel_score, policy_score, output, input, class_prob_cutoff, box_size, new_coords); } else { auto kernel_score = raw::region_softmax_class_score; auto policy_score = make_policy(kernel_score, output.size(), 0, stream); @@ -173,10 +207,10 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { #if !defined(__CUDA_ARCH__) || (__CUDA_ARCH__ >= 530) template void region(const Stream&, Span<__half>, View<__half>, View<__half>, - __half, __half, std::size_t, std::size_t, std::size_t, std::size_t, __half, std::size_t, std::size_t, bool); + __half, __half, std::size_t, std::size_t, std::size_t, std::size_t, __half, std::size_t, std::size_t, bool, bool); #endif template void region(const Stream&, Span, View, View, - float, float, std::size_t, std::size_t, std::size_t, std::size_t, float, std::size_t, std::size_t, bool); + float, float, std::size_t, std::size_t, std::size_t, std::size_t, float, std::size_t, std::size_t, bool, bool); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp index 092b1571af..0e44372fee 100644 --- a/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/eltwise_ops.hpp @@ -6,26 +6,29 @@ #define OPENCV_DNN_SRC_CUDA4DNN_KERNELS_ELTWISE_OPS_HPP #include "../csl/stream.hpp" -#include "../csl/span.hpp" +#include "../csl/tensor.hpp" #include namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { template - void eltwise_max_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_max_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_sum_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_min_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_sum_coeff_2(const csl::Stream& stream, csl::Span output, T coeff_x, csl::View x, T coeff_y, csl::View y); + void eltwise_sum_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); template - void eltwise_prod_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_sum_coeff_2(const csl::Stream& stream, csl::TensorSpan output, T coeff_x, csl::TensorView x, T coeff_y, csl::TensorView y); template - void eltwise_div_2(const csl::Stream& stream, csl::Span output, csl::View x, csl::View y); + void eltwise_prod_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); + + template + void eltwise_div_2(const csl::Stream& stream, csl::TensorSpan output, csl::TensorView x, csl::TensorView y); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/kernels/region.hpp b/modules/dnn/src/cuda4dnn/kernels/region.hpp index 87742d2f81..b815fb11c9 100644 --- a/modules/dnn/src/cuda4dnn/kernels/region.hpp +++ b/modules/dnn/src/cuda4dnn/kernels/region.hpp @@ -18,7 +18,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { namespace kernels { std::size_t boxes_per_cell, std::size_t box_size, std::size_t rows, std::size_t cols, T scale_x_y, std::size_t height_norm, std::size_t width_norm, - bool if_true_sigmoid_else_softmax); + bool if_true_sigmoid_else_softmax, bool new_coords); }}}} /* namespace cv::dnn::cuda4dnn::kernels */ diff --git a/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp b/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp index adcd7ab3f8..b46f0d870f 100644 --- a/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/eltwise.hpp @@ -25,7 +25,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { MAX, SUM, PRODUCT, - DIV + DIV, + MIN, }; class EltwiseOpBase : public CUDABackendNode { @@ -78,6 +79,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { switch (op) { case EltwiseOpType::MAX: kernels::eltwise_max_2(stream, output, input_x, input_y); break; + case EltwiseOpType::MIN: kernels::eltwise_min_2(stream, output, input_x, input_y); break; case EltwiseOpType::PRODUCT: kernels::eltwise_prod_2(stream, output, input_x, input_y); break; case EltwiseOpType::DIV: kernels::eltwise_div_2(stream, output, input_x, input_y); break; case EltwiseOpType::SUM: @@ -104,6 +106,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { switch (op) { case EltwiseOpType::MAX: kernels::eltwise_max_2(stream, output, output, input); break; + case EltwiseOpType::MIN: kernels::eltwise_min_2(stream, output, output, input); break; case EltwiseOpType::PRODUCT: kernels::eltwise_prod_2(stream, output, output, input); break; case EltwiseOpType::DIV: kernels::eltwise_div_2(stream, output, output, input); break; case EltwiseOpType::SUM: diff --git a/modules/dnn/src/cuda4dnn/primitives/padding.hpp b/modules/dnn/src/cuda4dnn/primitives/padding.hpp index cbf75131a1..ce2a5c3c47 100644 --- a/modules/dnn/src/cuda4dnn/primitives/padding.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/padding.hpp @@ -52,14 +52,19 @@ namespace cv { namespace dnn { namespace cuda4dnn { auto output_wrapper = outputs[0].dynamicCast(); auto output = output_wrapper->getSpan(); - auto effective_rank = get_effective_rank(input); - CV_Assert(get_effective_rank(input) == get_effective_rank(output)); - /* suppose we require padding for the first spatial axis (H in NCHW or D in NCDHW) * * there could be a case where the batch axis, channel axis, and the first spatial axis are all one * this would result in effective rank being less than the number of axes requiring padding */ + /* the effective rank of the input may be smaller than the effective rank of the output but the converse is never true + * input: [1, 1, 1, 3]; effective rank = 1 + * output: [1, 1, 3, 3]; effective rank = 2 + * + * hence, we use the effective rank of the output tensor for the padding operation + */ + auto effective_rank = get_effective_rank(output); + CV_Assert(get_effective_rank(input) <= effective_rank); effective_rank = std::max(effective_rank, dstRanges.size()); for (int i = effective_rank - dstRanges.size(); i < effective_rank; i++) diff --git a/modules/dnn/src/cuda4dnn/primitives/region.hpp b/modules/dnn/src/cuda4dnn/primitives/region.hpp index 7813a47bc7..d22d44214e 100644 --- a/modules/dnn/src/cuda4dnn/primitives/region.hpp +++ b/modules/dnn/src/cuda4dnn/primitives/region.hpp @@ -60,6 +60,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { T class_prob_cutoff; T nms_iou_threshold; + bool new_coords; }; template @@ -87,6 +88,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { class_prob_cutoff = config.class_prob_cutoff; nms_iou_threshold = config.nms_iou_threshold; + new_coords = config.new_coords; } void forward( @@ -115,7 +117,8 @@ namespace cv { namespace dnn { namespace cuda4dnn { boxes_per_cell, cell_box_size, rows, cols, scale_x_y, height_norm, width_norm, - if_true_sigmoid_else_softmax + if_true_sigmoid_else_softmax, + new_coords ); if (nms_iou_threshold > 0) { @@ -176,6 +179,7 @@ namespace cv { namespace dnn { namespace cuda4dnn { T object_prob_cutoff, class_prob_cutoff; T nms_iou_threshold; + bool new_coords; }; }}} /* namespace cv::dnn::cuda4dnn */ diff --git a/modules/dnn/src/darknet/darknet_io.cpp b/modules/dnn/src/darknet/darknet_io.cpp index 4915538ff7..99715df829 100644 --- a/modules/dnn/src/darknet/darknet_io.cpp +++ b/modules/dnn/src/darknet/darknet_io.cpp @@ -470,7 +470,7 @@ namespace cv { fused_layer_names.push_back(last_layer); } - void setYolo(int classes, const std::vector& mask, const std::vector& anchors, float thresh, float nms_threshold, float scale_x_y) + void setYolo(int classes, const std::vector& mask, const std::vector& anchors, float thresh, float nms_threshold, float scale_x_y, int new_coords) { cv::dnn::LayerParams region_param; region_param.name = "Region-name"; @@ -484,6 +484,7 @@ namespace cv { region_param.set("thresh", thresh); region_param.set("nms_threshold", nms_threshold); region_param.set("scale_x_y", scale_x_y); + region_param.set("new_coords", new_coords); std::vector usedAnchors(numAnchors * 2); for (int i = 0; i < numAnchors; ++i) @@ -882,6 +883,7 @@ namespace cv { float thresh = getParam(layer_params, "thresh", 0.2); float nms_threshold = getParam(layer_params, "nms_threshold", 0.0); float scale_x_y = getParam(layer_params, "scale_x_y", 1.0); + int new_coords = getParam(layer_params, "new_coords", 0); std::string anchors_values = getParam(layer_params, "anchors", std::string()); CV_Assert(!anchors_values.empty()); @@ -894,7 +896,7 @@ namespace cv { CV_Assert(classes > 0 && num_of_anchors > 0 && (num_of_anchors * 2) == anchors_vec.size()); setParams.setPermute(false); - setParams.setYolo(classes, mask_vec, anchors_vec, thresh, nms_threshold, scale_x_y); + setParams.setYolo(classes, mask_vec, anchors_vec, thresh, nms_threshold, scale_x_y, new_coords); } else { CV_Error(cv::Error::StsParseError, "Unknown layer type: " + layer_type); diff --git a/modules/dnn/src/debug_utils.cpp b/modules/dnn/src/debug_utils.cpp new file mode 100644 index 0000000000..d951205bd8 --- /dev/null +++ b/modules/dnn/src/debug_utils.cpp @@ -0,0 +1,91 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include + +#include +#include +#include + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +bool DNN_DIAGNOSTICS_RUN = false; +bool DNN_SKIP_REAL_IMPORT = false; + +void enableModelDiagnostics(bool isDiagnosticsMode) +{ + DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; + + if (DNN_DIAGNOSTICS_RUN) + { + detail::NotImplemented::Register(); + } + else + { + detail::NotImplemented::unRegister(); + } +} + +void skipModelImport(bool skip) +{ + DNN_SKIP_REAL_IMPORT = skip; +} + +void detail::LayerHandler::addMissing(const std::string& name, const std::string& type) +{ + cv::AutoLock lock(getLayerFactoryMutex()); + auto& registeredLayers = getLayerFactoryImpl(); + + // If we didn't add it, but can create it, it's custom and not missing. + if (layers.find(type) == layers.end() && registeredLayers.find(type) != registeredLayers.end()) + { + return; + } + + layers[type].insert(name); +} + +bool detail::LayerHandler::contains(const std::string& type) const +{ + return layers.find(type) != layers.end(); +} + +void detail::LayerHandler::printMissing() +{ + if (layers.empty()) + { + return; + } + + std::stringstream ss; + ss << "DNN: Not supported types:\n"; + for (const auto& type_names : layers) + { + const auto& type = type_names.first; + ss << "Type='" << type << "', affected nodes:\n["; + for (const auto& name : type_names.second) + { + ss << "'" << name << "', "; + } + ss.seekp(-2, std::ios_base::end); + ss << "]\n"; + } + CV_LOG_ERROR(NULL, ss.str()); +} + +LayerParams detail::LayerHandler::getNotImplementedParams(const std::string& name, const std::string& op) +{ + LayerParams lp; + lp.name = name; + lp.type = "NotImplemented"; + lp.set("type", op); + + return lp; +} + +CV__DNN_INLINE_NS_END +}} // namespace diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index 668cce8fa6..62607e247a 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -94,13 +94,6 @@ static bool DNN_CHECK_NAN_INF = utils::getConfigurationParameterBool("OPENCV_DNN static bool DNN_CHECK_NAN_INF_DUMP = utils::getConfigurationParameterBool("OPENCV_DNN_CHECK_NAN_INF_DUMP", false); static bool DNN_CHECK_NAN_INF_RAISE_ERROR = utils::getConfigurationParameterBool("OPENCV_DNN_CHECK_NAN_INF_RAISE_ERROR", false); -bool DNN_DIAGNOSTICS_RUN = false; - -void enableModelDiagnostics(bool isDiagnosticsMode) -{ - DNN_DIAGNOSTICS_RUN = isDiagnosticsMode; -} - using std::vector; using std::map; using std::make_pair; @@ -565,9 +558,9 @@ struct LayerPin struct LayerData { - LayerData() : id(-1), skip(false), flag(0) {} - LayerData(int _id, const String &_name, const String &_type, LayerParams &_params) - : id(_id), name(_name), type(_type), params(_params), skip(false), flag(0) + LayerData() : id(-1), dtype(CV_32F), skip(false), flag(0) {} + LayerData(int _id, const String &_name, const String &_type, const int &_dtype, LayerParams &_params) + : id(_id), name(_name), type(_type), dtype(_dtype), params(_params), skip(false), flag(0) { CV_TRACE_FUNCTION(); @@ -579,6 +572,7 @@ struct LayerData int id; String name; String type; + int dtype; // Datatype of output blobs. LayerParams params; std::vector inputBlobsId; @@ -935,7 +929,7 @@ public: } } - void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst, bool use_half) + void reuseOrCreate(const MatShape& shape, const LayerPin& lp, Mat& dst, const int& dtype) { if (!DNN_DISABLE_MEMORY_OPTIMIZATIONS) { @@ -957,7 +951,8 @@ public: { Mat& unusedBlob = hostIt->second; if (unusedBlob.total() >= targetTotal && - unusedBlob.total() < bestBlobTotal) + unusedBlob.total() < bestBlobTotal && + unusedBlob.type() == dtype) { bestBlobPin = hostIt->first; bestBlob = unusedBlob; @@ -976,14 +971,13 @@ public: { // if dst already has been allocated with total(shape) elements, // it won't be recreated and pointer of dst.data remains the same. - dst.create(shape, use_half ? CV_16S : CV_32F); + dst.create(shape, dtype); addHost(lp, dst); } } void allocateBlobsForLayer(LayerData &ld, const LayerShapes& layerShapes, - std::vector& pinsForInternalBlobs, - bool use_half = false) + std::vector& pinsForInternalBlobs) { CV_TRACE_FUNCTION(); @@ -1054,7 +1048,7 @@ public: reuse(ld.inputBlobsId[0], blobPin); } else - reuseOrCreate(shapes[index], blobPin, *blobs[index], use_half); + reuseOrCreate(shapes[index], blobPin, *blobs[index], ld.dtype); } } } @@ -1184,6 +1178,7 @@ struct Net::Impl : public detail::NetImplBase lastLayerId = 0; netWasAllocated = false; + netWasQuantized = false; fusion = true; isAsync = false; preferableBackend = DNN_BACKEND_DEFAULT; @@ -1208,6 +1203,7 @@ struct Net::Impl : public detail::NetImplBase int lastLayerId; bool netWasAllocated; + bool netWasQuantized; bool fusion; bool isAsync; std::vector layersTimings; @@ -1363,7 +1359,7 @@ struct Net::Impl : public detail::NetImplBase currLayer->unsetAttached(); } - + netWasAllocated = false; layersTimings.clear(); } @@ -2078,7 +2074,10 @@ struct Net::Impl : public detail::NetImplBase Ptr ieNode = node.dynamicCast(); CV_Assert(!ieNode.empty()); - ieNode->net->reset(); + + CV_Assert(ieNode->net); + InfEngineNgraphNet& ienet = *ieNode->net; + ienet.reset(); for (it = layers.begin(); it != layers.end(); ++it) { @@ -2095,16 +2094,26 @@ struct Net::Impl : public detail::NetImplBase { for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) { - InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); - dataPtr->setName(ld.name); + auto it = ienet.outputsDesc.find(ld.name); + if (it != ienet.outputsDesc.end()) + { + const InferenceEngine::TensorDesc& descriptor = it->second; + InferenceEngine::DataPtr dataPtr = ngraphDataOutputNode(ld.outputBlobsWrappers[i], descriptor, ld.name); + dataPtr->setName(ld.name); + } + else + { + InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); + dataPtr->setName(ld.name); + } } } - ieNode->net->addBlobs(ld.inputBlobsWrappers); - ieNode->net->addBlobs(ld.outputBlobsWrappers); + ienet.addBlobs(ld.inputBlobsWrappers); + ienet.addBlobs(ld.outputBlobsWrappers); ld.skip = true; } layers[lastLayerId].skip = false; - ieNode->net->init((Target)preferableTarget); + ienet.init((Target)preferableTarget); return; } @@ -2519,10 +2528,11 @@ struct Net::Impl : public detail::NetImplBase CV_Assert(layerShapesIt != layersShapes.end()); + if (preferableBackend == DNN_BACKEND_OPENCV && preferableTarget == DNN_TARGET_OPENCL_FP16 && ld.dtype == CV_32F) + ld.dtype = CV_16S; + std::vector pinsForInternalBlobs; - blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs, - preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16); + blobManager.allocateBlobsForLayer(ld, layerShapesIt->second, pinsForInternalBlobs); ld.outputBlobsWrappers.resize(ld.outputBlobs.size()); for (int i = 0; i < ld.outputBlobs.size(); ++i) ld.outputBlobsWrappers[i] = wrap(ld.outputBlobs[i]); @@ -2700,7 +2710,19 @@ struct Net::Impl : public detail::NetImplBase // we create a temporary backend node for eltwise layer to obtain the eltwise configuration cuda4dnn::csl::CSLContext context; // assume that initCUDA and EltwiseOp do not use the context during init const auto node = nextData->layerInstance->initCUDA(&context, nextData->inputBlobsWrappers, nextData->outputBlobsWrappers); - const auto eltwiseNode = node.dynamicCast(); + auto eltwiseNode = node.dynamicCast(); + + // broadcasting not supported in fused ops + auto required_shape = shape(nextData->outputBlobs[0]); + for (int i = 0; i < nextData->inputBlobs.size(); i++) + { + if (shape(*nextData->inputBlobs[i]) != required_shape) + { + eltwiseNode.reset(); + break; + } + } + // CUDA backend uses EltwiseOp when all operands have the same number of channels; otherwise, ShortcutOp is used. // Hence, a successful cast to EltwiseOp implies that the number of channels is same in all operand tensors. if (eltwiseNode.empty() || eltwiseNode->op != cuda4dnn::EltwiseOpType::SUM || !eltwiseNode->coeffs.empty()) @@ -3126,7 +3148,8 @@ struct Net::Impl : public detail::NetImplBase Mat& inp = layers[0].outputBlobs[i]; CV_Assert(inp.total()); if (preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16) + preferableTarget == DNN_TARGET_OPENCL_FP16 && + layers[0].dtype == CV_32F) { layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S); } @@ -3436,6 +3459,25 @@ struct Net::Impl : public detail::NetImplBase #endif } + void getQuantizationParams(const Mat& src, std::vector& scales, std::vector& zeropoints) + { + const int qmin = -128; // INT8_MIN + const int qmax = 127; // INT8_MAX + + double rmin, rmax, sc, zp; + cv::minMaxIdx(src, &rmin, &rmax); + + // 0 must be present in the range [rmin, rmax] + rmin = std::min(rmin, 0.0); + rmax = std::max(rmax, 0.0); + + sc = (rmax == rmin) ? 1.0 : (rmax - rmin)/(qmax - qmin); + zp = qmin - (rmin/sc); + + scales.push_back((float)sc); + zeropoints.push_back((int)std::round(zp)); + } + void getLayerShapesRecursively(int id, LayersShapesMap& inOutShapes) { std::vector& inputLayerIds = layers[id].inputBlobsId; @@ -3566,7 +3608,8 @@ struct Net::Impl : public detail::NetImplBase Mat& inp = layers[0].outputBlobs[i]; CV_Assert(inp.total()); if (preferableBackend == DNN_BACKEND_OPENCV && - preferableTarget == DNN_TARGET_OPENCL_FP16) + preferableTarget == DNN_TARGET_OPENCL_FP16 && + layers[0].dtype == CV_32F) { layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S); } @@ -3592,7 +3635,7 @@ struct Net::Impl : public detail::NetImplBase const MatShape& shape = layersShapes[inputLayerId].out[inputLayerIds[i].oid]; layersShapes[layerId].in.push_back(shape); } - it->second.layerInstance->updateMemoryShapes(layersShapes[layerId].in); + it->second.getLayerInstance()->updateMemoryShapes(layersShapes[layerId].in); } } } @@ -3997,33 +4040,59 @@ Net::~Net() { } -int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) +int Net::addLayer(const String &name, const String &type, const int &dtype, LayerParams ¶ms) { CV_TRACE_FUNCTION(); - if (impl->getLayerId(name) >= 0) + int id = impl->getLayerId(name); + if (id >= 0) { - CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net"); - return -1; + if (!DNN_DIAGNOSTICS_RUN || type != "NotImplemented") + { + CV_Error(Error::StsBadArg, "Layer \"" + name + "\" already into net"); + return -1; + } + else + { + LayerData& ld = impl->layers.find(id)->second; + ld.type = type; + ld.params = params; + return -1; + } } - int id = ++impl->lastLayerId; + id = ++impl->lastLayerId; impl->layerNameToId.insert(std::make_pair(name, id)); - impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params))); + impl->layers.insert(std::make_pair(id, LayerData(id, name, type, dtype, params))); if (params.get("has_dynamic_shapes", false)) impl->hasDynamicShapes = true; + if (dtype == CV_8S) + impl->netWasQuantized = true; + return id; } +int Net::addLayer(const String &name, const String &type, LayerParams ¶ms) +{ + CV_TRACE_FUNCTION(); + return addLayer(name, type, CV_32F, params); +} + +int Net::addLayerToPrev(const String &name, const String &type, const int &dtype, LayerParams ¶ms) +{ + CV_TRACE_FUNCTION(); + + int prvLid = impl->lastLayerId; + int newLid = this->addLayer(name, type, dtype, params); + this->connect(prvLid, 0, newLid, 0); + return newLid; +} + int Net::addLayerToPrev(const String &name, const String &type, LayerParams ¶ms) { CV_TRACE_FUNCTION(); - - int prvLid = impl->lastLayerId; - int newLid = this->addLayer(name, type, params); - this->connect(prvLid, 0, newLid, 0); - return newLid; + return addLayerToPrev(name, type, CV_32F, params); } void Net::connect(int outLayerId, int outNum, int inpLayerId, int inpNum) @@ -4136,16 +4205,19 @@ void Net::forward(OutputArrayOfArrays outputBlobs, const String& outputName) ld.outputBlobsWrappers[i]->copyToHost(); } } - if (ld.outputBlobs[0].depth() == CV_32F) + if (ld.outputBlobs[0].depth() == CV_16S) { - std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); - outputvec = ld.outputBlobs; - } else { std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); outputvec.resize(ld.outputBlobs.size()); for (int i = 0; i < outputvec.size(); i++) convertFp16(ld.outputBlobs[i], outputvec[i]); } + else + { + // Output depth can be CV_32F or CV_8S + std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); + outputvec = ld.outputBlobs; + } } else if (outputBlobs.isUMatVector()) { @@ -4198,8 +4270,8 @@ void Net::forward(OutputArrayOfArrays outputBlobs, matvec.push_back(impl->getBlob(pins[i])); } - std::vector & outputvec = *(std::vector *)outputBlobs.getObj(); - outputvec = matvec; + outputBlobs.create((int)matvec.size(), 1, CV_32F/*FIXIT*/, -1); // allocate vector + outputBlobs.assign(matvec); } void Net::forward(std::vector >& outputBlobs, @@ -4231,11 +4303,277 @@ void Net::forward(std::vector >& outputBlobs, } } +Net Net::quantize(InputArrayOfArrays calibData, int inputsDtype, int outputsDtype) +{ + CV_TRACE_FUNCTION(); + + // Net can be quantized only once. + if (impl->netWasQuantized) + CV_Error(Error::StsBadArg, "Cannot quantize a quantized net"); + + CV_CheckType(inputsDtype, inputsDtype == CV_32F || inputsDtype == CV_8S, "Input depth should be CV_32F or CV_8S"); + CV_CheckType(outputsDtype, outputsDtype == CV_32F || outputsDtype == CV_8S, "Output depth should be CV_32F or CV_8S"); + + bool originalFusion = impl->fusion; + int prefBackend = impl->preferableBackend; + int prefTarget = impl->preferableTarget; + + // Disable fusions and use CPU backend to quantize net + setPreferableBackend(DNN_BACKEND_OPENCV); + setPreferableTarget(DNN_TARGET_CPU); + enableFusion(false); + + if (calibData.isMat()) + { + setInput(calibData.getMat()); + } + else if (calibData.isMatVector()) + { + std::vector calibDataVec; + calibData.getMatVector(calibDataVec); + + std::vector inpNames = impl->netInputLayer->outNames; + CV_CheckEQ(calibDataVec.size(), inpNames.size(), "Calibration data size should be equal to number of inputs"); + for (int i = 0; i < calibDataVec.size(); i++) + setInput(calibDataVec[i], inpNames[i]); + } + + std::vector outNames = getUnconnectedOutLayersNames(); + std::vector pins; + for (int i = 0; i < outNames.size(); i++) + pins.push_back(impl->getPinByAlias(outNames[i])); + impl->setUpNet(pins); + + // Compute scales and zeropoints for all the layers + std::vector > scales; + std::vector > zeropoints; + for (Impl::MapIdToLayerData::iterator it = impl->layers.begin(); it != impl->layers.end(); it++) + { + LayerData& ld = it->second; + if (!ld.skip) + { + Ptr layer = ld.layerInstance; + std::vector inps(ld.inputBlobs.size()); + for (int i = 0; i < ld.inputBlobs.size(); ++i) + inps[i] = *ld.inputBlobs[i]; + layer->forward(inps, ld.outputBlobs, ld.internals); + } + + std::vector sc; + std::vector zp; + if (ld.type == "TanH") + { + sc.push_back(1.f/128); + zp.push_back(0); + } + else if (ld.type == "Sigmoid" || ld.type == "Softmax" || ld.type == "SoftMax") + { + if (ld.params.get("log_softmax", false)) + { + sc.push_back(16.f/256); + zp.push_back(127); + } + else + { + sc.push_back(1.f/256); + zp.push_back(-128); + } + } + else if (ld.type == "Split" || ld.type == "Slice" || ld.type == "Crop") + { + std::vector inp_sc; std::vector inp_zp; + impl->getQuantizationParams(*ld.inputBlobs[0], inp_sc, inp_zp); + sc.assign(ld.outputBlobs.size(), inp_sc[0]); + zp.assign(ld.outputBlobs.size(), inp_zp[0]); + } + else + { + for (int i = 0; i < ld.outputBlobs.size(); i++) + impl->getQuantizationParams(ld.outputBlobs[i], sc, zp); + } + scales.push_back(sc); + zeropoints.push_back(zp); + } + + // For some layers, the input and output scales/zeropoints must be equal so that rescaling of inputs + // is not needed during quantized inference. We start from the last layer and modify the layer's input scales/zeropoints + // TODO : Need a different approach. Current solution fails when 2 such layers have the same input layer + for (Impl::MapIdToLayerData::reverse_iterator it = impl->layers.rbegin(); it != impl->layers.rend(); ++it) + { + LayerData& ld = it->second; + // Layers with multiple outputs. Number of outputs is equal to number of inputs + if (ld.type == "Blank" || ld.type == "Dropout" || ld.type == "Identity" || ld.type == "Silence" || + ld.type == "Flatten" || ld.type == "Padding" || ld.type == "Permute" || ld.type == "Reshape" || + ld.type == "ReLU6" || ld.type == "Reorg" || ld.type == "ShuffleChannel" || ld.type == "Resize" || + (ld.type == "ReLU" && !ld.params.get("negative_slope", 0.f)) /* ReLU with negative slope 0 */) + { + for (int i = 0; i < ld.outputBlobs.size(); i++) + { + LayerPin &pin = ld.inputBlobsId[i]; + scales[pin.lid][pin.oid] = scales[ld.id][i]; + zeropoints[pin.lid][pin.oid] = zeropoints[ld.id][i]; + } + } + // Layers with multiple inputs and single output. + else if ((ld.type == "Pooling" && toLowerCase(ld.params.get("pool", "max")) == "max") /* Max Pooling */ || + (ld.type == "Eltwise" && toLowerCase(ld.params.get("operation", "sum")) == "max") /* Elementwise max */ || + ld.type == "Concat") + { + for (int i = 0; i < ld.inputBlobsId.size(); i++) + { + LayerPin &pin = ld.inputBlobsId[i]; + scales[pin.lid][pin.oid] = scales[ld.id][0]; + zeropoints[pin.lid][pin.oid] = zeropoints[ld.id][0]; + } + } + } + + // Create a new Net and add quantized layers to it. + Net dstNet; + dstNet.impl->netWasQuantized = true; + dstNet.setInputsNames(impl->netInputLayer->outNames); + dstNet.setPreferableBackend(prefBackend); + dstNet.setPreferableTarget(prefTarget); + dstNet.enableFusion(originalFusion); + + for (Impl::MapIdToLayerData::iterator it = impl->layers.begin(); it != impl->layers.end(); it++) + { + LayerData ld = it->second; + if (ld.id == 0) + { + LayerData &quantInpLd = dstNet.impl->layers[0]; + quantInpLd.dtype = inputsDtype; + quantInpLd.params.set("scales", DictValue::arrayReal(scales[0].data(), scales[0].size())); + quantInpLd.params.set("zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + continue; + } + + std::vector inpPins = ld.inputBlobsId; + // Fill input and output scales/zeropoints for the layer + std::vector > inp_out_sc(2); + std::vector > inp_out_zp(2); + for (int i = 0; i < inpPins.size(); i++) + { + LayerPin &pin = inpPins[i]; + inp_out_sc[0].push_back(scales[pin.lid][pin.oid]); + inp_out_zp[0].push_back(zeropoints[pin.lid][pin.oid]); + } + inp_out_sc[1] = scales[ld.id]; + inp_out_zp[1] = zeropoints[ld.id]; + + // Quantize layer + Ptr layer = ld.layerInstance; + if (layer->tryQuantize(inp_out_sc, inp_out_zp, ld.params)) + { + ld.type += "Int8"; + ld.dtype = CV_8S; + } + ld.params.set("scales", DictValue::arrayReal(inp_out_sc[1].data(), inp_out_sc[1].size())); + ld.params.set("zeropoints", DictValue::arrayInt(inp_out_zp[1].data(), inp_out_zp[1].size())); + + // Check and add quantize/dequantize node before layer + for (int i = 0; i < inpPins.size(); i++) + { + LayerPin &pin = inpPins[i]; + LayerData &inpLd = dstNet.impl->getLayerData(impl->getLayerName(pin.lid)); + pin.lid = inpLd.id; + if (inpLd.dtype != ld.dtype) + { + String layerName = (inpLd.dtype == CV_32F && ld.dtype == CV_8S) ? cv::format("quantize/%s/%d", inpLd.name.c_str(), pin.oid) + : cv::format("dequantize/%s/%d", inpLd.name.c_str(), pin.oid); + // Check if quantize/dequantize node for the input layer already exists + if (dstNet.impl->getLayerId(layerName) >= 0) + { + pin.lid = dstNet.impl->getLayerId(layerName); + pin.oid = 0; + } + else + { + LayerParams lp; + lp.set("scales", inp_out_sc[0][i]); + lp.set("zeropoints", inp_out_zp[0][i]); + lp.name = layerName; + lp.type = (inpLd.dtype == CV_32F && ld.dtype == CV_8S) ? "Quantize" : "Dequantize"; + int newLid = dstNet.addLayer(lp.name, lp.type, ld.dtype, lp); + dstNet.connect(pin.lid, pin.oid, newLid, 0); + pin.lid = newLid; pin.oid = 0; + } + } + } + + // Add quantized layer to Net and connect to its inputs. + int newLid = dstNet.addLayer(ld.name, ld.type, ld.dtype, ld.params); + for( int i = 0; i < inpPins.size(); i++ ) + dstNet.connect(inpPins[i].lid, inpPins[i].oid, newLid, i); + + // If the layer is a output layer, add quantize/dequantize node after it based on output's data type. + if (ld.requiredOutputs.size() == 0 && ld.dtype != outputsDtype) + { + LayerParams lp; + lp.set("scales", inp_out_sc[1][0]); + lp.set("zeropoints", inp_out_zp[1][0]); + lp.name = ((ld.dtype == CV_32F && outputsDtype == CV_8S) ? "quantize/" : "dequantize/") + ld.name; + lp.type = (ld.dtype == CV_32F && outputsDtype == CV_8S) ? "Quantize" : "Dequantize"; + dstNet.addLayerToPrev(lp.name, lp.type, outputsDtype, lp); + } + } + // Restore FP32 Net's backend, target and fusion + setPreferableBackend(prefBackend); + setPreferableTarget(prefTarget); + enableFusion(originalFusion); + return dstNet; +} + +void Net::getInputDetails(std::vector& scales, std::vector& zeropoints) const +{ + if (!impl->netWasQuantized) + CV_Error(Error::StsBadFunc, "Net isn't quantized"); + + LayerParams &lp = impl->layers[0].params; + DictValue sc = lp.get("scales"); + DictValue zp = lp.get("zeropoints"); + + for (int i = 0; i < sc.size(); i++) + { + scales.push_back(sc.get(i)); + zeropoints.push_back(zp.get(i)); + } +} + +void Net::getOutputDetails(std::vector& scales, std::vector& zeropoints) const +{ + if (!impl->netWasQuantized) + CV_Error(Error::StsBadFunc, "Net isn't quantized"); + + std::vector outLayerIds = getUnconnectedOutLayers(); + for (auto &lid : outLayerIds) + { + LayerParams &lp = impl->layers[lid].params; + DictValue sc = lp.get("scales"); + DictValue zp = lp.get("zeropoints"); + + for (int i = 0; i < sc.size(); i++) + { + scales.push_back(sc.get(i)); + zeropoints.push_back(zp.get(i)); + } + } +} + void Net::setPreferableBackend(int backendId) { CV_TRACE_FUNCTION(); CV_TRACE_ARG(backendId); + if (backendId == DNN_BACKEND_DEFAULT) + backendId = (Backend)PARAM_DNN_BACKEND_DEFAULT; + + if (impl->netWasQuantized && backendId != DNN_BACKEND_OPENCV) + { + CV_LOG_WARNING(NULL, "DNN: Only default backend supports quantized networks"); + backendId = DNN_BACKEND_OPENCV; + } + #ifdef HAVE_INF_ENGINE if (backendId == DNN_BACKEND_INFERENCE_ENGINE) backendId = getInferenceEngineBackendTypeParam(); @@ -4244,7 +4582,6 @@ void Net::setPreferableBackend(int backendId) if( impl->preferableBackend != backendId ) { impl->preferableBackend = backendId; - impl->netWasAllocated = false; impl->clear(); } } @@ -4254,6 +4591,13 @@ void Net::setPreferableTarget(int targetId) CV_TRACE_FUNCTION(); CV_TRACE_ARG(targetId); + if (impl->netWasQuantized && targetId != DNN_TARGET_CPU && + targetId != DNN_TARGET_OPENCL && targetId != DNN_TARGET_OPENCL_FP16) + { + CV_LOG_WARNING(NULL, "DNN: Only CPU and OpenCL/OpenCL FP16 target is supported by quantized networks"); + targetId = DNN_TARGET_CPU; + } + if( impl->preferableTarget != targetId ) { impl->preferableTarget = targetId; @@ -4273,7 +4617,6 @@ void Net::setPreferableTarget(int targetId) impl->preferableTarget = DNN_TARGET_OPENCL; #endif } - impl->netWasAllocated = false; impl->clear(); } } @@ -4902,9 +5245,10 @@ void Net::getMemoryConsumption(const int layerId, ShapesVec inLayerShapes, outLayerShapes; getLayerShapes(netInputShapes, layerId, inLayerShapes, outLayerShapes); + size_t elemSize = (impl->netWasQuantized) ? sizeof(char) : sizeof(float); for(int i = 0; i < outLayerShapes.size(); i++) { - blobs += total(outLayerShapes[i]) * sizeof(float); + blobs += total(outLayerShapes[i]) * elemSize; } } @@ -4953,7 +5297,7 @@ void Net::getMemoryConsumption(const std::vector& netInputShapes, std::vector > inLayerShapes, outLayerShapes; getLayersShapes(netInputShapes, layerIds, inLayerShapes, outLayerShapes); - + size_t elemSize = (impl->netWasQuantized) ? sizeof(char) : sizeof(float); for(int i = 0; i < layerIds.size(); i++) { int w = 0, b = 0; @@ -4968,7 +5312,7 @@ void Net::getMemoryConsumption(const std::vector& netInputShapes, for(int j = 0; j < outLayerShapes[i].size(); j++) { - b += total(outLayerShapes[i][j]) * sizeof(float); + b += total(outLayerShapes[i][j]) * elemSize; } weights.push_back(w); @@ -4988,7 +5332,6 @@ void Net::enableFusion(bool fusion) if( impl->fusion != fusion ) { impl->fusion = fusion; - impl->netWasAllocated = false; impl->clear(); } } @@ -5162,6 +5505,12 @@ void Layer::getScaleShift(Mat& scale, Mat& shift) const shift = Mat(); } +void Layer::getScaleZeropoint(float& scale, int& zeropoint) const +{ + scale = 1.f; + zeropoint = 0; +} + void Layer::unsetAttached() { setActivation(Ptr()); @@ -5288,6 +5637,12 @@ void Layer::run(const std::vector &inputs, std::vector &outputs, std:: this->forward(inputs, outputs, internals); } +bool Layer::tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) +{ + return false; +} + Layer::~Layer() {} bool Layer::getMemoryShapes(const std::vector &inputs, @@ -5306,7 +5661,7 @@ bool Layer::updateMemoryShapes(const std::vector &inputs) } ////////////////////////////////////////////////////////////////////////// -static Mutex& getLayerFactoryMutex() +Mutex& getLayerFactoryMutex() { static Mutex* volatile instance = NULL; if (instance == NULL) diff --git a/modules/dnn/src/dnn_common.hpp b/modules/dnn/src/dnn_common.hpp index ff8f5e8467..3c68322e09 100644 --- a/modules/dnn/src/dnn_common.hpp +++ b/modules/dnn/src/dnn_common.hpp @@ -5,6 +5,9 @@ #ifndef __OPENCV_DNN_COMMON_HPP__ #define __OPENCV_DNN_COMMON_HPP__ +#include +#include + #include namespace cv { namespace dnn { @@ -13,7 +16,57 @@ CV__DNN_INLINE_NS_BEGIN Mutex& getInitializationMutex(); void initializeLayerFactory(); +extern bool DNN_DIAGNOSTICS_RUN; +extern bool DNN_SKIP_REAL_IMPORT; + namespace detail { +#define CALL_MEMBER_FN(object, ptrToMemFn) ((object).*(ptrToMemFn)) + +class NotImplemented : public Layer +{ +public: + static Ptr create(const LayerParams ¶ms); + + static void Register(); + static void unRegister(); +}; + +template +Net readNet(Args&& ... args) +{ + Net net; + Importer importer(net, std::forward(args)...); + return net; +} + +template +Net readNetDiagnostic(Args&& ... args) +{ + Net maybeDebugNet = readNet(std::forward(args)...); + if (DNN_DIAGNOSTICS_RUN && !DNN_SKIP_REAL_IMPORT) + { + // if we just imported the net in diagnostic mode, disable it and import again + enableModelDiagnostics(false); + Net releaseNet = readNet(std::forward(args)...); + enableModelDiagnostics(true); + return releaseNet; + } + return maybeDebugNet; +} + +class LayerHandler +{ +public: + void addMissing(const std::string& name, const std::string& type); + bool contains(const std::string& type) const; + void printMissing(); + +protected: + LayerParams getNotImplementedParams(const std::string& name, const std::string& op); + +private: + std::unordered_map> layers; +}; struct NetImplBase { diff --git a/modules/dnn/src/ie_ngraph.cpp b/modules/dnn/src/ie_ngraph.cpp index 7484032714..a61766337e 100644 --- a/modules/dnn/src/ie_ngraph.cpp +++ b/modules/dnn/src/ie_ngraph.cpp @@ -657,7 +657,11 @@ void InfEngineNgraphNet::initPlugin(InferenceEngine::CNNNetwork& net) try { InferenceEngine::IExtensionPtr extension = +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + std::make_shared(libName); +#else InferenceEngine::make_so_pointer(libName); +#endif ie.AddExtension(extension, "CPU"); CV_LOG_INFO(NULL, "DNN-IE: Loaded extension plugin: " << libName); @@ -788,21 +792,32 @@ void NgraphBackendLayer::forward(InputArrayOfArrays inputs, OutputArrayOfArrays } -static InferenceEngine::Layout estimateLayout(const Mat& m) +static InferenceEngine::Layout estimateLayout(int dims) { - if (m.dims == 4) + if (dims == 4) return InferenceEngine::Layout::NCHW; - else if (m.dims == 3) + else if (dims == 3) return InferenceEngine::Layout::CHW; - else if (m.dims == 2) + else if (dims == 2) return InferenceEngine::Layout::NC; - else if (m.dims == 1) + else if (dims == 1) return InferenceEngine::Layout::C; - else if (m.dims == 5) + else if (dims == 5) return InferenceEngine::Layout::NCDHW; else return InferenceEngine::Layout::ANY; } +static inline +InferenceEngine::Layout estimateLayout(size_t dims) +{ + return estimateLayout((int)dims); +} + +static inline +InferenceEngine::Layout estimateLayout(const Mat& m) +{ + return estimateLayout(m.dims); +} static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "") { @@ -838,6 +853,7 @@ InferenceEngine::Blob::Ptr wrapToNgraphBlob(const Mat& m, InferenceEngine::Layou NgraphBackendWrapper::NgraphBackendWrapper(int targetId, const cv::Mat& m) : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, targetId) + , host((Mat*)&m) { dataPtr = wrapToInfEngineDataNode(m); blob = wrapToNgraphBlob(m, estimateLayout(m)); @@ -889,7 +905,11 @@ InferenceEngine::Blob::Ptr copyBlob(const InferenceEngine::Blob::Ptr& blob) copy = InferenceEngine::make_shared_blob(description); } else - CV_Error(Error::StsNotImplemented, "Unsupported blob precision"); + { + std::ostringstream msg; + msg << precision; + CV_Error_(Error::StsNotImplemented, ("Unsupported blob precision: %s", msg.str().c_str())); + } copy->allocate(); return copy; } @@ -902,6 +922,66 @@ InferenceEngine::DataPtr ngraphDataNode(const Ptr& ptr) return p->dataPtr; } +static +InferenceEngine::Blob::Ptr reallocateBlob(Mat &m, const InferenceEngine::TensorDesc& description) +{ + auto dims = description.getDims(); + auto layout = estimateLayout(dims.size()); + MatShape matShape(dims.begin(), dims.end()); + if (description.getPrecision() == InferenceEngine::Precision::FP32) + { + m.create(matShape, CV_32FC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (float*)m.data); + } + else if (description.getPrecision() == InferenceEngine::Precision::I32) + { + m.create(matShape, CV_32SC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (int*)m.data); + } + else if (description.getPrecision() == InferenceEngine::Precision::U8) + { + m.create(matShape, CV_8UC1); + return InferenceEngine::make_shared_blob( + {description.getPrecision(), dims, layout}, (uchar*)m.data); + } + std::ostringstream msg; + msg << "Unsupported IE precision: " << description.getPrecision(); + CV_Error(Error::StsNotImplemented, msg.str()); +} + +InferenceEngine::DataPtr ngraphDataOutputNode( + const Ptr& ptr, + const InferenceEngine::TensorDesc& description, + const std::string name) +{ + CV_Assert(!ptr.empty()); + Ptr p = ptr.dynamicCast(); + CV_Assert(!p.empty()); + NgraphBackendWrapper& w = *p; + const InferenceEngine::TensorDesc& blobDesc = w.blob.get()->getTensorDesc(); + auto dims = description.getDims(); + bool reallocate = false; + if (blobDesc.getPrecision() != description.getPrecision()) + { + reallocate = true; + CV_LOG_WARNING(NULL, "Reallocate output '" << name << "' blob due to wrong precision: " << blobDesc.getPrecision() << " => " << description.getPrecision() << " ndims=" << dims.size()); + } + if (dims.size() != blobDesc.getDims().size()) + { + reallocate = true; + CV_LOG_WARNING(NULL, "Reallocate output '" << name << "' blob due to wrong dims: " << blobDesc.getDims().size() << " => " << dims.size()); + } + if (reallocate) + { + auto layout = estimateLayout(dims.size()); + w.dataPtr = InferenceEngine::DataPtr(new InferenceEngine::Data(name, + {description.getPrecision(), dims, layout})); + w.blob = reallocateBlob(*w.host, description); + } + return w.dataPtr; +} void forwardNgraph(const std::vector >& outBlobsWrappers, Ptr& node, bool isAsync) @@ -917,6 +997,13 @@ void InfEngineNgraphNet::reset() allBlobs.clear(); infRequests.clear(); isInit = false; + + outputsDesc.clear(); + for (const auto& it : cnn.getOutputsInfo()) + { + const std::string& name = it.first; + outputsDesc.insert({name, it.second->getTensorDesc()}); + } } void InfEngineNgraphNet::addBlobs(const std::vector >& ptrs) @@ -1005,35 +1092,54 @@ void InfEngineNgraphNet::forward(const std::vector >& outBlo reqWrapper->req.SetInput(inpBlobs); reqWrapper->req.SetOutput(outBlobs); +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + InferenceEngine::InferRequest infRequest = reqWrapper->req; + NgraphReqWrapper* wrapperPtr = reqWrapper.get(); + CV_Assert(wrapperPtr && "Internal error"); +#else InferenceEngine::IInferRequest::Ptr infRequestPtr = reqWrapper->req; - infRequestPtr->SetUserData(reqWrapper.get(), 0); + CV_Assert(infRequestPtr); + InferenceEngine::IInferRequest& infRequest = *infRequestPtr.get(); + infRequest.SetUserData(reqWrapper.get(), 0); +#endif - infRequestPtr->SetCompletionCallback( - [](InferenceEngine::IInferRequest::Ptr request, InferenceEngine::StatusCode status) +#if INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + // do NOT capture 'reqWrapper' (smart ptr) in the lambda callback + infRequest.SetCompletionCallback>( + [wrapperPtr](InferenceEngine::InferRequest /*request*/, InferenceEngine::StatusCode status) +#else + infRequest.SetCompletionCallback( + [](InferenceEngine::IInferRequest::Ptr requestPtr, InferenceEngine::StatusCode status) +#endif { CV_LOG_DEBUG(NULL, "DNN(nGraph): completionCallback(" << (int)status << ")"); +#if !INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2021_4) + CV_Assert(requestPtr); + InferenceEngine::IInferRequest& request = *requestPtr.get(); - NgraphReqWrapper* wrapper; - request->GetUserData((void**)&wrapper, 0); - CV_Assert(wrapper && "Internal error"); + NgraphReqWrapper* wrapperPtr; + request.GetUserData((void**)&wrapperPtr, 0); + CV_Assert(wrapperPtr && "Internal error"); +#endif + NgraphReqWrapper& wrapper = *wrapperPtr; size_t processedOutputs = 0; try { - for (; processedOutputs < wrapper->outProms.size(); ++processedOutputs) + for (; processedOutputs < wrapper.outProms.size(); ++processedOutputs) { - const std::string& name = wrapper->outsNames[processedOutputs]; - Mat m = ngraphBlobToMat(wrapper->req.GetBlob(name)); + const std::string& name = wrapper.outsNames[processedOutputs]; + Mat m = ngraphBlobToMat(wrapper.req.GetBlob(name)); try { CV_Assert(status == InferenceEngine::StatusCode::OK); - wrapper->outProms[processedOutputs].setValue(m.clone()); + wrapper.outProms[processedOutputs].setValue(m.clone()); } catch (...) { try { - wrapper->outProms[processedOutputs].setException(std::current_exception()); + wrapper.outProms[processedOutputs].setException(std::current_exception()); } catch(...) { CV_LOG_ERROR(NULL, "DNN: Exception occurred during async inference exception propagation"); } @@ -1043,16 +1149,16 @@ void InfEngineNgraphNet::forward(const std::vector >& outBlo catch (...) { std::exception_ptr e = std::current_exception(); - for (; processedOutputs < wrapper->outProms.size(); ++processedOutputs) + for (; processedOutputs < wrapper.outProms.size(); ++processedOutputs) { try { - wrapper->outProms[processedOutputs].setException(e); + wrapper.outProms[processedOutputs].setException(e); } catch(...) { CV_LOG_ERROR(NULL, "DNN: Exception occurred during async inference exception propagation"); } } } - wrapper->isReady = true; + wrapper.isReady = true; } ); } diff --git a/modules/dnn/src/ie_ngraph.hpp b/modules/dnn/src/ie_ngraph.hpp index 7a8c4bef8d..617f1d4542 100644 --- a/modules/dnn/src/ie_ngraph.hpp +++ b/modules/dnn/src/ie_ngraph.hpp @@ -54,7 +54,8 @@ public: void setNodePtr(std::shared_ptr* ptr); void reset(); -private: + +//private: detail::NetImplBase& netImpl_; void release(); @@ -89,6 +90,8 @@ private: bool hasNetOwner; std::vector requestedOutputs; std::unordered_set> unconnectedNodes; + + std::map outputsDesc; }; class InfEngineNgraphNode : public BackendNode @@ -121,12 +124,17 @@ public: virtual void copyToHost() CV_OVERRIDE; virtual void setHostDirty() CV_OVERRIDE; + Mat* host; InferenceEngine::DataPtr dataPtr; InferenceEngine::Blob::Ptr blob; AsyncArray futureMat; }; InferenceEngine::DataPtr ngraphDataNode(const Ptr& ptr); +InferenceEngine::DataPtr ngraphDataOutputNode( + const Ptr& ptr, + const InferenceEngine::TensorDesc& description, + const std::string name); // This is a fake class to run networks from Model Optimizer. Objects of that // class simulate responses of layers are imported by OpenCV and supported by diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 698168817f..123cb170b7 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -139,6 +139,48 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(FlowWarp, FlowWarpLayer); CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer); + CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer); + CV_DNN_REGISTER_LAYER_CLASS(CumSum, CumSumLayer); + + CV_DNN_REGISTER_LAYER_CLASS(Quantize, QuantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(Dequantize, DequantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(Requantize, RequantizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(ConvolutionInt8, ConvolutionLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(InnerProductInt8, InnerProductLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(PoolingInt8, PoolingLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(EltwiseInt8, EltwiseLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(BatchNormInt8, BatchNormLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ScaleInt8, ScaleLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ShiftInt8, ShiftLayerInt8); + + CV_DNN_REGISTER_LAYER_CLASS(ReLUInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ReLU6Int8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SigmoidInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(TanHInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SwishInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(MishInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(ELUInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(BNLLInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(AbsValInt8, ActivationLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SoftmaxInt8, SoftmaxLayerInt8); + CV_DNN_REGISTER_LAYER_CLASS(SoftMaxInt8, SoftmaxLayerInt8); + + CV_DNN_REGISTER_LAYER_CLASS(ConcatInt8, ConcatLayer); + CV_DNN_REGISTER_LAYER_CLASS(FlattenInt8, FlattenLayer); + CV_DNN_REGISTER_LAYER_CLASS(PaddingInt8, PaddingLayer); + CV_DNN_REGISTER_LAYER_CLASS(BlankInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(DropoutInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(IdentityInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(SilenceInt8, BlankLayer); + CV_DNN_REGISTER_LAYER_CLASS(ConstInt8, ConstLayer); + CV_DNN_REGISTER_LAYER_CLASS(ReshapeInt8, ReshapeLayer); + CV_DNN_REGISTER_LAYER_CLASS(ResizeInt8, ResizeLayer); + CV_DNN_REGISTER_LAYER_CLASS(SplitInt8, SplitLayer); + CV_DNN_REGISTER_LAYER_CLASS(SliceInt8, SliceLayer); + CV_DNN_REGISTER_LAYER_CLASS(CropInt8, CropLayer); + CV_DNN_REGISTER_LAYER_CLASS(PermuteInt8, PermuteLayer); + CV_DNN_REGISTER_LAYER_CLASS(ReorgInt8, ReorgLayer); + CV_DNN_REGISTER_LAYER_CLASS(ShuffleChannelInt8, ShuffleChannelLayer); } CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/int8layers/batch_norm_layer.cpp b/modules/dnn/src/int8layers/batch_norm_layer.cpp new file mode 100644 index 0000000000..c5b8c3d9e9 --- /dev/null +++ b/modules/dnn/src/int8layers/batch_norm_layer.cpp @@ -0,0 +1,178 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include + +namespace cv +{ +namespace dnn +{ + +class BatchNormLayerInt8Impl CV_FINAL : public BatchNormLayerInt8 +{ +public: + Mat origin_weights, origin_bias; + Mat weights_, bias_; + mutable int dims; + + BatchNormLayerInt8Impl(const LayerParams& params) + : dims(-1) + { + setParamsFrom(params); + useGlobalStats = params.get("use_global_stats", true); + input_sc = params.get("input_scale"); + input_zp = params.get("input_zeropoint"); + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + + CV_Assert(blobs.size() == 2); + size_t n = blobs[0].total(); + CV_Assert(blobs[1].total() == n && + blobs[0].isContinuous() && blobs[1].isContinuous() && + blobs[0].type() == CV_32F && blobs[1].type() == CV_32F); + + origin_weights = blobs[0]; + origin_bias = blobs[1]; + } + + virtual void finalize(InputArrayOfArrays, OutputArrayOfArrays) CV_OVERRIDE + { + origin_weights.convertTo(weights_, CV_32F, input_sc/output_sc); + addWeighted(origin_bias, 1.0/output_sc, weights_, -input_zp, output_zp, bias_, CV_32F); + } + + void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + scale = origin_weights; + shift = origin_bias; + } + + void getScaleZeropoint(float& scale, int& zeropoint) const CV_OVERRIDE + { + scale = output_sc; + zeropoint = output_zp; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Mat w_, b_; + top->getScaleShift(w_, b_); + if (w_.empty() && b_.empty()) + return false; + + const int numChannels = weights_.total(); + const int numFusedWeights = w_.total(); + const int numFusedBias = b_.total(); + + if ((numFusedWeights != numChannels && numFusedWeights != 1 && !w_.empty()) || + (numFusedBias != numChannels && numFusedBias != 1 && !b_.empty())) + return false; + + float new_sc; + int new_zp; + top->getScaleZeropoint(new_sc, new_zp); + + Mat w = numFusedWeights == 1 ? Mat(1, numChannels, CV_32F, Scalar(w_.at(0))) : + (w_.empty() ? Mat::ones(1, numChannels, CV_32F) : w_.reshape(1, 1)); + + Mat b = numFusedBias == 1 ? Mat(1, numChannels, CV_32F, Scalar(b_.at(0))) : + (b_.empty() ? Mat::zeros(1, numChannels, CV_32F) : b_.reshape(1, 1)); + + weights_ = Mat(); bias_ = Mat(); + multiply(origin_weights, w, weights_, input_sc/new_sc, CV_32F); + multiply(origin_bias, w, bias_); + add(bias_, b, bias_); + addWeighted(bias_, 1.0/new_sc, weights_, -input_zp, new_zp, bias_, CV_32F); + return true; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + dims = inputs[0].size(); + if (!useGlobalStats && inputs[0][0] != 1) + CV_Error(Error::StsNotImplemented, "Batch normalization in training mode with batch size > 1"); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(blobs.size() == 2); + CV_Assert(inputs.size() == 1); + + Mat &inpBlob = inputs[0]; + int planeSize = 1; + for (size_t i = 2; i < inpBlob.dims; i++) { + planeSize *= inpBlob.size[i]; + } + + for (size_t ii = 0; ii < outputs.size(); ii++) + { + Mat &outBlob = outputs[ii]; + + for(int num = 0; num < outBlob.size[0]; num++) + { + for (int n = 0; n < outBlob.size[1]; n++) + { + float w = weights_.at(n); + float b = bias_.at(n); + Mat inpBlobPlane(1, planeSize, CV_8S, inpBlob.ptr(num, n)); + Mat outBlobPlane(1, planeSize, CV_8S, outBlob.ptr(num, n)); + inpBlobPlane.convertTo(outBlobPlane, CV_8S, w, b); + } + } + } + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + + int64 flops = 0; + for(int i = 0; i < inputs.size(); i++) + { + flops += 3*total(inputs[i]); + } + return flops; + } + +private: + bool useGlobalStats; +}; + +Ptr BatchNormLayerInt8::create(const LayerParams& params) +{ + return Ptr(new BatchNormLayerInt8Impl(params)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/int8layers/convolution_layer.cpp b/modules/dnn/src/int8layers/convolution_layer.cpp new file mode 100644 index 0000000000..ea29610222 --- /dev/null +++ b/modules/dnn/src/int8layers/convolution_layer.cpp @@ -0,0 +1,1142 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +#include "opencv2/core/hal/hal.hpp" +#include "opencv2/core/hal/intrin.hpp" +#include +#include + +namespace cv +{ +namespace dnn +{ + +#if CV_SIMD +static inline void v_expand_mul_add(const v_int8x16& a, const v_int8x16& b, + v_int32x4& out0, v_int32x4& out1, v_int32x4& out2, v_int32x4& out3) +{ + v_int16x8 a0, a1, b0, b1; + v_expand(a, a0, a1); + v_expand(b, b0, b1); + + v_int32x4 t0, t1; + v_mul_expand(a0, b0, t0, t1); + out0 += t0; out1 += t1; + + v_mul_expand(a1, b1, t0, t1); + out2 += t0; out3 += t1; +} +#endif + +class BaseConvolutionLayerInt8Impl : public ConvolutionLayerInt8 +{ +public: + BaseConvolutionLayerInt8Impl(const LayerParams ¶ms) + { + setParamsFrom(params); + getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads); + + numOutput = params.get("num_output"); + int ngroups = params.get("group", 1); + CV_Assert(numOutput % ngroups == 0); + + input_zp = params.get("input_zeropoint"); + output_zp = params.get("zeropoints"); + output_sc = params.get("scales"); + + if (kernel_size.size() == 2) { + kernel = Size(kernel_size[1], kernel_size[0]); + stride = Size(strides[1], strides[0]); + for (int i = 0; i < pads_begin.size(); i++) { + if (pads_begin[i] != pads_end[i]) + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer"); + } + pad = Size(pads_begin[1], pads_begin[0]); + dilation = Size(dilations[1], dilations[0]); + + adjustPad.height = adjust_pads[0]; + adjustPad.width = adjust_pads[1]; + } + + for (int i = 0; i < adjust_pads.size(); i++) { + CV_Assert(adjust_pads[i] < strides[i]); + } + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + // blobs[0] - Weights (INT8) + // blobs[1] - Biases (INT32) + // blobs[2] - Multipliers for convolution output stage (FP32) + CV_Assert(!inputs.empty() && blobs.size() == 3); + MatSize weightShape = blobs[0].size; + + CV_Assert(inputs[0].dims == outputs[0].dims); + if (weightShape.dims() == 3) + { + kernel_size.assign(1, kernel_size[0]); + strides.assign(1, strides[0]); + dilations.assign(1, dilations[0]); + pads_begin.assign(1, pads_begin[0]); + pads_end.assign(1, pads_end[0]); + } + CV_Assert(weightShape.dims() == kernel_size.size() + 2); + for (int i = 0; i < kernel_size.size(); i++) { + CV_Assert(weightShape[i + 2] == kernel_size[i]); + } + + const Mat &input = inputs[0]; + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || input.dims == 4 || input.dims == 5) && input.type() == CV_8S); + for (size_t i = 0; i < outputs.size(); i++) + { + CV_Assert(inputs[i].type() == input.type()); + CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || inputs[i].dims == 4 || inputs[i].dims == 5) && inputs[i].size[1] == input.size[1]); + for (int j = 0; j < inputs[i].dims; j++) { + CV_Assert(inputs[i].size[j] == input.size[j]); + } + } + + std::vector inpShape; + std::vector outShape; + for (int i = 2; i < inputs[0].dims; i++) { + inpShape.push_back(inputs[0].size[i]); + outShape.push_back(outputs[0].size[i]); + } + getConvPoolPaddings(inpShape, kernel_size, strides, padMode, pads_begin, pads_end); + if (pads_begin.size() == 2) { + for (int i = 0; i < pads_begin.size(); i++) { + if (pads_begin[i] != pads_end[i]) + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding in convolution layer"); + } + pad = Size(pads_begin[1], pads_begin[0]); + } + } + + virtual MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const = 0; + bool is1x1() const + { + return (kernel.height == 1 && kernel.width == 1) && + (stride.height == 1 && stride.width == 1) && + (dilation.height == 1 && dilation.width == 1); + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Mat w, b; + top->getScaleShift(w, b); + if (w.empty() && b.empty()) + return false; + + CV_Assert((w.empty() || w.type() == CV_32F) && + (b.empty() || b.type() == CV_32F)); + + float new_sc; + int new_zp; + top->getScaleZeropoint(new_sc, new_zp); + fuseWeights(w, b, new_sc); + output_sc = new_sc; + output_zp = new_zp; + return true; + } + + virtual void fuseWeights(const Mat& w_, const Mat& b_, const float& new_sc) = 0; +}; + +//TODO: simultaneously convolution and bias addition for cache optimization +class ConvolutionLayerInt8Impl CV_FINAL : public BaseConvolutionLayerInt8Impl +{ +public: + enum { VEC_ALIGN = 32, DFT_TYPE = CV_8S }; + Mat weightsMat; + std::vector biasvec; + std::vector outputMultiplier; + Mat activationLUT; + Ptr activ; + + ConvolutionLayerInt8Impl(const LayerParams ¶ms) : BaseConvolutionLayerInt8Impl(params){} + + MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const CV_OVERRIDE + { + CV_Assert(!blobs.empty()); + int dims = inpShape.size(); + int inpD = dims == 5 ? inpShape[2] : 1; + int inpH = inpShape[dims - 2]; + int inpW = inpShape.back(); + int inpGroupCn = blobs[0].size[1]; + int ksize = inpGroupCn * std::accumulate(kernel_size.begin(), kernel_size.end(), + 1, std::multiplies()); + return shape(inpD * inpH * inpW, ksize); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + size_t ksize = kernel_size.size(); + // Only default backend and Conv1D/Conv2D/Conv3D are supported + return backendId == DNN_BACKEND_OPENCV && ksize >= 1 && ksize <= 3; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(!blobs.empty()); + const int* weightShape = blobs[0].size.p; + CV_Assert(blobs[1].total() == (size_t)weightShape[0]); + + internals.clear(); + + CV_Assert(inputs.size() != 0); + std::vector inpShape(inputs[0].begin() + 2, inputs[0].end()); + + int outCn = weightShape[0]; + std::vector outShape; + outShape.push_back(inputs[0][0]); + outShape.push_back(outCn); + + int inpCn = inputs[0][1]; + if (padMode.empty()) + { + for (int i = 0; i < inpShape.size(); i++) + outShape.push_back((inpShape[i] + pads_begin[i] + pads_end[i] - dilations[i] * (kernel_size[i] - 1) - 1) / strides[i] + 1); + } + else + { + getConvPoolOutParams(inpShape, kernel_size, strides, padMode, dilations, outShape); + } + + int ngroups = inpCn / weightShape[1]; + if (ngroups == 0 || ngroups * weightShape[1] != inpCn) + CV_Error(Error::StsError, format("Number of input channels should " + "be multiple of %d but got %d", weightShape[1], inpCn)); + CV_Assert(ngroups > 0 && inpCn % ngroups == 0 && outCn % ngroups == 0); + + outputs.resize(1, outShape); + + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + BaseConvolutionLayerInt8Impl::finalize(inputs_arr, outputs_arr); + + std::vector inputs; + inputs_arr.getMatVector(inputs); + // prepare weightsMat where each row is aligned and has enough zero padding on the right to + // use vectorized (i.e. with intrinsics) loops without tail processing + Mat wm = blobs[0].reshape(1, numOutput); + if( wm.step1() % VEC_ALIGN != 0 ) + { + int newcols = (int)alignSize(wm.step1(), VEC_ALIGN); + Mat wm_buffer = Mat(numOutput, newcols, wm.type()); + Mat wm_padding = wm_buffer.colRange(wm.cols, newcols); + wm_padding.setTo(Scalar::all(0)); + Mat wm_aligned = wm_buffer.colRange(0, wm.cols); + wm.copyTo(wm_aligned); + wm = wm_aligned; + } + weightsMat = wm; + + Mat biasMat = blobs[1]; + biasvec.resize(numOutput+2); + + Mat outMult = blobs[2]; + outputMultiplier.resize(numOutput+2); + for(int i = 0; i < numOutput; i++ ) + { + biasvec[i] = biasMat.at(i); + outputMultiplier[i] = outMult.at(i); + } + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activ_int8->blobs[0].convertTo(activationLUT, CV_32S); + return true; + } + return false; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + return BaseConvolutionLayerInt8Impl::tryFuse(top); + } + + void fuseWeights(const Mat& w_, const Mat& b_, const float& new_sc) CV_OVERRIDE + { + const int outCn = weightsMat.size[0]; + Mat w = w_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(w_.at(0))) : w_; + Mat b = b_.total() == 1 ? Mat(1, outCn, CV_32F, Scalar(b_.at(0))) : b_; + CV_Assert_N(!weightsMat.empty(), biasvec.size() == outCn + 2, + w.empty() || outCn == w.total(), b.empty() || outCn == b.total()); + + for (int i = 0; i < outCn; ++i) + { + float off = outputMultiplier[i] * output_sc; + if (!w.empty()) + off *= w.at(i); + + if (!b.empty()) + biasvec[i] += (int)std::round(b.at(i)/off); + + outputMultiplier[i] = off/new_sc; + } + biasvec[outCn] = biasvec[outCn+1] = biasvec[outCn-1]; + outputMultiplier[outCn] = outputMultiplier[outCn+1] = outputMultiplier[outCn-1]; + } + + class ParallelConv : public cv::ParallelLoopBody + { + public: + enum { BLK_SIZE = 32, BLK_SIZE_CN = 64 }; + + const Mat* input_; + const Mat* weights_; + Mat* output_; + int outShape[4]; // used only for conv2d + std::vector kernel_size, pads_begin, pads_end, strides, dilations; + int ngroups_, nstripes_; + std::vector ofstab_; + const std::vector* biasvec_; + const Mat* activLUT_; + const ActivationLayerInt8* activ_; + bool is1x1_; + bool useAVX2; + bool useAVX512; + int blk_size_cn; + int inpZp, outZp; + const std::vector* multiplier; + + ParallelConv() + : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), + biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false) + , blk_size_cn(0), inpZp(0), outZp(0), multiplier(0) + {} + + static void run( const Mat& input, Mat& output, const Mat& weights, const std::vector& multipliers, + const std::vector& biasvec, const Mat& activLUT, + const std::vector& kernel_size, const std::vector& strides, + const std::vector& pads_begin, const std::vector& pads_end, + const std::vector& dilations, + const ActivationLayerInt8* activ, int ngroups, int nstripes, int inp_Zp, int out_Zp) + { + size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(), + 1, std::multiplies()); + bool isConv1D = input.dims == 3; + bool isConv2D = input.dims == 4; + bool isConv3D = input.dims == 5; + CV_CheckEQ(static_cast(kernel_size.size()), input.dims - 2, ""); + CV_Assert_N(input.dims == output.dims, + input.size[0] == output.size[0], + weights.rows == output.size[1], + weights.cols == (input.size[1]/ngroups)*karea, + input.type() == CV_8SC1, + output.type() == CV_32SC1, + input.type() == weights.type(), + input.isContinuous(), + output.isContinuous(), + biasvec.size() == (size_t)output.size[1]+2); + CV_Check(weights.step1(), weights.step1() % VEC_ALIGN == 0, ""); + ParallelConv p; + + p.input_ = &input; + p.weights_ = &weights; + p.output_ = &output; + int max_ind = isConv1D? 3: 4; + for( int i = 0; i < max_ind; i++ ) p.outShape[i] = output.size[i]; + p.outShape[1] /= ngroups; + + p.kernel_size = kernel_size; p.strides = strides; p.dilations = dilations; + p.pads_begin = pads_begin; p.pads_end = pads_end; + + p.ngroups_ = ngroups; + p.nstripes_ = nstripes; + + int inpCnAll = input.size[1]; + int depth = (input.dims == 5) ? input.size[2] : 1; + int width = input.size[input.dims - 1]; + int height = isConv1D? 1 : input.size[input.dims - 2]; + int inpCn = inpCnAll / ngroups; + + p.is1x1_ = (isConv2D && kernel_size[0] == 1 && kernel_size[1] == 1 && + pads_begin[0] == 0 && pads_begin[1] == 0) || + (isConv1D && pads_begin[0] == 0 && kernel_size[0] == 1); + + p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D; + p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D; + + int kernel_d = isConv3D? kernel_size[0] : 1; + int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + int blk_size_cn0 = cvCeil(1600./(kernel_w*kernel_h)); + int ncn = 32; + while (ncn*2 < blk_size_cn0 && ncn < inpCn) + ncn *= 2; + ncn = std::min(ncn, inpCn); + p.blk_size_cn = ncn; + + int dil_d = isConv3D? dilations[0] : 1; + int dil_h = isConv1D? 1 : dilations[dilations.size() - 2]; + int dil_w = dilations.back(); + + p.inpZp = inp_Zp; + p.outZp = out_Zp; + p.multiplier = &multipliers; + + p.ofstab_.resize(karea * ncn); + int* ofstab = &p.ofstab_[0]; + + if (isConv1D) + { + for( int k = 0; k < ncn; k++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[k*kernel_w + k_c] = k*width + k_c*dil_w; + } + else if (isConv2D) + { + for( int k = 0; k < ncn; k++ ) + for( int k_r = 0; k_r < kernel_h; k_r++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[(k*kernel_h + k_r)*kernel_w + k_c] = + (k*height + k_r*dil_h)*width + k_c*dil_w; + } + else + { + for( int k = 0; k < ncn; k++ ) + for (int k_d = 0; k_d < kernel_d; k_d++) + for( int k_r = 0; k_r < kernel_h; k_r++ ) + for( int k_c = 0; k_c < kernel_w; k_c++ ) + ofstab[(k*kernel_d*kernel_h + k_d*kernel_h + k_r)*kernel_w + k_c] = + (k*depth*height + k_d*dil_d*height + k_r*dil_h)*width + k_c*dil_w; + } + + p.biasvec_ = &biasvec; + p.activLUT_ = &activLUT; + p.activ_ = !activLUT.empty() ? activ : 0; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + virtual void operator ()(const Range &r0) const CV_OVERRIDE + { + const int valign = ConvolutionLayerInt8Impl::VEC_ALIGN; + int ngroups = ngroups_, batchSize = input_->size[0]*ngroups; + bool isConv1D = input_->dims == 3; + bool isConv2D = input_->dims == 4; + bool isConv3D = input_->dims == 5; + + int outW = output_->size[output_->dims - 1]; + int outH = isConv1D? 1 : output_->size[output_->dims - 2]; + int outCn = output_->size[1]/ngroups; + + int depth = isConv3D? input_->size[2] : 1; + int height = isConv1D? 1 : input_->size[input_->dims - 2]; + int width = input_->size[input_->dims - 1]; + int inpCn = input_->size[1]/ngroups; + + const int nstripes = nstripes_; + + int kernel_d = isConv3D? kernel_size[0] : 1; + int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + int karea = kernel_w*kernel_h*kernel_d; + + int pad_d = isConv3D? pads_begin[0] : 0; + int pad_t = isConv1D? 0 : pads_begin[pads_begin.size() - 2]; + int pad_l = pads_begin.back(); + + int stride_d = isConv3D? strides[0] : 0; + int stride_h = isConv1D? 0 : strides[strides.size() - 2]; + int stride_w = strides.back(); + + int dilation_d = isConv3D? dilations[0] : 1; + int dilation_h = isConv1D? 1 : dilations[dilations.size() - 2]; + int dilation_w = dilations.back(); + + int i, j, k, d; + int inpPlaneSize = (int)input_->total(2); + int outPlaneSize = (int)output_->total(2); + bool is1x1 = is1x1_; + + int stripesPerSample; + int stripeSize; + Range r = r0; + bool depthWiseConvolution = !is1x1 && isConv2D && ngroups > 1 && inpCn == 1 && + outCn == 1 && kernel_d == 1 && dilation_d == 1 && stride_d == 0 && pad_d == 0 && + width >= 16 + dilation_w*(kernel_w - 1); + // for now only 3x3 depth-wise convolutions are supported + depthWiseConvolution = depthWiseConvolution && kernel_w == 3 && kernel_h == 3 && + // computing at most 1 pixel from each side can involve padding + max(stride_w, dilation_w) >= pad_l && max(stride_h, dilation_h) >= pad_t && + pad_l <= 1 && pad_t <= 1; + + if( !depthWiseConvolution && nstripes >= batchSize*2 ) + { + stripesPerSample = nstripes/batchSize; + stripeSize = (int)alignSize((outPlaneSize + stripesPerSample - 1)/stripesPerSample, 8); + stripeSize = std::min(stripeSize, outPlaneSize); + } + else + { + stripesPerSample = 1; + int samplesPerStripe = std::max((batchSize + nstripes - 1)/nstripes, 1); + r.start *= samplesPerStripe; + r.end *= samplesPerStripe; + stripeSize = outPlaneSize; + } + + const int8_t* data_inp0_ = input_->ptr(); + const int* ofstab = &ofstab_[0]; + const int8_t* wptr_orig_ = weights_->ptr(); + size_t wstep = weights_->step1(); + const int* biasptr_ = &biasvec_->at(0); + const float* multptr_ = &multiplier->at(0); + const int* lutptr_ = !activLUT_->empty() ? activLUT_->ptr() : 0; + int* data_out0_ = output_->ptr(); + AutoBuffer rowbuf0_; + int8_t* rowbuf0 = 0; + bool use_rowbuf = !depthWiseConvolution; + int blk_size = depthWiseConvolution ? outPlaneSize : min((int)BLK_SIZE, stripeSize); + + // im2row buffer is not used for depth-wise convolution + if(use_rowbuf) + { + size_t rowbufsz = alignSize(karea*blk_size_cn, valign)*min((int)BLK_SIZE, blk_size); + //printf("karea=%d, blk_size_cn=%d, rowbufsz=%d, stripeSize=%d\n", karea, blk_size_cn, (int)rowbufsz, stripeSize); + rowbuf0_.allocate(rowbufsz + valign); + rowbuf0 = alignPtr(rowbuf0_.data(), (int)(valign*sizeof(int8_t))); + // we clear the buffer once; ultimately, it lets us to avoid + // tail processing after running the unrolled/vectorized loop. + // the main idea is to make sure that the tail (a.k.a. padding) of each row + // (i.e. the elements with indices between vsz=karea*ncn and vsz_a) + // does not contain NaNs or Infs. Because the padding in the weights + // matrix is explicitly initialized with 0's, we handle all other + // cases nicely, i.e. we can skip expliciting re-initialization + // of the padding - we just retain elements from the previous iteration + // of the loop over channels (cn0). + memset(rowbuf0, (int8_t)inpZp, rowbufsz*sizeof(rowbuf0[0]) ); + } + + for( int stripe = r.start; stripe < r.end; stripe++ ) + { + int subsampleIdx = stripe/stripesPerSample; + if( subsampleIdx >= batchSize ) + break; + int stripeStart = (int)((stripe - subsampleIdx*stripesPerSample)*stripeSize); + int stripeEnd = (int)std::min(stripeStart + stripeSize, outPlaneSize); + const int8_t* data_inp0 = data_inp0_ + subsampleIdx*inpPlaneSize*inpCn; + int* data_out0 = data_out0_ + subsampleIdx*outPlaneSize*outCn; + int startOutCn = (subsampleIdx % ngroups)*outCn; + const int8_t* wptr_orig = wptr_orig_ + wstep*startOutCn; + const int* biasptr = biasptr_ + startOutCn; + const float* multptr = multptr_ + startOutCn; + + for( int cn0 = 0; cn0 < inpCn; cn0 += blk_size_cn ) + { + int cn1 = std::min(cn0 + blk_size_cn, inpCn); + int ncn = cn1 - cn0, vsz = karea*ncn; + int vsz_a = (int)alignSize(vsz, valign); + const int8_t* wptr = wptr_orig + cn0*karea; + + for( int ofs0 = stripeStart; ofs0 < stripeEnd; ofs0 += blk_size ) + { + int ofs, ofs1 = std::min(ofs0 + blk_size, stripeEnd); + int bsz = ofs1 - ofs0; + + int out_d = ofs0 / (outH * outW); + int out_i = (ofs0 - out_d * outH * outW) / outW; + int out_j = ofs0 % outW; + + if (depthWiseConvolution) + { + CV_Assert(out_i == 0 && out_j == 0); + int in_d = out_d * stride_d - pad_d; + const int8_t* inptr_ = data_inp0 + (cn0*depth*height + in_d*height)*width; + int* outptr_ = data_out0 + ofs0; + + #if CV_TRY_AVX2 + if(useAVX2) + opt_AVX2::fastDepthwiseConv(wptr, kernel_h, kernel_w, + stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, + biasptr, multptr, inptr_, height, width, outptr_, out_d, outH, outW, inpZp, outZp); + else + #endif + { + const int8_t w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + int bias = biasptr[out_d], biasCopy; + float mult = multptr[out_d]; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const int8_t* imgptr0 = inptr_ + in_i*width; + const int8_t* imgptr1 = imgptr0 + dilation_h*width; + const int8_t* imgptr2 = imgptr0 + (dilation_h*2)*width; + int8_t w00 = w00_, w01 = w01_, w02 = w02_; + int8_t w20 = w20_, w21 = w21_, w22 = w22_; + int out, out1; + // Bias has a fused offset component. bias = bias_quantized - input_zeropoint*sum_of_weights. + // In some cases below, certain weights are not used for convolution or set to zero. + // So we create a copy of bias at the start and remove the weight's components as necessary. + biasCopy = bias; + + if (in_i < 0) + { + biasCopy += inpZp * (w00 + w01 + w02); + w00 = w01 = w02 = 0; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + biasCopy += inpZp * (w20 + w21 + w22); + w20 = w21 = w22 = 0; + imgptr2 = imgptr1; + } + int* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = (int)imgptr0[0]*w01 + (int)imgptr0[dilation_w]*w02 + + (int)imgptr1[0]*w11 + (int)imgptr1[dilation_w]*w12 + + (int)imgptr2[0]*w21 + (int)imgptr2[dilation_w]*w22 + + biasCopy + inpZp*(w00 + w10 + w20); + out1 = outZp + (int)std::round(out*mult); + outptr[0] = std::min(std::max(out1, -128), 127); + out_j = 1; + } + #if CV_SIMD + if( stride_w == 1 ) + { + const int out_delta = 16; + v_int8x16 vw00 = v_setall_s8(w00), vw01 = v_setall_s8(w01), vw02 = v_setall_s8(w02), + vw10 = v_setall_s8(w10), vw11 = v_setall_s8(w11), vw12 = v_setall_s8(w12), + vw20 = v_setall_s8(w20), vw21 = v_setall_s8(w21), vw22 = v_setall_s8(w22); + v_int32x4 vout0, vout1, vout2, vout3, vbias = v_setall_s32(biasCopy), voutzp = v_setall_s32(outZp), + outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_float32x4 vmult = v_setall_f32(mult); + for( ; out_j < outW1; out_j += out_delta ) + { + if (out_j + out_delta > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - out_delta; + } + int in_j = out_j * stride_w - pad_l; + v_int8x16 v00 = v_load(imgptr0 + in_j), + v01 = v_load(imgptr0 + in_j + dilation_w), + v02 = v_load(imgptr0 + in_j + dilation_w*2), + v10 = v_load(imgptr1 + in_j), + v11 = v_load(imgptr1 + in_j + dilation_w), + v12 = v_load(imgptr1 + in_j + dilation_w*2), + v20 = v_load(imgptr2 + in_j), + v21 = v_load(imgptr2 + in_j + dilation_w), + v22 = v_load(imgptr2 + in_j + dilation_w*2); + + vout0 = vout1 = vout2 = vout3 = vbias; + v_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + v_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + v_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + v_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + v_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + v_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + v_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + v_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + v_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = voutzp + v_round(v_cvt_f32(vout0)*vmult); + vout1 = voutzp + v_round(v_cvt_f32(vout1)*vmult); + vout2 = voutzp + v_round(v_cvt_f32(vout2)*vmult); + vout3 = voutzp + v_round(v_cvt_f32(vout3)*vmult); + + vout0 = v_min(v_max(vout0, outmin), outmax); + vout1 = v_min(v_max(vout1, outmin), outmax); + vout2 = v_min(v_max(vout2, outmin), outmax); + vout3 = v_min(v_max(vout3, outmin), outmax); + + v_store(outptr + out_j, vout0); + v_store(outptr + out_j + 4, vout1); + v_store(outptr + out_j + 8, vout2); + v_store(outptr + out_j + 12, vout3); + } + } + #endif + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = (int)imgptr0[in_j]*w00 + (int)imgptr0[in_j + dilation_w]*w01 + (int)imgptr0[in_j + dilation_w*2]*w02 + + (int)imgptr1[in_j]*w10 + (int)imgptr1[in_j + dilation_w]*w11 + (int)imgptr1[in_j + dilation_w*2]*w12 + + (int)imgptr2[in_j]*w20 + (int)imgptr2[in_j + dilation_w]*w21 + (int)imgptr2[in_j + dilation_w*2]*w22 + biasCopy; + out1 = outZp + (int)std::round(out*mult); + outptr[out_j] = std::min(std::max(out1, -128), 127); + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + int s0 = 1, s1 = 1, s2 = 1; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0; + biasCopy += inpZp*(w00 + w10 + w20); + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0; + biasCopy += inpZp*(w01 + w11 + w21); + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0; + biasCopy += inpZp*(w02 + w12 + w22); + } + out = (int)imgptr0[in_j0]*w00*s0 + (int)imgptr0[in_j1]*w01*s1 + (int)imgptr0[in_j2]*w02*s2 + + (int)imgptr1[in_j0]*w10*s0 + (int)imgptr1[in_j1]*w11*s1 + (int)imgptr1[in_j2]*w12*s2 + + (int)imgptr2[in_j0]*w20*s0 + (int)imgptr2[in_j1]*w21*s1 + (int)imgptr2[in_j2]*w22*s2 + biasCopy; + out1 = outZp + (int)std::round(out*mult); + outptr[out_j] = std::min(std::max(out1, -128), 127); + } + } + } + continue; + } + // do im2row for a part of input tensor + int8_t* rowbuf = rowbuf0; + + if (isConv1D) + { + for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + cn0*width + in_j; + ofs += delta; + + // do im2row for a part of input tensor + if( is1x1 ) + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w ) + { + for( k = 0; k < vsz; k++ ) + rowbuf[k] = imgptr[k*inpPlaneSize]; + } + } + else + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + // this condition should be true for most of the tensor elements, i.e. + // most of the time the kernel aperture is inside the tensor X-Y plane. + if( out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w ) + { + for( k = 0; k < vsz; k++ ) + { + int k1 = ofstab[k]; + int8_t v0 = imgptr[k1]; + int8_t v1 = imgptr[k1 + stride_w]; + rowbuf[k] = v0; + rowbuf[k+vsz_a] = v1; + } + out_j++; + rowbuf += vsz_a; + imgptr += stride_w; + in_j += stride_w; + } + else + { + int i0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int i1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for( i = i0; i < i1; i++ ) + { + int imgofs = k*width + i*dilation_w; + rowbuf[k*kernel_w + i] = imgptr[imgofs]; + } + } + } + } + } + } + } + else if (isConv2D) + { + if( is1x1 && stride_w == 1 && stride_h == 1 ) + { + const int8_t* imgptr = data_inp0 + (cn0*height + out_i)*width + out_j; + for( int j = 0; j < bsz; j++, rowbuf += vsz_a ) + { + if( j + 4 <= bsz ) + { + k = 0; + for( ; k < vsz; k++ ) + { + const int8_t* inp = imgptr + j + k*inpPlaneSize; + int8_t v0 = inp[0], v1 = inp[1], v2 = inp[2], v3 = inp[3]; + rowbuf[k] = v0; + rowbuf[k + vsz_a] = v1; + rowbuf[k + vsz_a*2] = v2; + rowbuf[k + vsz_a*3] = v3; + } + j += 3; + rowbuf += vsz_a*3; + } + else + { + for( k = 0; k < vsz; k++ ) + { + rowbuf[k] = imgptr[j + k*inpPlaneSize]; + } + } + } + } + else + for( ofs = ofs0; ofs < ofs1; out_j = 0, ++out_i ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_i = out_i * stride_h - pad_t; + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + (cn0*height + in_i)*width + in_j; + ofs += delta; + + // do im2row for a part of input tensor + if( is1x1 ) + { + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w ) + { + for( k = 0; k < vsz; k++ ) + rowbuf[k] = imgptr[k*inpPlaneSize]; + } + } + else + { + bool ok_i = 0 <= in_i && in_i < height - (kernel_h-1)*dilation_h; + int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h); + int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h); + + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + // this condition should be true for most of the tensor elements, i.e. + // most of the time the kernel aperture is inside the tensor X-Y plane. + if( ok_i && out_j + 2 <= out_j1 && 0 <= in_j && in_j + stride_w*2 <= width - (kernel_w-1)*dilation_w ) + { + for( k = 0; k < vsz; k++ ) + { + int k1 = ofstab[k]; + int8_t v0 = imgptr[k1]; + int8_t v1 = imgptr[k1 + stride_w]; + rowbuf[k] = v0; + rowbuf[k+vsz_a] = v1; + } + out_j++; + rowbuf += vsz_a; + imgptr += stride_w; + in_j += stride_w; + } + else + { + int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for( i = i0; i < i1; i++ ) + { + for( j = j0; j < j1; j++ ) + { + int imgofs = k*(width*height) + i*(dilation_h*width) + j*dilation_w; + rowbuf[(k*kernel_h + i)*kernel_w + j] = imgptr[imgofs]; + } + } + } + } + } + } + } + } + else + { + for( ofs = ofs0; ofs < ofs1; out_d += (out_i + 1) / outH, out_i = (out_i + 1) % outH, out_j = 0 ) + { + int delta = std::min(ofs1 - ofs, outW - out_j); + int out_j1 = out_j + delta; + + int in_d = out_d * stride_d - pad_d; + int in_i = out_i * stride_h - pad_t; + int in_j = out_j * stride_w - pad_l; + const int8_t* imgptr = data_inp0 + (cn0*depth*height + in_d*height + in_i)*width + in_j; + ofs += delta; + + int d0 = std::max(0, (-in_d + dilation_d - 1) / dilation_d); + int d1 = std::min(kernel_d, (depth - in_d + dilation_d - 1) / dilation_d); + + int i0 = std::max(0, (-in_i + dilation_h-1)/dilation_h); + int i1 = std::min(kernel_h, (height - in_i + dilation_h-1)/dilation_h); + + for( ; out_j < out_j1; out_j++, rowbuf += vsz_a, imgptr += stride_w, in_j += stride_w ) + { + int j0 = std::max(0, (-in_j + dilation_w-1)/dilation_w); + int j1 = std::min(kernel_w, (width - in_j + dilation_w-1)/dilation_w); + + // here some non-continuous sub-row of the row will not be + // filled from the tensor; we need to make sure that the uncovered + // elements are explicitly set to 0's. the easiest way is to + // set all the elements to 0's before the loop. + memset(rowbuf, (int8_t)inpZp, vsz*sizeof(rowbuf[0])); + for( k = 0; k < ncn; k++ ) + { + for ( d = d0; d < d1; d++) + { + for( i = i0; i < i1; i++ ) + { + for( j = j0; j < j1; j++ ) + { + int imgofs = k*(depth*width*height) + d*dilation_d*width*height + i*(dilation_h*width) + j*dilation_w; + rowbuf[(k*kernel_d*kernel_h + d*kernel_h + i)*kernel_w + j] = imgptr[imgofs]; + } + } + } + } + } + } + } + // now compute dot product of the weights + // and im2row-transformed part of the tensor + #if CV_TRY_AVX512_SKX + if(useAVX512) + opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn); + else + #endif + #if CV_TRY_AVX2 + if(useAVX2) + opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn); + else + #endif + for( int i = 0; i < outCn; i += 2 ) + { + const int8_t* wptr0 = wptr + i*wstep; + const int8_t* wptr1 = wptr0 + wstep; + int* outptr0 = data_out0 + ofs0 + i*outPlaneSize; + int* outptr1 = outptr0 + outPlaneSize; + int bias0 = biasptr[i], bias1 = biasptr[i+1]; + float mult0 = multptr[i], mult1 = multptr[i+1]; + + if( i+1 >= outCn ) + { + wptr1 = wptr0; + outptr1 = outptr0; + bias1 = bias0; + mult1 = mult0; + } + int j = 0; + #if CV_SIMD128 + v_int32x4 voutzp = v_setall_s32(outZp), outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_float32x4 vmult0 = v_setall_f32(mult0), vmult1 = v_setall_f32(mult1); + for( ; j <= bsz - 4; j += 4 ) + { + const int8_t* rptr = rowbuf0 + j*vsz_a; + v_int32x4 s0, s1; + + if( cn0 == 0 ) + { + s0 = v_setall_s32(bias0); + s1 = v_setall_s32(bias1); + } + else + { + s0 = v_load(outptr0 + j); + s1 = v_load(outptr1 + j); + } + + v_int32x4 vs00 = v_setzero_s32(), vs01 = v_setzero_s32(), + vs02 = v_setzero_s32(), vs03 = v_setzero_s32(), + vs10 = v_setzero_s32(), vs11 = v_setzero_s32(), + vs12 = v_setzero_s32(), vs13 = v_setzero_s32(); + for( k = 0; k < vsz; k += 16, rptr += 16 ) + { + v_int8x16 w0 = v_load_aligned(wptr0 + k); + v_int8x16 w1 = v_load_aligned(wptr1 + k); + v_int8x16 r0 = v_load_aligned(rptr); + v_int8x16 r1 = v_load_aligned(rptr + vsz_a); + v_int8x16 r2 = v_load_aligned(rptr + vsz_a*2); + v_int8x16 r3 = v_load_aligned(rptr + vsz_a*3); + + vs00 = v_dotprod_expand_fast(w0, r0, vs00); + vs01 = v_dotprod_expand_fast(w0, r1, vs01); + vs02 = v_dotprod_expand_fast(w0, r2, vs02); + vs03 = v_dotprod_expand_fast(w0, r3, vs03); + + vs10 = v_dotprod_expand_fast(w1, r0, vs10); + vs11 = v_dotprod_expand_fast(w1, r1, vs11); + vs12 = v_dotprod_expand_fast(w1, r2, vs12); + vs13 = v_dotprod_expand_fast(w1, r3, vs13); + } + s0 += v_int32x4(v_reduce_sum(vs00), v_reduce_sum(vs01), v_reduce_sum(vs02), v_reduce_sum(vs03)); + s1 += v_int32x4(v_reduce_sum(vs10), v_reduce_sum(vs11), v_reduce_sum(vs12), v_reduce_sum(vs13)); + if( cn1 == inpCn ) + { + s0 = voutzp + v_round(v_cvt_f32(s0)*vmult0); + s1 = voutzp + v_round(v_cvt_f32(s1)*vmult1); + + s0 = v_min(v_max(s0, outmin), outmax); + s1 = v_min(v_max(s1, outmin), outmax); + } + v_store(outptr0 + j, s0); + v_store(outptr1 + j, s1); + } + #endif + for( ; j < bsz; j++ ) + { + const int8_t* rptr = rowbuf0 + j*vsz_a; + int s00, s10; + + if( cn0 == 0 ) + { + s00 = bias0; + s10 = bias1; + } + else + { + s00 = outptr0[j]; + s10 = outptr1[j]; + } + + for( k = 0; k < vsz; k++ ) + { + int8_t r0 = rptr[k]; + s00 += (int)wptr0[k] * r0; + s10 += (int)wptr1[k] * r0; + } + if( cn1 == inpCn ) + { + int out0 = outZp + (int)std::round(s00*mult0); + int out1 = outZp + (int)std::round(s10*mult1); + + s00 = std::min(std::max(out0, -128), 127); + s10 = std::min(std::max(out1, -128), 127); + } + + outptr0[j] = s00; + outptr1[j] = s10; + } + } + } + } + if( activ_ ) + activ_->forwardSlice(data_out0 + stripeStart, lutptr_, + data_out0 + stripeStart, (int)(stripeEnd - stripeStart), + outPlaneSize, startOutCn, startOutCn + outCn); + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + +#if CV_SSE3 + uint32_t ftzMode = _MM_GET_FLUSH_ZERO_MODE(); + uint32_t dazMode = _MM_GET_DENORMALS_ZERO_MODE(); + _MM_SET_FLUSH_ZERO_MODE(_MM_FLUSH_ZERO_ON); + _MM_SET_DENORMALS_ZERO_MODE(_MM_DENORMALS_ZERO_ON); +#endif + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + /*if (inputs[0].dims > 3) { + printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", + name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3], + kernel.width, kernel.height, pad.width, pad.height, + stride.width, stride.height, dilation.width, dilation.height); + } + else { + printf("conv %s: input (%d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n", + name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], + kernel.width, kernel.height, pad.width, pad.height, + stride.width, stride.height, dilation.width, dilation.height); + }*/ + + int inpGroupCn = blobs[0].size[1]; + CV_Assert_N(inputs.size() == (size_t)1, inputs[0].size[1] % inpGroupCn == 0, + outputs.size() == 1, inputs[0].data != outputs[0].data); + + int ngroups = inputs[0].size[1] / inpGroupCn; + CV_Assert(outputs[0].size[1] % ngroups == 0); + + int nstripes = std::max(getNumThreads(), 1); + Mat outputInt32 = Mat(shape(outputs[0]), CV_32S); + + ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides, + pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, input_zp, output_zp); + + outputInt32.convertTo(outputs[0], CV_8S); + +#if CV_SSE3 + _MM_SET_FLUSH_ZERO_MODE(ftzMode); + _MM_SET_DENORMALS_ZERO_MODE(dazMode); +#endif + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_Assert(inputs.size() == outputs.size()); + + int64 flops = 0; + int karea = std::accumulate(kernel_size.begin(), kernel_size.end(), 1, std::multiplies()); + for (int i = 0; i < outputs.size(); i++) + { + flops += total(outputs[i])*(CV_BIG_INT(2)*karea*inputs[i][1] + 1); + } + return flops; + } +}; + +Ptr ConvolutionLayerInt8::create(const LayerParams ¶ms) +{ + return Ptr(new ConvolutionLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/elementwise_layers.cpp b/modules/dnn/src/int8layers/elementwise_layers.cpp new file mode 100644 index 0000000000..75118b6bc1 --- /dev/null +++ b/modules/dnn/src/int8layers/elementwise_layers.cpp @@ -0,0 +1,190 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include +#include + +namespace cv +{ +namespace dnn +{ + +class ActivationLayerInt8Impl CV_FINAL : public ActivationLayerInt8 +{ +public: + ActivationLayerInt8Impl(const LayerParams ¶ms) + { + setParamsFrom(params); + activationLUT = !blobs.empty() ? blobs[0] : Mat(); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + class Activation : public cv::ParallelLoopBody + { + public: + const Mat* src; + const Mat* lut; + Mat* dst; + int nstripes; + + Activation() : src(0), lut(0), dst(0), nstripes(0){} + + static void run(const Mat& src, const Mat& lut, Mat& dst, int nstripes) + { + Activation p; + + p.src = &src; + p.lut = &lut; + p.dst = &dst; + p.nstripes = nstripes; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range &r) const CV_OVERRIDE + { + const int8_t* table = lut->ptr(); + int nsamples = 1, outCn = 1; + size_t planeSize = 1; + + if (src->dims > 1) + { + nsamples = src->size[0]; + outCn = src->size[1]; + } + else + outCn = src->size[0]; + + for (int i = 2; i < src->dims; ++i) + planeSize *= src->size[i]; + + size_t stripeSize = (planeSize + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, planeSize); + int len = (int)(stripeEnd - stripeStart); + + for( int i = 0; i < nsamples; i++ ) + { + const int8_t* srcptr = src->ptr(i) + stripeStart; + int8_t* dstptr = dst->ptr(i) + stripeStart; + for( int cn = 0; cn < outCn; cn++, srcptr += planeSize, dstptr += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int8x16 out(table[srcptr[i] + 128], table[srcptr[i+1] + 128], table[srcptr[i+2] + 128], table[srcptr[i+3] + 128], + table[srcptr[i+4] + 128], table[srcptr[i+5] + 128], table[srcptr[i+6] + 128], table[srcptr[i+7] + 128], + table[srcptr[i+8] + 128], table[srcptr[i+9] + 128], table[srcptr[i+10] + 128], table[srcptr[i+11] + 128], + table[srcptr[i+12] + 128], table[srcptr[i+13] + 128], table[srcptr[i+14] + 128], table[srcptr[i+15] + 128]); + v_store(dstptr + i, out); + } +#endif + for( ; i < len; i++ ) + { + dstptr[i] = table[srcptr[i] + 128]; + } + } + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + for (size_t i = 0; i < inputs.size(); i++) + { + const Mat &src = inputs[i]; + if (!activationLUT.empty()) + { + const int nstripes = getNumThreads(); + Mat &dst = outputs[i]; + CV_Assert(src.size == dst.size && src.type() == dst.type() && + src.isContinuous() && dst.isContinuous() && src.type() == CV_8S); + + Activation::run(src, activationLUT, dst, nstripes); + } + else + { + src.copyTo(outputs[i]); + } + } + } + + void forwardSlice(const int8_t* src, const int8_t* lut, int8_t* dst, int len, size_t planeSize, int cn0, int cn1) const CV_OVERRIDE + { + for( int cn = cn0; cn < cn1; cn++, src += planeSize, dst += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int8x16 out(lut[src[i] + 128], lut[src[i+1] + 128], lut[src[i+2] + 128], lut[src[i+3] + 128], + lut[src[i+4] + 128], lut[src[i+5] + 128], lut[src[i+6] + 128], lut[src[i+7] + 128], + lut[src[i+8] + 128], lut[src[i+9] + 128], lut[src[i+10] + 128], lut[src[i+11] + 128], + lut[src[i+12] + 128], lut[src[i+13] + 128], lut[src[i+14] + 128], lut[src[i+15] + 128]); + v_store(dst + i, out); + } +#endif + for( ; i < len; i++ ) + dst[i] = lut[src[i] + 128]; + } + } + + void forwardSlice(const int* src, const int* lut, int* dst, int len, size_t planeSize, int cn0, int cn1) const CV_OVERRIDE + { + for( int cn = cn0; cn < cn1; cn++, src += planeSize, dst += planeSize ) + { + int i = 0; +#if CV_SIMD128 + for( ; i <= len - 16; i += 16 ) + { + v_int32x4 out0(lut[src[i] + 128], lut[src[i+1] + 128], lut[src[i+2] + 128], lut[src[i+3] + 128]); + v_int32x4 out1(lut[src[i+4] + 128], lut[src[i+5] + 128], lut[src[i+6] + 128], lut[src[i+7] + 128]); + v_int32x4 out2(lut[src[i+8] + 128], lut[src[i+9] + 128], lut[src[i+10] + 128], lut[src[i+11] + 128]); + v_int32x4 out3(lut[src[i+12] + 128], lut[src[i+13] + 128], lut[src[i+14] + 128], lut[src[i+15] + 128]); + + v_store(dst + i, out0); + v_store(dst + i + 4, out1); + v_store(dst + i + 8, out2); + v_store(dst + i + 12, out3); + } +#endif + for( ; i < len; i++ ) + dst[i] = lut[src[i] + 128]; + } + + } + + Mat activationLUT; +}; + +Ptr ActivationLayerInt8::create(const LayerParams& params) +{ + return Ptr(new ActivationLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/eltwise_layer.cpp b/modules/dnn/src/int8layers/eltwise_layer.cpp new file mode 100644 index 0000000000..be7a32b1ef --- /dev/null +++ b/modules/dnn/src/int8layers/eltwise_layer.cpp @@ -0,0 +1,577 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include + +namespace cv +{ +namespace dnn +{ + +class EltwiseLayerInt8Impl CV_FINAL : public EltwiseLayerInt8 +{ +public: + enum EltwiseOp + { + PROD = 0, + SUM = 1, + MAX = 2 + } op; + std::vector coeffs; + std::vector zeropoints; + + enum OutputChannelsMode + { + ELTWISE_CHANNNELS_SAME = 0, //!< number of channels from inputs must be the same and equal to output's number of channels + ELTWISE_CHANNNELS_INPUT_0, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to number of channels of first input + //!< number of channels of other inputs should not be greater than number of channels of first input + ELTWISE_CHANNNELS_INPUT_0_TRUNCATE, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to number of channels of first input + //!< there is restriction on number of channels of other inputs + //!< extra channels of other inputs is ignored + ELTWISE_CHANNNELS_USE_MAX, //!< number of channels from inputs may be different, + //!< output's number of channels is equal to maximal number of input channels + //!< @note supported operation: `SUM` + } channelsModeInput; + + + mutable OutputChannelsMode channelsMode; //!< "optimized" channels mode (switch to ELTWISE_CHANNNELS_SAME if number of input channels are equal) + mutable /*size_t*/int outputChannels; + + EltwiseLayerInt8Impl(const LayerParams& params) + : outputChannels(0) + { + setParamsFrom(params); + offset = params.get("offset", 0.f); + hasVecInput = false; + op = SUM; + if (params.has("operation")) + { + String operation = toLowerCase(params.get("operation")); + if (operation == "prod") + op = PROD; + else if (operation == "sum") + op = SUM; + else if (operation == "max") + op = MAX; + else + CV_Error(cv::Error::StsBadArg, "Unknown operation type \"" + operation + "\""); + } + + if (params.has("coeff")) + { + DictValue paramCoeff = params.get("coeff"); + int i, n = paramCoeff.size(); + coeffs.resize(n); + for (i = 0; i < n; i++) + { + coeffs[i] = paramCoeff.get(i); + } + } + + if (params.has("input_zeropoints")) + { + DictValue zp = params.get("input_zeropoints"); + int i, n = zp.size(); + zeropoints.resize(n); + for (i = 0; i < n; i++) + { + zeropoints[i] = zp.get(i); + } + } + + channelsModeInput = ELTWISE_CHANNNELS_SAME; + if (params.has("output_channels_mode")) + { + String v = toLowerCase(params.get("output_channels_mode")); + if (v == "same") + { + channelsModeInput = ELTWISE_CHANNNELS_SAME; + } + else if (v == "input_0") + { + channelsModeInput = ELTWISE_CHANNNELS_INPUT_0; + } + else if (v == "input_0_truncate") + { + channelsModeInput = ELTWISE_CHANNNELS_INPUT_0_TRUNCATE; + } + else if (v == "max_input_channels") + { + channelsModeInput = ELTWISE_CHANNNELS_USE_MAX; + if (op != SUM) + CV_Error(cv::Error::StsBadArg, "[" + type + "]:(" + name + ") 'max' channels mode is limited to SUM operation only"); + } + else + CV_Error(cv::Error::StsBadArg, "[" + type + "]:(" + name + ") unknown channels mode: \"" + v + "\""); + } + channelsMode = channelsModeInput; + + // TODO Must have checks for other unknown options + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() >= 2); + CV_Assert(inputs[0].size() >= 2); + CV_Assert(coeffs.size() == 0 || coeffs.size() == inputs.size()); + CV_Assert(op == SUM || op == PROD || coeffs.size() == 0); + + int dims = inputs[0].size(); + // Number of channels in output shape is determined by the first input tensor. + bool variableChannels = false; + int numChannels = inputs[0][1]; + for (size_t i = 1; i < inputs.size(); i++) + { + CV_Assert(inputs[0][0] == inputs[i][0]); // batch sizes are equal + + int input_channels = inputs[i][1]; + if (numChannels != input_channels) + variableChannels = true; + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME) + { + CV_Assert(numChannels == input_channels); + } + else if (channelsModeInput == ELTWISE_CHANNNELS_INPUT_0) + { + CV_Assert(numChannels >= input_channels); + } + else if (channelsModeInput == ELTWISE_CHANNNELS_INPUT_0_TRUNCATE) + { + // nothing to check + } + else if (channelsModeInput == ELTWISE_CHANNNELS_USE_MAX) + { + numChannels = std::max(numChannels, input_channels); + } + else + { + CV_Assert(0 && "Internal error"); + } + } + + channelsMode = variableChannels ? channelsModeInput : ELTWISE_CHANNNELS_SAME; + outputChannels = numChannels; + + outputs.assign(1, inputs[0]); + outputs[0][1] = numChannels; + + if (dims > 2) + { + size_t vecIdx = 0; + bool isVecFound = false; + for (size_t i = 0; i < inputs.size(); i++) + { + bool allOnes = isAllOnes(inputs[i], 2, dims); + if (!allOnes && !isVecFound) + { + vecIdx = i; + isVecFound = true; + } + + if (!allOnes && i != vecIdx) + { + for (size_t j = 2; j < dims; j++) + { + CV_Assert(inputs[vecIdx][j] == inputs[i][j]); + } + } + } + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME && isVecFound) + { + for (size_t j = 2; j < dims; j++) + { + outputs[0][j] = inputs[vecIdx][j]; + } + } + } + + return false; + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector inputs; + inputs_arr.getMatVector(inputs); + + for (size_t i = 0; i < inputs.size(); i++) + { + MatShape inpShape = shape(inputs[i].size); + if (isAllOnes(inpShape, 2, inputs[i].dims)) + { + hasVecInput = true; + return; + } + } + } + + class EltwiseInvoker : public ParallelLoopBody + { + EltwiseLayerInt8Impl& self; + std::vector srcs; + std::vector srcNumChannels; + int nsrcs; + Mat* dst; + Mat* buf; + std::vector coeffs; + std::vector zeropoints; + int nstripes; + const Mat* activLUT; + const ActivationLayerInt8* activ; + int channels; + size_t planeSize; + float offset; + + EltwiseInvoker(EltwiseLayerInt8Impl& self_) + : self(self_) + , nsrcs(0), dst(0), buf(0), nstripes(0), activ(0), channels(0) + , planeSize(0), offset(0) + {} + + public: + static void run(EltwiseLayerInt8Impl& self, + const Mat* srcs, int nsrcs, Mat& buf, Mat& dst, + int nstripes, float offset) + { + const EltwiseOp op = self.op; + CV_Check(dst.dims, 1 < dst.dims && dst.dims <= 5, ""); CV_CheckTypeEQ(dst.type(), CV_8SC1, ""); CV_Assert(dst.isContinuous()); + CV_Assert(self.coeffs.empty() || self.coeffs.size() == (size_t)nsrcs); + CV_CheckGE(nsrcs, 2, ""); + + CV_Assert(self.outputChannels == dst.size[1]); + + EltwiseInvoker p(self); + p.srcs.resize(nsrcs); + p.srcNumChannels.resize(nsrcs); + p.coeffs = self.coeffs; // can be sorted + p.zeropoints = self.zeropoints; + + bool sortInputs = false; + for( int i = 0; i < nsrcs; i++ ) + { + p.srcs[i] = &srcs[i]; + CV_CheckEQ(srcs[i].dims, dst.dims, ""); + CV_Assert(srcs[i].isContinuous()); + CV_Assert(srcs[i].type() == dst.type()); + p.srcNumChannels[i] = (srcs[i].dims >= 4) ? srcs[i].size[1] : 1; + + if (self.channelsMode == ELTWISE_CHANNNELS_SAME) + { + CV_Assert(srcs[i].size == dst.size); + } + else if (self.channelsMode == ELTWISE_CHANNNELS_INPUT_0) + { + if (i == 0) + CV_Assert(srcs[0].size == dst.size); + CV_Assert(self.outputChannels >= p.srcNumChannels[i]); + sortInputs = true; + } + else if (self.channelsMode == ELTWISE_CHANNNELS_INPUT_0_TRUNCATE) + { + if (i == 0) + CV_Assert(srcs[0].size == dst.size); + sortInputs = true; + } + else if (self.channelsMode == ELTWISE_CHANNNELS_USE_MAX) + { + CV_Assert(op == SUM); + CV_Assert(self.outputChannels >= p.srcNumChannels[i]); + sortInputs = true; + } + else + { + CV_Assert(0 && "Internal error"); + } + + if (sortInputs) + { + // Sort srcs and coefficients in the desc order by number of channels + for (int j = i; j >= 1; j--) + { + if (std::min(self.outputChannels, p.srcs[j - 1]->size[1]) < std::min(self.outputChannels, p.srcs[j]->size[1])) + { + std::swap(p.srcs[j - 1], p.srcs[j]); + std::swap(p.srcNumChannels[j - 1], p.srcNumChannels[j]); + if (!p.coeffs.empty()) + std::swap(p.coeffs[j - 1], p.coeffs[j]); + if (!p.zeropoints.empty()) + std::swap(p.zeropoints[j - 1], p.zeropoints[j]); + } + else + break; + } + } + } + + p.nsrcs = nsrcs; + p.dst = &dst; + p.buf = &buf; + p.nstripes = nstripes; + p.offset = offset; + p.channels = (dst.dims >= 4 ? dst.size[1] : 1); + + p.planeSize = dst.total(dst.dims >= 4 ? 2 : 1); + CV_CheckEQ(dst.total(), dst.size[0] * p.channels * p.planeSize, ""); + p.activLUT = &self.activationLUT; + p.activ = !self.activationLUT.empty() ? self.activ.get() : 0; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + const EltwiseOp op = self.op; + size_t total = dst->size[0]*planeSize; + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, total); + const float* coeffsptr = !coeffs.empty() ? &coeffs[0] : 0; + const int* zeropointsptr = !zeropoints.empty() ? &zeropoints[0] : 0; + const int8_t* lutptr = !activLUT->empty() ? activLUT->ptr() : 0; + int8_t* dstptr0 = dst->ptr(); + float* bufptr0 = buf->ptr(); + int blockSize0 = 1 << 12; + + for (size_t ofs = stripeStart; ofs < stripeEnd; ) + { + int sampleIdx = (int)(ofs / planeSize); + int delta = (int)ofs - sampleIdx * planeSize; + int blockSize = std::min(blockSize0, std::min((int)(stripeEnd - ofs), (int)planeSize - delta)); + if( blockSize <= 0 ) + break; + ofs += blockSize; + + for (int c = 0; c < channels; c++) + { + size_t dstIdx = delta + (sampleIdx*channels + c)*planeSize; + int8_t* dstptr = dstptr0 + dstIdx; + float* bufptr = bufptr0 + dstIdx; + + // process first two inputs + { + const int8_t* srcptr0 = srcs[0]->ptr() + dstIdx; + + const int inputIdx = 1; + int src1_channels = srcNumChannels[inputIdx]; + if (c >= src1_channels) + { + // no data from second input + if (!coeffsptr) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = srcptr0[j]; + } + } + else + { + float c0 = coeffsptr[0]; + int z0 = op == PROD ? zeropointsptr[0] : 0; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = c0 * (srcptr0[j] - z0); + } + } + } + else + { + size_t srcIdx = delta + (sampleIdx * src1_channels + c) * planeSize; + const int8_t* srcptrI = srcs[inputIdx]->ptr() + srcIdx; + + if (op == PROD) + { + float c0 = coeffsptr[0]; + float c1 = coeffsptr[1]; + int z0 = zeropointsptr[0]; + int z1 = zeropointsptr[1]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = (c0*(srcptr0[j] - z0)) * (c1*(srcptrI[j] - z1)); + } + } + else if (op == MAX) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::max(srcptr0[j], srcptrI[j]); + } + } + else if (op == SUM) + { + float c0 = coeffsptr[0]; + float c1 = coeffsptr[1]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] = c0*srcptr0[j] + c1*srcptrI[j]; + } + } + else + CV_Error(Error::StsInternal, ""); + } + } + + // aggregate other inputs (3+) + for (size_t inputIdx = 2; inputIdx < nsrcs; inputIdx++) + { + int srcI_channels = srcNumChannels[inputIdx]; + if (c >= srcI_channels) + continue; // no data from second input + size_t srcIdx = delta + (sampleIdx * srcI_channels + c) * planeSize; + const int8_t* srcptrI = srcs[inputIdx]->ptr() + srcIdx; + + if (op == PROD) + { + float cI = coeffsptr[inputIdx]; + int zI = zeropointsptr[inputIdx]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] *= cI*(srcptrI[j] - zI); + } + } + else if (op == MAX) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::max(dstptr[j], srcptrI[j]); + } + } + else if (op == SUM) + { + float cI = coeffsptr[inputIdx]; + for (int j = 0; j < blockSize; j++) + { + bufptr[j] += cI * srcptrI[j]; + } + } + else + CV_Error(Error::StsInternal, ""); + } + + // add offset and saturate cast to int8 + if (op == SUM || op == PROD) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = saturate_cast(std::round(bufptr[j] + offset)); + } + } + } + if( activ ) + { + int8_t* ptr = dstptr0 + delta + sampleIdx*channels*planeSize; + activ->forwardSlice(ptr, lutptr, ptr, blockSize, planeSize, 0, channels); + } + } + } + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(outputs.size() == 1); + const int nstripes = getNumThreads(); + + if (channelsModeInput == ELTWISE_CHANNNELS_SAME && inputs[0].dims > 2) + { + for (size_t i = 0; i < inputs.size(); i++) + { + MatShape inpShape = shape(inputs[i].size); + bool allOnes = isAllOnes(inpShape, 2, inputs[i].dims); + + if (allOnes) + { + Mat tmpInput = inputs[i]; + MatShape outShape = shape(outputs[0].size); + size_t xSize = outShape[2]; + for (size_t j = 3; j < outShape.size(); j++) + xSize *= outShape[j]; + + int dimVec[3] = {outShape[0], outShape[1], (int) xSize}; + std::vector matSizesVec(&dimVec[0], &dimVec[0] + 3); + inputs[i] = Mat(matSizesVec, tmpInput.type()); + + std::vector idx(outShape.size(), 0); + std::vector outIdx(inpShape.size(), 0); + + for (size_t j = 0; j < outShape[0]; j++) + { + outIdx[0] = idx[0] = j; + for(size_t k = 0; k < outShape[1]; k++) + { + outIdx[1] = idx[1] = k; + for (size_t x = 0; x < xSize; x++) + { + outIdx[2] = x; + inputs[i].at(outIdx.data()) = tmpInput.at(idx.data()); + } + } + } + inputs[i] = inputs[i].reshape(0, outShape); + } + } + } + + Mat buf = Mat(shape(outputs[0]), CV_32F); // to store intermediate results + EltwiseInvoker::run(*this, &inputs[0], (int)inputs.size(), buf, outputs[0], nstripes, offset); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + CV_Assert(inputs.size()); + + // FIXIT: handle inputs with different number of channels + long flops = inputs.size() * total(inputs[0]); + + return flops; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activationLUT = activ_int8->blobs[0]; + return true; + } + return false; + } + + Mat activationLUT; + Ptr activ; + +private: + bool hasVecInput; + float offset; +}; + +Ptr EltwiseLayerInt8::create(const LayerParams& params) +{ + return Ptr(new EltwiseLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/fully_connected_layer.cpp b/modules/dnn/src/int8layers/fully_connected_layer.cpp new file mode 100644 index 0000000000..83da677a47 --- /dev/null +++ b/modules/dnn/src/int8layers/fully_connected_layer.cpp @@ -0,0 +1,266 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +namespace cv +{ +namespace dnn +{ + +class FullyConnectedLayerInt8Impl CV_FINAL : public InnerProductLayerInt8 +{ +public: + enum { VEC_ALIGN = 32 }; + FullyConnectedLayerInt8Impl(const LayerParams& params) + { + setParamsFrom(params); + output_zp = params.get("zeropoints"); + axis = params.get("axis", 1); + if (blobs.size() == 3) + { + // blobs[0] - Weights + // blobs[1] - Bias fused with offset + // blobs[2] - Multipliers for output stage + int numOutput = params.get("num_output"); + int innerSize = (int)blobs[0].total() / numOutput; + + CV_Assert(blobs[0].dims >= 2 && (size_t)(innerSize * numOutput) == blobs[0].total()); + CV_Assert((size_t)numOutput == blobs[1].total()); + + weightsMat = blobs[0] = blobs[0].reshape(1, numOutput); + int vecsize = weightsMat.cols; + if (vecsize % VEC_ALIGN != 0) + { + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + Mat weightsBuf(weightsMat.rows, vecsize_aligned, weightsMat.type()); + Mat wpadding = weightsBuf.colRange(vecsize, vecsize_aligned); + wpadding.setTo(Scalar::all(0)); + weightsMat = weightsBuf.colRange(0, vecsize); + blobs[0].copyTo(weightsMat); + } + biasMat = blobs[1] = blobs[1].reshape(1, 1); + outputMultiplier = blobs[2]; + } + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &) const CV_OVERRIDE + { + int numOutput, cAxis; + CV_CheckEQ(inputs.size(), (size_t)1, ""); + CV_CheckEQ(blobs[0].dims, 2, ""); + numOutput = blobs[0].size[0]; + CV_Assert((size_t)numOutput == blobs[1].total()); + cAxis = normalize_axis(axis, inputs[0]); + + MatShape outShape(cAxis + 1); + for (int i = 0; i < cAxis; ++i) + outShape[i] = inputs[0][i]; + outShape.back() = numOutput; + + outputs.resize(1, outShape); + return false; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + activ = activ_int8; + if (!activ_int8->blobs.empty()) + activ_int8->blobs[0].convertTo(activationLUT, CV_32S); + return true; + } + return false; + } + + class FullyConnected : public ParallelLoopBody + { + public: + FullyConnected() : srcMat(0), weights(0), biasMat(0), outputMultiplier(0), activationLUT(0), activ(0), + dstMat(0), nstripes(0), outZp(0), useAVX2(false), useAVX512(false) {} + + static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat, const Mat& outputMultiplier, + const Mat& activationLUT, Mat& dstMat, const ActivationLayerInt8* activ, int nstripes, int outZp) + { + CV_Assert( srcMat.dims == 2 && srcMat.cols == weights.cols && + dstMat.rows == srcMat.rows && dstMat.cols == weights.rows && + srcMat.type() == weights.type() && srcMat.type() == CV_8S && + dstMat.type() == CV_32S && biasMat.type() == CV_32S && + biasMat.isContinuous() && (int)biasMat.total() == dstMat.cols ); + + FullyConnected p; + + p.srcMat = &srcMat; + p.weights = &weights; + p.biasMat = &biasMat; + p.outputMultiplier = &outputMultiplier; + p.activationLUT = &activationLUT; + p.dstMat = &dstMat; + p.nstripes = nstripes; + p.outZp = outZp; + p.activ = !activationLUT.empty() ? activ : 0; + p.useAVX2 = checkHardwareSupport(CPU_AVX2); + p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + int valign = FullyConnectedLayerInt8Impl::VEC_ALIGN; + int nsamples = srcMat->rows; + int nw0 = weights->rows; + int k, vecsize = srcMat->cols; + int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN); + size_t total = (size_t)nsamples*nw0; + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = r.end == nstripes ? total : std::min(r.end*stripeSize, total); + size_t wstep = weights->step1(); + AutoBuffer srcbuf(vecsize_aligned + valign); + int8_t* sptr = alignPtr(srcbuf.data(), (int)(valign*sizeof(int8_t))); + const int* lutptr = !activationLUT->empty() ? activationLUT->ptr() : 0; + + for( k = vecsize; k < vecsize_aligned; k++ ) + sptr[k] = 0; + + for( size_t ofs = stripeStart; ofs < stripeEnd; ) + { + int sampleIdx = (int)(ofs / nw0); + int delta = (int)(ofs - (size_t)sampleIdx*nw0); + const int8_t* sptr_ = srcMat->ptr(sampleIdx); + const int8_t* wptr = weights->ptr(delta); + int* dptr = dstMat->ptr(sampleIdx) + delta; + const int* biasptr = biasMat->ptr() + delta; + const float* multptr = outputMultiplier->ptr() + delta; + int nw = std::min(nw0 - delta, (int)(stripeEnd - ofs)); + + memcpy(sptr, sptr_, vecsize*sizeof(sptr[0])); + #if CV_TRY_AVX512_SKX + if( useAVX512 ) + opt_AVX512_SKX::fastGEMM1T( sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp ); + else + #endif + #if CV_TRY_AVX2 + if( useAVX2 ) + opt_AVX2::fastGEMM1T( sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp ); + else + #endif + { + int i = 0; + #if CV_SIMD + for( ; i <= nw - 4; i += 4, wptr += 4*wstep ) + { + v_int32x4 vs0 = v_setzero_s32(), vs1 = v_setzero_s32(), + vs2 = v_setzero_s32(), vs3 = v_setzero_s32(); + v_int32x4 outzp = v_setall_s32(outZp), outmin = v_setall_s32(-128), outmax = v_setall_s32(127); + v_int32x4 s = v_load(biasptr + i); + v_float32x4 mult = v_load(multptr + i); + + for( k = 0; k < vecsize; k += 16 ) + { + v_int8x16 v = v_load_aligned(sptr + k); + vs0 = v_dotprod_expand_fast(v, v_load_aligned(wptr + k), vs0); + vs1 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep + k), vs1); + vs2 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep*2 + k), vs2); + vs3 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep*3 + k), vs3); + } + + s += v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3)); + v_int32x4 out = outzp + v_round(v_cvt_f32(s)*mult); + v_store(dptr + i, v_min(v_max(out, outmin), outmax)); + } + #endif + + for( ; i < nw; i++, wptr += wstep ) + { + int s0 = biasptr[i]; + float mult0 = multptr[i]; + + for( k = 0; k < vecsize; k++ ) + { + int8_t v = sptr[k]; + s0 += (int)v*wptr[k]; + } + int out0 = outZp + (int)std::round(s0*mult0); + dptr[i] = std::min(std::max(out0, -128), 127); + } + } + + if(activ) + activ->forwardSlice(dptr, lutptr, dptr, 1, 1, delta, delta + nw); + + ofs += nw; + } + } + + const Mat *srcMat, *weights, *biasMat, *outputMultiplier, *activationLUT; + const ActivationLayerInt8* activ; + Mat* dstMat; + int nstripes, outZp; + bool useAVX2; + bool useAVX512; + }; + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector input, output; + inputs_arr.getMatVector(input); + outputs_arr.getMatVector(output); + + int axisCan = normalize_axis(axis, input[0].dims); + int outerSize = input[0].total(0, axisCan); + Mat srcMat = input[0].reshape(1, outerSize); + + Mat dstMat = output[0].reshape(1, outerSize); + Mat dstMatInt32= Mat(shape(dstMat), CV_32S); + + const int nstripes = getNumThreads(); + FullyConnected::run(srcMat, weightsMat, biasMat, outputMultiplier, activationLUT, dstMatInt32, activ.get(), nstripes, output_zp); + dstMatInt32.convertTo(dstMat, CV_8S); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(inputs); // suppress unused variable warning + long flops = 0; + + int innerSize = blobs[0].size[1]; + for(int i = 0; i < outputs.size(); i++) + { + flops += CV_BIG_INT(3)*innerSize*total(outputs[i]); + } + + return flops; + + } + + Mat weightsMat, biasMat, outputMultiplier, activationLUT; + Ptr activ; +}; + +Ptr InnerProductLayerInt8::create(const LayerParams& params) +{ + return Ptr(new FullyConnectedLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/layers_common.hpp b/modules/dnn/src/int8layers/layers_common.hpp new file mode 100644 index 0000000000..cb185a9eda --- /dev/null +++ b/modules/dnn/src/int8layers/layers_common.hpp @@ -0,0 +1,41 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef __OPENCV_DNN_LAYERS_LAYERS_COMMON_HPP__ +#define __OPENCV_DNN_LAYERS_LAYERS_COMMON_HPP__ +#include +#include + +#define CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +// dispatched AVX/AVX2 optimizations +#include "./layers_common.simd.hpp" +#include "int8layers/layers_common.simd_declarations.hpp" +#undef CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +#ifdef HAVE_OPENCL +#include "../ocl4dnn/include/ocl4dnn.hpp" +#endif + +namespace cv +{ +namespace dnn +{ +void getConvolutionKernelParams(const LayerParams ¶ms, std::vector& kernel, std::vector& pads_begin, + std::vector& pads_end, std::vector& strides, std::vector& dilations, + cv::String &padMode, std::vector& adjust_pads); + +void getPoolingKernelParams(const LayerParams ¶ms, std::vector& kernel, std::vector& globalPooling, + std::vector& pads_begin, std::vector& pads_end, std::vector& strides, cv::String &padMode); + +void getConvPoolOutParams(const std::vector& inp, const std::vector& kernel, + const std::vector& stride, const String &padMode, + const std::vector& dilation, std::vector& out); + + void getConvPoolPaddings(const std::vector& inp, const std::vector& kernel, + const std::vector& strides, const String &padMode, + std::vector& pads_begin, std::vector& pads_end); +} +} + +#endif diff --git a/modules/dnn/src/int8layers/layers_common.simd.hpp b/modules/dnn/src/int8layers/layers_common.simd.hpp new file mode 100644 index 0000000000..bf6149e5c9 --- /dev/null +++ b/modules/dnn/src/int8layers/layers_common.simd.hpp @@ -0,0 +1,637 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "opencv2/core/hal/intrin.hpp" + +namespace cv { +namespace dnn { +CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN + +void fastConv( const int8_t* weights, size_t wstep, const int* bias, + const int8_t* rowbuf, int* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, int outZp, + const float* multiplier, bool initOutput, bool finalOutput ); +void fastDepthwiseConv( const int8_t* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const int* biasptr, const float* multptr, + const int8_t* inptr_, + int height, int width, + int* outptr_, + int out_d, int outH, int outW, + int inpZp, int outZp ); +void fastGEMM1T( const int8_t* vec, const int8_t* weights, + size_t wstep, const int* bias, const float* multiplier, + int* dst, int nvecs, int vecsize, int outZp ); + +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX2 +#define OPENCV_FMADD_EPI8(_Tpvec, func) \ + inline _Tpvec _##func##_fmaddepi8_epi32(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \ + { \ + _Tpvec even_a = _##func##_srai_epi16(_##func##_bslli_epi128(a, 1), 8); \ + _Tpvec odd_a = _##func##_srai_epi16(a, 8); \ + \ + _Tpvec even_b = _##func##_srai_epi16(_##func##_bslli_epi128(b, 1), 8); \ + _Tpvec odd_b = _##func##_srai_epi16(b, 8); \ + \ + _Tpvec prod0 = _##func##_madd_epi16(even_a, even_b); \ + _Tpvec prod1 = _##func##_madd_epi16(odd_a, odd_b); \ + return _##func##_add_epi32(_##func##_add_epi32(prod0, prod1), c); \ + } +OPENCV_FMADD_EPI8(__m256i, mm256) +//OPENCV_FMADD_EPI8(__m512i, mm512) + +enum { FASCONV_BASE_VECSZ = 4 }; + +void fastConv( const int8_t* weights, size_t wstep, const int* bias, + const int8_t* rowbuf, int* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, int outZp, + const float* multiplier, bool initOutput, bool finalOutput ) +{ + int outCn = outShape[1]; + size_t outPlaneSize = outShape[2]*outShape[3]; + int CV_DECL_ALIGNED(16) maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + __m128 mask = _mm_loadu_ps((const float*)maskbuf); + + // now compute dot product of the weights + // and im2row-transformed part of the tensor + for( int i = 0; i < outCn; i += 3 ) + { + const int8_t* wptr0 = weights + i*wstep; + const int8_t* wptr1 = wptr0 + wstep; + const int8_t* wptr2 = wptr1 + wstep; + int* outptr0 = output + i*outPlaneSize; + int* outptr1 = outptr0 + outPlaneSize; + int* outptr2 = outptr1 + outPlaneSize; + int bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2]; + float mult0 = multiplier[i], mult1 = multiplier[i+1], mult2 = multiplier[i+2]; + + if( i+2 >= outCn ) + { + wptr2 = wptr1; + outptr2 = outptr1; + bias2 = bias1; + mult2 = mult1; + + if( i+1 >= outCn ) + { + wptr2 = wptr1 = wptr0; + outptr2 = outptr1 = outptr0; + bias2 = bias1 = bias0; + mult2 = mult1 = mult0; + } + } + int j = 0; + for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) + { + bool tail = false; + if (j + FASCONV_BASE_VECSZ > blockSize) + { + if (j == 0) + break; + j = blockSize - FASCONV_BASE_VECSZ; + tail = true; + } + int k = 0; + const int8_t* rptr = rowbuf + j*vecsize_aligned; + + __m256i vs00 = _mm256_setzero_si256(), vs01 = _mm256_setzero_si256(), + vs02 = _mm256_setzero_si256(), vs03 = _mm256_setzero_si256(), + vs10 = _mm256_setzero_si256(), vs11 = _mm256_setzero_si256(), + vs12 = _mm256_setzero_si256(), vs13 = _mm256_setzero_si256(), + vs20 = _mm256_setzero_si256(), vs21 = _mm256_setzero_si256(), + vs22 = _mm256_setzero_si256(), vs23 = _mm256_setzero_si256(); + + /* TODO : Fix AVX-512 path. Segmentation fault in Conv2D Tests. +#if CV_AVX512_SKX // AVX512VL is necessary to avoid register spilling + if (vecsize >= 64) + { + __m512i vs00_5 = _mm512_setzero_si512(), vs01_5 = _mm512_setzero_si512(), + vs02_5 = _mm512_setzero_si512(), vs03_5 = _mm512_setzero_si512(), + vs10_5 = _mm512_setzero_si512(), vs11_5 = _mm512_setzero_si512(), + vs12_5 = _mm512_setzero_si512(), vs13_5 = _mm512_setzero_si512(), + vs20_5 = _mm512_setzero_si512(), vs21_5 = _mm512_setzero_si512(), + vs22_5 = _mm512_setzero_si512(), vs23_5 = _mm512_setzero_si512(); + + for (; k <= vecsize - 64; k += 64, rptr += 64) + { + __m512i w0 = _mm512_load_si512(wptr0 + k); + __m512i w1 = _mm512_load_si512(wptr1 + k); + __m512i w2 = _mm512_load_si512(wptr2 + k); + __m512i r0 = _mm512_load_si512(rptr); + + vs00_5 = _mm512_fmaddepi8_epi32(w0, r0, vs00_5); + vs10_5 = _mm512_fmaddepi8_epi32(w1, r0, vs10_5); + vs20_5 = _mm512_fmaddepi8_epi32(w2, r0, vs20_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned); + vs01_5 = _mm512_fmaddepi8_epi32(w0, r0, vs01_5); + vs11_5 = _mm512_fmaddepi8_epi32(w1, r0, vs11_5); + vs21_5 = _mm512_fmaddepi8_epi32(w2, r0, vs21_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned*2); + vs02_5 = _mm512_fmaddepi8_epi32(w0, r0, vs02_5); + vs12_5 = _mm512_fmaddepi8_epi32(w1, r0, vs12_5); + vs22_5 = _mm512_fmaddepi8_epi32(w2, r0, vs22_5); + + r0 = _mm512_load_si512(rptr + vecsize_aligned*3); + vs03_5 = _mm512_fmaddepi8_epi32(w0, r0, vs03_5); + vs13_5 = _mm512_fmaddepi8_epi32(w1, r0, vs13_5); + vs23_5 = _mm512_fmaddepi8_epi32(w2, r0, vs23_5); + } + + // now fold the 512 bit accumulator vectors into 256 bit vectors so that the AVX2 code can finish + // the tail of the vector + + vs00 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs00_5, 0), _mm512_extracti32x8_epi32(vs00_5, 1)); + vs10 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs10_5, 0), _mm512_extracti32x8_epi32(vs10_5, 1)); + vs20 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs20_5, 0), _mm512_extracti32x8_epi32(vs20_5, 1)); + + vs01 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs01_5, 0), _mm512_extracti32x8_epi32(vs01_5, 1)); + vs11 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs11_5, 0), _mm512_extracti32x8_epi32(vs11_5, 1)); + vs21 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs21_5, 0), _mm512_extracti32x8_epi32(vs21_5, 1)); + + vs02 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs02_5, 0), _mm512_extracti32x8_epi32(vs02_5, 1)); + vs12 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs12_5, 0), _mm512_extracti32x8_epi32(vs12_5, 1)); + vs22 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs22_5, 0), _mm512_extracti32x8_epi32(vs22_5, 1)); + + vs03 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs03_5, 0), _mm512_extracti32x8_epi32(vs03_5, 1)); + vs13 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs13_5, 0), _mm512_extracti32x8_epi32(vs13_5, 1)); + vs23 = _mm256_add_epi32( _mm512_extracti32x8_epi32(vs23_5, 0), _mm512_extracti32x8_epi32(vs23_5, 1)); + } +#endif + */ + for (; k < vecsize; k += 32, rptr += 32 ) + { + __m256i w0 = _mm256_load_si256((const __m256i*)(wptr0 + k)); + __m256i w1 = _mm256_load_si256((const __m256i*)(wptr1 + k)); + __m256i w2 = _mm256_load_si256((const __m256i*)(wptr2 + k)); + __m256i r0 = _mm256_load_si256((const __m256i*)rptr); + + vs00 = _mm256_fmaddepi8_epi32(w0, r0, vs00); + vs10 = _mm256_fmaddepi8_epi32(w1, r0, vs10); + vs20 = _mm256_fmaddepi8_epi32(w2, r0, vs20); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned)); + vs01 = _mm256_fmaddepi8_epi32(w0, r0, vs01); + vs11 = _mm256_fmaddepi8_epi32(w1, r0, vs11); + vs21 = _mm256_fmaddepi8_epi32(w2, r0, vs21); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*2)); + vs02 = _mm256_fmaddepi8_epi32(w0, r0, vs02); + vs12 = _mm256_fmaddepi8_epi32(w1, r0, vs12); + vs22 = _mm256_fmaddepi8_epi32(w2, r0, vs22); + + r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*3)); + vs03 = _mm256_fmaddepi8_epi32(w0, r0, vs03); + vs13 = _mm256_fmaddepi8_epi32(w1, r0, vs13); + vs23 = _mm256_fmaddepi8_epi32(w2, r0, vs23); + } + + __m256i t0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs00, vs01), _mm256_hadd_epi32(vs02, vs03)); + __m256i t1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs10, vs11), _mm256_hadd_epi32(vs12, vs13)); + __m256i t2 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs20, vs21), _mm256_hadd_epi32(vs22, vs23)); + + t0 = _mm256_add_epi32(t0, _mm256_permute2x128_si256(t0, t0, 1)); + t1 = _mm256_add_epi32(t1, _mm256_permute2x128_si256(t1, t1, 1)); + t2 = _mm256_add_epi32(t2, _mm256_permute2x128_si256(t2, t2, 1)); + + __m128i s0, s1, s2; + + if( initOutput ) + { + s0 = _mm_set1_epi32(bias0); + s1 = _mm_set1_epi32(bias1); + s2 = _mm_set1_epi32(bias2); + } + else + { + s0 = _mm_loadu_si128((__m128i*)(outptr0 + j)); + s1 = _mm_loadu_si128((__m128i*)(outptr1 + j)); + s2 = _mm_loadu_si128((__m128i*)(outptr2 + j)); + } + + s0 = _mm_add_epi32(s0, _mm256_castsi256_si128(t0)); + s1 = _mm_add_epi32(s1, _mm256_castsi256_si128(t1)); + s2 = _mm_add_epi32(s2, _mm256_castsi256_si128(t2)); + + if( finalOutput ) + { + __m128i voutzp = _mm_set1_epi32(outZp); + __m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127); + s0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s0), _mm_set1_ps(mult0)))); + s1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s1), _mm_set1_ps(mult1)))); + s2 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s2), _mm_set1_ps(mult2)))); + + s0 = _mm_min_epi32(_mm_max_epi32(s0, outmin), outmax); + s1 = _mm_min_epi32(_mm_max_epi32(s1, outmin), outmax); + s2 = _mm_min_epi32(_mm_max_epi32(s2, outmin), outmax); + } + if( tail ) + { + s0 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr0 + j), _mm_castsi128_ps(s0), mask)); + s1 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr1 + j), _mm_castsi128_ps(s1), mask)); + s2 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr2 + j), _mm_castsi128_ps(s2), mask)); + } + _mm_storeu_si128((__m128i*)(outptr0 + j), s0); + _mm_storeu_si128((__m128i*)(outptr1 + j), s1); + _mm_storeu_si128((__m128i*)(outptr2 + j), s2); + } + + for( ; j <= blockSize - 2; j += 2 ) + { + const int8_t* rptr0 = rowbuf + j*vecsize_aligned; + const int8_t* rptr1 = rowbuf + (j+1)*vecsize_aligned; + int s00, s01, s10, s11, s20, s21; + + if( initOutput ) + { + s00 = s01 = bias0; + s10 = s11 = bias1; + s20 = s21 = bias2; + } + else + { + s00 = outptr0[j]; s01 = outptr0[j+1]; + s10 = outptr1[j]; s11 = outptr1[j+1]; + s20 = outptr2[j]; s21 = outptr2[j+1]; + } + + for( int k = 0; k < vecsize; k++ ) + { + int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k]; + int8_t r = rptr0[k]; + s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r; + r = rptr1[k]; + s01 += (int)w0*r; s11 += (int)w1*r; s21 += (int)w2*r; + } + + if( finalOutput ) + { + s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127); + s01 = std::min(std::max(outZp + (int)std::round(s01*mult0), -128), 127); + s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127); + s11 = std::min(std::max(outZp + (int)std::round(s11*mult1), -128), 127); + s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127); + s21 = std::min(std::max(outZp + (int)std::round(s21*mult2), -128), 127); + } + outptr0[j] = s00; + outptr0[j+1] = s01; + outptr1[j] = s10; + outptr1[j+1] = s11; + outptr2[j] = s20; + outptr2[j+1] = s21; + } + + for( ; j < blockSize; j++ ) + { + const int8_t* rptr0 = rowbuf + j*vecsize_aligned; + int s00, s10, s20; + + if( initOutput ) + { + s00 = bias0; + s10 = bias1; + s20 = bias2; + } + else + { + s00 = outptr0[j]; + s10 = outptr1[j]; + s20 = outptr2[j]; + } + + for( int k = 0; k < vecsize; k++ ) + { + int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k]; + int8_t r = rptr0[k]; + s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r; + } + + if( finalOutput ) + { + s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127); + s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127); + s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127); + } + outptr0[j] = s00; + outptr1[j] = s10; + outptr2[j] = s20; + } + } + _mm256_zeroupper(); +} + +static inline void _mm256_expand_mul_add(const __m256i& a, const __m256i& b, + __m256i& out0, __m256i& out1, __m256i& out2, __m256i& out3) +{ + __m256i a0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(a)); + __m256i a1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(a, 1)); + + __m256i b0 = _mm256_cvtepi8_epi16(_mm256_castsi256_si128(b)); + __m256i b1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(b, 1)); + + __m256i a0b0 = _mm256_mullo_epi16(a0, b0); + __m256i a1b1 = _mm256_mullo_epi16(a1, b1); + + out0 = _mm256_add_epi32(out0, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(a0b0))); + out1 = _mm256_add_epi32(out1, _mm256_cvtepi16_epi32(_mm256_extracti128_si256(a0b0, 1))); + out2 = _mm256_add_epi32(out2, _mm256_cvtepi16_epi32(_mm256_castsi256_si128(a1b1))); + out3 = _mm256_add_epi32(out3, _mm256_cvtepi16_epi32(_mm256_extracti128_si256(a1b1, 1))); +} + +static inline void _mm256_load_deinterleave(const int8_t* ptr, __m256i& a, __m256i& b) +{ + __m256i t0 = _mm256_loadu_si256((const __m256i*)ptr); + __m256i t1 = _mm256_loadu_si256((const __m256i*)(ptr + 32)); + + const __m256i sh = _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, + 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); + __m256i p0 = _mm256_shuffle_epi8(t0, sh); + __m256i p1 = _mm256_shuffle_epi8(t1, sh); + __m256i lo = _mm256_permute2x128_si256(p0, p1, 0 + 2*16); + __m256i hi = _mm256_permute2x128_si256(p0, p1, 1 + 3*16); + a = _mm256_unpacklo_epi64(lo, hi); + b = _mm256_unpackhi_epi64(lo, hi); +} + +void fastDepthwiseConv( const int8_t* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const int* biasptr, const float* multptr, + const int8_t* inptr_, + int height, int width, + int* outptr_, + int out_d, int outH, int outW, + int inpZp, int outZp) +{ + const int8_t w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + float mult = multptr[out_d]; + int bias = biasptr[out_d]; + int biasCopy; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const int8_t* imgptr0 = inptr_ + in_i*width; + const int8_t* imgptr1 = imgptr0 + dilation_h*width; + const int8_t* imgptr2 = imgptr0 + (dilation_h*2)*width; + int8_t w00 = w00_, w01 = w01_, w02 = w02_; + int8_t w20 = w20_, w21 = w21_, w22 = w22_; + int out; + biasCopy = bias; + if (in_i < 0) + { + biasCopy += inpZp * (w00 + w01 + w02); + w00 = w01 = w02 = 0; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + biasCopy += inpZp * (w20 + w21 + w22); + w20 = w21 = w22 = 0; + imgptr2 = imgptr1; + } + int* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = (int)imgptr0[0]*w01 + (int)imgptr0[dilation_w]*w02 + + (int)imgptr1[0]*w11 + (int)imgptr1[dilation_w]*w12 + + (int)imgptr2[0]*w21 + (int)imgptr2[dilation_w]*w22 + + biasCopy + inpZp*(w00 + w10 + w20); + outptr[0] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + out_j = 1; + } + + if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) + { + const int VECSZ = 32; + __m256i vw00 = _mm256_set1_epi8(w00), vw01 = _mm256_set1_epi8(w01), vw02 = _mm256_set1_epi8(w02), + vw10 = _mm256_set1_epi8(w10), vw11 = _mm256_set1_epi8(w11), vw12 = _mm256_set1_epi8(w12), + vw20 = _mm256_set1_epi8(w20), vw21 = _mm256_set1_epi8(w21), vw22 = _mm256_set1_epi8(w22); + __m256i vbias = _mm256_set1_epi32(biasCopy), voutzp = _mm256_set1_epi32(outZp), + outmin = _mm256_set1_epi32(-128), outmax = _mm256_set1_epi32(127); + __m256 vmult = _mm256_set1_ps(mult); + __m256i vout0, vout1, vout2, vout3; + + if( stride_w == 1 ) + { + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - VECSZ; + } + int in_j = out_j * stride_w - pad_l; + __m256i v00 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j)), + v01 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j + dilation_w)), + v02 = _mm256_loadu_si256((const __m256i*)(imgptr0 + in_j + dilation_w*2)), + v10 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j)), + v11 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j + dilation_w)), + v12 = _mm256_loadu_si256((const __m256i*)(imgptr1 + in_j + dilation_w*2)), + v20 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j)), + v21 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j + dilation_w)), + v22 = _mm256_loadu_si256((const __m256i*)(imgptr2 + in_j + dilation_w*2)); + + vout0 = vout1 = vout2 = vout3 = vbias; + _mm256_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout0), vmult))); + vout1 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout1), vmult))); + vout2 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout2), vmult))); + vout3 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout3), vmult))); + + vout0 = _mm256_min_epi32(_mm256_max_epi32(vout0, outmin), outmax); + vout1 = _mm256_min_epi32(_mm256_max_epi32(vout1, outmin), outmax); + vout2 = _mm256_min_epi32(_mm256_max_epi32(vout2, outmin), outmax); + vout3 = _mm256_min_epi32(_mm256_max_epi32(vout3, outmin), outmax); + + _mm256_storeu_si256((__m256i*)(outptr + out_j), vout0); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 8), vout1); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 16), vout2); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 24), vout3); + } + } + else + { + for( ; out_j < outW1; out_j += VECSZ ) + { + if (out_j + VECSZ > outW1) + { + if (out_j <= pad_l) + break; + out_j = outW1 - VECSZ; + } + int in_j = out_j * stride_w - pad_l; + __m256i v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; + _mm256_load_deinterleave(imgptr0 + in_j, v00, v01); + _mm256_load_deinterleave(imgptr0 + in_j + 2, v02, unused); + _mm256_load_deinterleave(imgptr1 + in_j, v10, v11); + _mm256_load_deinterleave(imgptr1 + in_j + 2, v12, unused); + _mm256_load_deinterleave(imgptr2 + in_j, v20, v21); + _mm256_load_deinterleave(imgptr2 + in_j + 2, v22, unused); + + vout0 = vout1 = vout2 = vout3 = vbias; + _mm256_expand_mul_add(v00, vw00, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v01, vw01, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v02, vw02, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v10, vw10, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v11, vw11, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v12, vw12, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v20, vw20, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v21, vw21, vout0, vout1, vout2, vout3); + _mm256_expand_mul_add(v22, vw22, vout0, vout1, vout2, vout3); + + vout0 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout0), vmult))); + vout1 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout1), vmult))); + vout2 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout2), vmult))); + vout3 = _mm256_add_epi32(voutzp, _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(vout3), vmult))); + + vout0 = _mm256_min_epi32(_mm256_max_epi32(vout0, outmin), outmax); + vout1 = _mm256_min_epi32(_mm256_max_epi32(vout1, outmin), outmax); + vout2 = _mm256_min_epi32(_mm256_max_epi32(vout2, outmin), outmax); + vout3 = _mm256_min_epi32(_mm256_max_epi32(vout3, outmin), outmax); + + _mm256_storeu_si256((__m256i*)(outptr + out_j), vout0); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 8), vout1); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 16), vout2); + _mm256_storeu_si256((__m256i*)(outptr + out_j + 24), vout3); + } + } + } + + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = (int)imgptr0[in_j]*w00 + (int)imgptr0[in_j + dilation_w]*w01 + (int)imgptr0[in_j + dilation_w*2]*w02 + + (int)imgptr1[in_j]*w10 + (int)imgptr1[in_j + dilation_w]*w11 + (int)imgptr1[in_j + dilation_w*2]*w12 + + (int)imgptr2[in_j]*w20 + (int)imgptr2[in_j + dilation_w]*w21 + (int)imgptr2[in_j + dilation_w*2]*w22 + biasCopy; + outptr[out_j] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + int s0 = 1, s1 = 1, s2 = 1; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0; + biasCopy += inpZp*(w00 + w10 + w20); + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0; + biasCopy += inpZp*(w01 + w11 + w21); + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0; + biasCopy += inpZp*(w02 + w12 + w22); + } + out = (int)imgptr0[in_j0]*w00*s0 + (int)imgptr0[in_j1]*w01*s1 + (int)imgptr0[in_j2]*w02*s2 + + (int)imgptr1[in_j0]*w10*s0 + (int)imgptr1[in_j1]*w11*s1 + (int)imgptr1[in_j2]*w12*s2 + + (int)imgptr2[in_j0]*w20*s0 + (int)imgptr2[in_j1]*w21*s1 + (int)imgptr2[in_j2]*w22*s2 + biasCopy; + outptr[out_j] = std::min(std::max(outZp + (int)std::round(out*mult), -128), 127); + } + } + _mm256_zeroupper(); +} + +// dst = vec * weights^t + bias +void fastGEMM1T( const int8_t* vec, const int8_t* weights, + size_t wstep, const int* bias, const float* multiplier, + int* dst, int nvecs, int vecsize, int outZp ) +{ + int i = 0; + + for( ; i <= nvecs - 8; i += 8 ) + { + const int8_t* wptr = weights + i*wstep; + __m256i vs0 = _mm256_setzero_si256(), vs1 = _mm256_setzero_si256(), + vs2 = _mm256_setzero_si256(), vs3 = _mm256_setzero_si256(), + vs4 = _mm256_setzero_si256(), vs5 = _mm256_setzero_si256(), + vs6 = _mm256_setzero_si256(), vs7 = _mm256_setzero_si256(); + + __m128i voutzp = _mm_set1_epi32(outZp); + __m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127); + + for( int k = 0; k < vecsize; k += 32, wptr += 32 ) + { + __m256i v = _mm256_load_si256((const __m256i*)(vec + k)); + + vs0 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)wptr), v, vs0); + vs1 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep)), v, vs1); + vs2 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*2)), v, vs2); + vs3 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*3)), v, vs3); + vs4 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*4)), v, vs4); + vs5 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*5)), v, vs5); + vs6 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*6)), v, vs6); + vs7 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)(wptr + wstep*7)), v, vs7); + } + + __m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs1), _mm256_hadd_epi32(vs2, vs3)); + __m256i s1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs4, vs5), _mm256_hadd_epi32(vs6, vs7)); + + s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1)); + s1 = _mm256_add_epi32(s1, _mm256_permute2x128_si256(s1, s1, 1)); + + __m128i t0 = _mm_add_epi32(_mm256_castsi256_si128(s0), _mm_loadu_si128((__m128i*)(bias + i))); + __m128i t1 = _mm_add_epi32(_mm256_castsi256_si128(s1), _mm_loadu_si128((__m128i*)(bias + i + 4))); + + t0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t0), _mm_loadu_ps(multiplier + i)))); + t1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t1), _mm_loadu_ps(multiplier + i + 4)))); + + t0 = _mm_min_epi32(_mm_max_epi32(t0, outmin), outmax); + t1 = _mm_min_epi32(_mm_max_epi32(t1, outmin), outmax); + + _mm_storeu_si128((__m128i*)(dst + i), t0); + _mm_storeu_si128((__m128i*)(dst + i + 4), t1); + } + + for( ; i < nvecs; i++ ) + { + const int8_t* wptr = weights + i*wstep; + __m256i vs0 = _mm256_setzero_si256(); + + for( int k = 0; k < vecsize; k += 32, wptr += 32 ) + { + __m256i v = _mm256_load_si256((const __m256i*)(vec + k)); + vs0 = _mm256_fmaddepi8_epi32(_mm256_load_si256((const __m256i*)wptr), v, vs0); + } + + __m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs0), vs0); + s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1)); + int temp = _mm_extract_epi32(_mm256_castsi256_si128(s0), 0); + dst[i] = outZp + (int)std::round((temp + bias[i]) * multiplier[i]); + } + + _mm256_zeroupper(); +} +#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY + +CV_CPU_OPTIMIZATION_NAMESPACE_END +}} // namespace diff --git a/modules/dnn/src/int8layers/pooling_layer.cpp b/modules/dnn/src/int8layers/pooling_layer.cpp new file mode 100644 index 0000000000..20a0486a46 --- /dev/null +++ b/modules/dnn/src/int8layers/pooling_layer.cpp @@ -0,0 +1,595 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include "opencv2/core/hal/intrin.hpp" + +#include +#include +#include +using std::max; +using std::min; + +namespace cv +{ +namespace dnn +{ + +class PoolingLayerInt8Impl CV_FINAL : public PoolingLayerInt8 +{ +public: + PoolingLayerInt8Impl(const LayerParams& params) + { + computeMaxIdx = false; + globalPooling = false; + isGlobalPooling = std::vector(3, false); + output_zp = params.get("zeropoints"); + input_zp = params.get("input_zeropoint", 0); + multiplier = params.get("multiplier", 1.f); + + hasDynamicShapes = params.get("has_dynamic_shapes", false); + shapesInitialized = !hasDynamicShapes; + + if (params.has("pool") || params.has("kernel_size") || + params.has("kernel_w") || params.has("kernel_h")) + { + String pool = toLowerCase(params.get("pool", "max")); + if (pool == "max") + type = MAX; + else if (pool == "ave") + type = AVE; + else if (pool == "sum") + type = SUM; + else + CV_Error(Error::StsBadArg, "Unknown pooling type \"" + pool + "\""); + + getPoolingKernelParams(params, kernel_size, isGlobalPooling, pads_begin, pads_end, strides, padMode); + globalPooling = isGlobalPooling[0] || isGlobalPooling[1] || isGlobalPooling[2]; + } + else + CV_Error(Error::StsBadArg, "Cannot determine pooling type"); + setParamsFrom(params); + ceilMode = params.get("ceil_mode", true); + spatialScale = params.get("spatial_scale", 1); + avePoolPaddedArea = params.get("ave_pool_padded_area", true); + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + CV_Assert(!inputs.empty()); + CV_Assert(outputs.size() == 1); + + std::vector inp; + std::vector out; + for (int i = 2; i < inputs[0].dims; i++) { + inp.push_back(inputs[0].size[i]); + out.push_back(outputs[0].size[i]); + } + if (globalPooling) { + std::vector finalKernel; + for (int i = 0; i < inp.size(); i++) { + int idx = isGlobalPooling.size() - inp.size() + i; + finalKernel.push_back(isGlobalPooling[idx] ? inp[i] : kernel_size[idx]); + } + kernel_size = finalKernel; + } + + getConvPoolPaddings(inp, kernel_size, strides, padMode, pads_begin, pads_end); + + if (inputs[0].dims == 3) + { + // Pool1D + kernel_size.assign(1, kernel_size[0]); + strides.assign(1, strides[0]); + pads_begin.assign(1, pads_begin[0]); + pads_end.assign(1, pads_end[0]); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + if (backendId == DNN_BACKEND_OPENCV) + { + if (kernel_size.size() == 3) + return preferableTarget == DNN_TARGET_CPU; + if (kernel_size.size() <= 2) + return true; + else + return false; + } + return false; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + switch (type) + { + case MAX: + { + CV_Assert_N(inputs.size() == 1, outputs.size() == 1); + maxPooling(inputs[0], outputs[0]); + break; + } + case AVE: case SUM: + CV_Assert_N(inputs.size() == 1, outputs.size() == 1); + avePooling(inputs[0], outputs[0]); + break; + default: + CV_Error(Error::StsNotImplemented, "Not implemented"); + break; + } + } + + class PoolingInvoker : public ParallelLoopBody + { + public: + const Mat* src, *rois; + Mat *dst; + int pad_l, pad_t, pad_r, pad_b; + bool avePoolPaddedArea; + int nstripes, inpZp, outZp; + std::vector ofsbuf; + int poolingType; + float multiplier; + float spatialScale; + + std::vector pads_begin, pads_end; + std::vector kernel_size; + std::vector strides; + + PoolingInvoker() : src(0), rois(0), dst(0), pad_l(0), pad_t(0), pad_r(0), pad_b(0), + avePoolPaddedArea(false), nstripes(0), inpZp(0), outZp(0), + poolingType(MAX), multiplier(1), spatialScale(0){} + + static void run(const Mat& src, const Mat& rois, Mat& dst, + std::vector kernel_size, std::vector strides, + std::vector pads_begin, std::vector pads_end, + bool avePoolPaddedArea, int poolingType, float spatialScale, + float multiplier, int inpZp, int outZp, int nstripes) + { + CV_Assert_N( + src.isContinuous(), dst.isContinuous(), + src.type() == CV_8S, src.type() == dst.type(), + src.dims == 3 || src.dims == 4 || src.dims == 5, dst.dims == 3 || dst.dims == 4 || dst.dims == 5, + src.size[0] == dst.size[0], src.size[1] == dst.size[1], rois.empty()); + + PoolingInvoker p; + + bool isPool1D = src.dims == 3; + bool isPool3D = src.dims == 5; + + p.src = &src; + p.rois = &rois; + p.dst = &dst; + + p.kernel_size = kernel_size; + p.strides = strides; + p.pads_begin = pads_begin; + p.pads_end = pads_end; + + p.pad_l = pads_begin.back(); + p.pad_t = isPool1D ? 0 : pads_begin[pads_begin.size() - 2]; + p.pad_r = pads_end.back(); + p.pad_b = isPool1D ? 0 : pads_end[pads_end.size() - 2]; + + p.avePoolPaddedArea = avePoolPaddedArea; + p.nstripes = nstripes; + p.inpZp = inpZp; + p.outZp = outZp; + p.poolingType = poolingType; + p.spatialScale = spatialScale; + p.multiplier = multiplier; + + int height = isPool1D ? 1 : src.size[src.dims - 2]; + int width = src.size[src.dims - 1]; + + int kernel_d = isPool3D ? kernel_size[0] : 1; + int kernel_h = isPool1D ? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + p.ofsbuf.resize(kernel_d * kernel_h * kernel_w); + for (int i = 0; i < kernel_d; ++i) { + for (int j = 0; j < kernel_h; ++j) { + for (int k = 0; k < kernel_w; ++k) { + p.ofsbuf[i * kernel_h * kernel_w + j * kernel_w + k] = width * height * i + width * j + k; + } + } + } + + parallel_for_(Range(0, nstripes), p, nstripes); + } + + void operator()(const Range& r) const CV_OVERRIDE + { + int channels = dst->size[1]; + + bool isPool3D = src->dims == 5; + bool isPool2D = src->dims == 4; + bool isPool1D = src->dims == 3; + int depth = isPool3D? dst->size[2] : 1; + int height = isPool1D? 1 : dst->size[dst->dims - 2]; + int width = dst->size[dst->dims - 1]; + + int inp_depth = isPool3D? src->size[2] : 1; + int inp_height = isPool1D? 1 : src->size[src->dims - 2]; + int inp_width = src->size[src->dims - 1]; + + size_t total = dst->total(); + size_t stripeSize = (total + nstripes - 1)/nstripes; + size_t stripeStart = r.start*stripeSize; + size_t stripeEnd = std::min(r.end*stripeSize, total); + + int kernel_d = isPool3D? kernel_size[0] : 1; + int kernel_h = isPool1D? 1 : kernel_size[kernel_size.size() - 2]; + int kernel_w = kernel_size.back(); + + int stride_d = isPool3D? strides[0] : 0; + int stride_h = isPool1D? 1 :strides[strides.size() - 2]; + int stride_w = strides.back(); + +#if CV_SIMD128 + const int* ofsptr = (const int*)&ofsbuf[0]; + if (poolingType == MAX && !ofsptr) + CV_Error(Error::StsBadArg, "ofsbuf should be initialized in this mode"); +#endif + + for( size_t ofs0 = stripeStart; ofs0 < stripeEnd; ) + { + size_t ofs = ofs0; + int x0 = (int)(ofs % width); + ofs /= width; + int y0 = (int)(ofs % height); + ofs /= height; + + int d0 = (int)(ofs % depth); + ofs /= depth; + + int c = (int)(ofs % channels); + int n = (int)(ofs / channels); + int ystart, yend; + int dstart = 0, dend = 1; + + const int8_t *srcData = 0; + int pad_d_begin = (pads_begin.size() == 3) ? pads_begin[0] : 0; + dstart = d0 * stride_d - pad_d_begin; + dend = min(dstart + kernel_d, (int)(inp_depth + pads_end[0])); + + ystart = y0 * stride_h - pad_t; + yend = min(ystart + kernel_h, inp_height + pad_b); + srcData = src->ptr(n, c); + + int ddelta = dend - dstart; + dstart = max(dstart, 0); + dend = min(dend, inp_depth); + int ydelta = yend - ystart; + ystart = max(ystart, 0); + yend = min(yend, inp_height); + int8_t *dstData = &dst->ptr(n, c, d0)[y0 * width]; + + int delta = std::min((int)(stripeEnd - ofs0), width - x0); + ofs0 += delta; + int x1 = x0 + delta; + + if( poolingType == MAX ) + for( ; x0 < x1; x0++ ) + { + int xstart = x0 * stride_w - pad_l; + int xend = min(xstart + kernel_w, inp_width); + xstart = max(xstart, 0); + if (xstart >= xend || ystart >= yend) + { + dstData[x0] = (int8_t)outZp; + continue; + } +#if CV_SIMD128 + if( isPool2D && xstart > 0 && x0 + 15 < x1 && (x0 + 15) * stride_w - pad_l + kernel_w < inp_width ) + { + v_int8x16 max_val0 = v_setall_s8(-128); + if( yend - ystart == kernel_h ) + { + const int8_t* srcData1 = srcData + ystart*inp_width + xstart; + if( stride_w == 1 ) + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0 = v_load(srcData1 + index); + max_val0 = v_max(max_val0, v0); + } + else if( stride_w == 2 ) + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0, dummy; + v_load_deinterleave(srcData1 + index, v0, dummy); + max_val0 = v_max(max_val0, v0); + } + else + for (int k = 0; k < kernel_w*kernel_h; k++) + { + int index = ofsptr[k]; + v_int8x16 v0(srcData1[index], srcData1[index + stride_w], + srcData1[index + stride_w*2], srcData1[index + stride_w*3], + srcData1[index + stride_w*4], srcData1[index + stride_w*5], + srcData1[index + stride_w*6], srcData1[index + stride_w*7], + srcData1[index + stride_w*8], srcData1[index + stride_w*9], + srcData1[index + stride_w*10], srcData1[index + stride_w*11], + srcData1[index + stride_w*12], srcData1[index + stride_w*13], + srcData1[index + stride_w*14], srcData1[index + stride_w*15]); + max_val0 = v_max(max_val0, v0); + } + } + else + { + for (int y = ystart; y < yend; ++y) + { + for (int x = xstart; x < xend; ++x) + { + const int index = y * inp_width + x; + v_int8x16 v0(srcData[index], srcData[index + stride_w], + srcData[index + stride_w*2], srcData[index + stride_w*3], + srcData[index + stride_w*4], srcData[index + stride_w*5], + srcData[index + stride_w*6], srcData[index + stride_w*7], + srcData[index + stride_w*8], srcData[index + stride_w*9], + srcData[index + stride_w*10], srcData[index + stride_w*11], + srcData[index + stride_w*12], srcData[index + stride_w*13], + srcData[index + stride_w*14], srcData[index + stride_w*15]); + max_val0 = v_max(max_val0, v0); + } + } + } + v_store(dstData + x0, max_val0); + x0 += 15; + } + else +#else + CV_UNUSED(isPool2D); +#endif + if( isPool1D ) + { + const int8_t* first = srcData + xstart; + const int8_t* last = srcData + xend; + const int8_t* max_elem = std::max_element(first, last); + if (max_elem != last) + dstData[x0] = *max_elem; + } + else + { + int8_t max_val = -128; + for (int d = dstart; d < dend; ++d) { + for (int y = ystart; y < yend; ++y) { + for (int x = xstart; x < xend; ++x) { + const int index = d * inp_width * inp_height + y * inp_width + x; + int8_t val = srcData[index]; + max_val = std::max(max_val, val); + } + } + } + dstData[x0] = max_val; + } + } + else if (poolingType == AVE || poolingType == SUM) + { + for( ; x0 < x1; ++x0) + { + int xstart = x0 * stride_w - pad_l; + int xend = min(xstart + kernel_w, inp_width + pad_r); + int xdelta = xend - xstart; + xstart = max(xstart, 0); + xend = min(xend, inp_width); + + int real_kernel_area = (dend - dstart) * (yend - ystart) * (xend - xstart); + int padded_kernel_area = xdelta * ydelta * ddelta; + int kernel_area = avePoolPaddedArea ? padded_kernel_area : real_kernel_area; + + int bias = (avePoolPaddedArea ? (padded_kernel_area - real_kernel_area) * inpZp : 0) + - (inpZp * kernel_area); + float inv_kernel_area = poolingType == AVE ? multiplier / kernel_area : multiplier; +#if CV_SIMD128 + if( isPool2D && xstart > 0 && x0 + 15 < x1 && (x0 + 15) * stride_w - pad_l + kernel_w < inp_width ) + { + v_int32x4 sum_val0 = v_setall_s32(bias), sum_val1 = v_setall_s32(bias), + sum_val2 = v_setall_s32(bias), sum_val3 = v_setall_s32(bias), + voutzp = v_setall_s32(outZp); + v_float32x4 ikarea = v_setall_f32(inv_kernel_area); + + for (int y = ystart; y < yend; ++y) + { + for (int x = xstart; x < xend; ++x) + { + const int index = y * inp_width + x; + v_int32x4 v0((int)srcData[index], (int)srcData[index + stride_w], + (int)srcData[index + stride_w*2], (int)srcData[index + stride_w*3]); + v_int32x4 v1((int)srcData[index + stride_w*4], (int)srcData[index + stride_w*5], + (int)srcData[index + stride_w*6], (int)srcData[index + stride_w*7]); + v_int32x4 v2((int)srcData[index + stride_w*8], (int)srcData[index + stride_w*9], + (int)srcData[index + stride_w*10], (int)srcData[index + stride_w*11]); + v_int32x4 v3((int)srcData[index + stride_w*12], (int)srcData[index + stride_w*13], + (int)srcData[index + stride_w*14], (int)srcData[index + stride_w*15]); + sum_val0 += v0; + sum_val1 += v1; + sum_val2 += v2; + sum_val3 += v3; + } + } + + sum_val0 = v_round(v_cvt_f32(sum_val0)*ikarea) + voutzp; + sum_val1 = v_round(v_cvt_f32(sum_val1)*ikarea) + voutzp; + sum_val2 = v_round(v_cvt_f32(sum_val2)*ikarea) + voutzp; + sum_val3 = v_round(v_cvt_f32(sum_val3)*ikarea) + voutzp; + + v_store(dstData + x0, v_pack(v_pack(sum_val0, sum_val1), v_pack(sum_val2, sum_val3))); + x0 += 15; + } + else +#endif + if( isPool1D ) + { + const int8_t* first = srcData + xstart; + const int8_t* last = srcData + xend; + int sum_val = bias + std::accumulate(first, last, 0); + dstData[x0] = saturate_cast(outZp + std::round(sum_val*inv_kernel_area)); + } + else + { + int sum_val = bias; + for (int d = dstart; d < dend; ++d) { + for (int y = ystart; y < yend; ++y) { + for (int x = xstart; x < xend; ++x) { + const int index = d * inp_width * inp_height + y * inp_width + x; + int8_t val = srcData[index]; + sum_val += (int)val; + } + } + } + dstData[x0] = saturate_cast(outZp + std::round(sum_val*inv_kernel_area)); + } + } + } + } + } + }; + + void maxPooling(Mat &src, Mat &dst) + { + const int nstripes = getNumThreads(); + Mat rois; + PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type, + spatialScale, multiplier, input_zp, output_zp, nstripes); + } + + void avePooling(Mat &src, Mat &dst) + { + const int nstripes = getNumThreads(); + Mat rois; + PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type, + spatialScale, multiplier, input_zp, output_zp, nstripes); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() != 0); + + bool isPool1D = inputs[0].size() == 3; + std::vector inpShape(inputs[0].begin() + 2, inputs[0].end()); + std::vector outShape(inputs[0].begin(), inputs[0].begin() + 2); + + std::vector local_kernel; + if (globalPooling) { + for (int i = 0; i < inpShape.size(); i++) { + int idx = isGlobalPooling.size() - inpShape.size() + i; + local_kernel.push_back(isGlobalPooling[idx] ? inpShape[i] : kernel_size[idx]); + } + } else { + local_kernel = kernel_size; + } + + if (hasDynamicShapes && !shapesInitialized) + { + //Just copy input shapes for width and height to prevent errors on loading stage + for (int i = 0; i < inpShape.size(); i++) + outShape.push_back(inpShape[i]); + } + else if (padMode.empty()) + { + int addedDims = isPool1D? inpShape.size() : local_kernel.size(); + for (int i = 0; i < addedDims; i++) { + float dst = (float) (inpShape[i] + pads_begin[i] + pads_end[i] - local_kernel[i]) / strides[i]; + outShape.push_back(1 + (ceilMode ? ceil(dst) : floor(dst))); + } + + // If we have padding, ensure that the last pooling starts strictly + // inside the image (instead of at the padding); otherwise clip the last. + for (int i = 0; i < addedDims; i++) { + if (pads_end[i] && (outShape[2 + i] - 1) * strides[i] >= inpShape[i] + pads_end[i]) { + --outShape[2 + i]; + CV_Assert((outShape[2 + i] - 1) * strides[i] < inpShape[i] + pads_end[i]); + } + } + } + else { + getConvPoolOutParams(inpShape, local_kernel, strides, padMode, + std::vector(local_kernel.size(), 1), outShape); + } + + outputs.assign(1, outShape); + return false; + } + + bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + int dims = inputs[0].size(); + CV_Assert(inputs[0][dims - 1] > 0 && inputs[0][dims - 2] > 0); + shapesInitialized = true; + return true; + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(inputs); // suppress unused variable warning + long flops = 0; + bool isPool1D = inputs[0].size() == 3; + size_t karea = std::accumulate(kernel_size.begin(), isPool1D? kernel_size.begin() + 1 : kernel_size.end(), + 1, std::multiplies()); + for(int i = 0; i < outputs.size(); i++) + { + if (type == MAX) + { + if (i%2 == 0) + flops += total(outputs[i])*karea; + } + else + { + flops += total(outputs[i])*(karea + 1); + } + } + return flops; + } +private: + enum Type + { + MAX, + AVE, + STOCHASTIC, + SUM, + ROI, // RoI pooling, https://arxiv.org/pdf/1504.08083.pdf + PSROI // Position-sensitive RoI pooling, https://arxiv.org/pdf/1605.06409.pdf + }; + bool hasDynamicShapes; + bool shapesInitialized; + float multiplier; +}; + +Ptr PoolingLayerInt8::create(const LayerParams& params) +{ + return Ptr(new PoolingLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/quantization_utils.cpp b/modules/dnn/src/int8layers/quantization_utils.cpp new file mode 100644 index 0000000000..0346f147ba --- /dev/null +++ b/modules/dnn/src/int8layers/quantization_utils.cpp @@ -0,0 +1,210 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +namespace cv +{ +namespace dnn +{ + +// Quantize FP32/FP16 Inputs to INT8 +class QuantizeLayerImpl CV_FINAL : public QuantizeLayer +{ +public: + QuantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scales", 1.0f); + zeropoint = params.get("zeropoints", 0); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + +#ifdef HAVE_OPENCL + bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) + { + std::vector inputs, outputs; + inputs_.getUMatVector(inputs); + outputs_.getUMatVector(outputs); + + if (inputs_.depth() == CV_16S) + { + UMat inputFp32(shape(inputs[0]), CV_32F); + convertFp16(inputs[0], inputFp32); + inputFp32.copyTo(inputs[0]); + } + + inputs[0].convertTo(outputs[0], CV_8S, 1.f/scale, zeropoint); + return true; + } +#endif + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + forward_ocl(inputs_arr, outputs_arr, internals_arr)) + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_8S, 1.f/scale, zeropoint); + } +}; + +// Dequantize INT8 Inputs to FP32/FP16 +class DequantizeLayerImpl CV_FINAL : public DequantizeLayer +{ +public: + DequantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scales", 1.0f); + zeropoint = params.get("zeropoints", 0); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + +#ifdef HAVE_OPENCL + bool forward_ocl(InputArrayOfArrays inputs_, OutputArrayOfArrays outputs_, OutputArrayOfArrays internals_) + { + std::vector inputs, outputs; + inputs_.getUMatVector(inputs); + outputs_.getUMatVector(outputs); + + UMat outputFp32(shape(outputs[0]), CV_32F); + inputs[0].convertTo(outputFp32, CV_32F, scale, -(scale*zeropoint)); + + if (outputs_.depth() == CV_16S) + convertFp16(outputFp32, outputs[0]); + else + outputFp32.copyTo(outputs[0]); + return true; + } +#endif + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + forward_ocl(inputs_arr, outputs_arr, internals_arr)) + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_32F, scale, -(scale*zeropoint)); + } +}; + +// Rescale/Requantize INT8 Inputs from (scale1, zeropoint1) to (scale2, zeropoint2) +class RequantizeLayerImpl CV_FINAL : public RequantizeLayer +{ +public: + RequantizeLayerImpl(const LayerParams& params) + { + scale = params.get("scale", 1.f); + shift = params.get("shift", 0.f); + setParamsFrom(params); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return false; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE + { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + inputs[0].convertTo(outputs[0], CV_8S, scale, shift); + } +}; + +Ptr QuantizeLayer::create(const LayerParams& params) +{ + return Ptr(new QuantizeLayerImpl(params)); +} + +Ptr DequantizeLayer::create(const LayerParams& params) +{ + return Ptr(new DequantizeLayerImpl(params)); +} + +Ptr RequantizeLayer::create(const LayerParams& params) +{ + return Ptr(new RequantizeLayerImpl(params)); +} + +} +} diff --git a/modules/dnn/src/int8layers/scale_layer.cpp b/modules/dnn/src/int8layers/scale_layer.cpp new file mode 100644 index 0000000000..d7f676d047 --- /dev/null +++ b/modules/dnn/src/int8layers/scale_layer.cpp @@ -0,0 +1,211 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include +#include + +namespace cv +{ +namespace dnn +{ + +class ScaleLayerInt8Impl CV_FINAL : public ScaleLayerInt8 +{ +public: + Mat weights, bias; + ScaleLayerInt8Impl(const LayerParams& params) + { + setParamsFrom(params); + hasBias = params.get("bias_term", false); + axis = params.get("axis", 1); + hasWeights = false; + + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + + DictValue inpSc = params.get("input_scales"); + DictValue inpZp = params.get("input_zeropoints"); + + for (int i = 0; i < inpSc.size(); i++) + { + inp_sc.push_back(inpSc.get(i)); + inp_zp.push_back(inpZp.get(i)); + } + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + outputs.assign(1, inputs[0]); + return true; + } + + virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector inputs; + inputs_arr.getMatVector(inputs); + hasWeights = blobs.size() == 2 || (blobs.size() <= 1 && !hasBias); + CV_Assert((inputs.size() == 2 && blobs.empty()) || blobs.size() == (int)hasWeights + (int)hasBias); + + if (!blobs.empty()) + { + Mat w = hasWeights ? blobs[0] : Mat::ones(blobs[0].size(), CV_32F); + Mat b = hasBias ? blobs.back() : Mat::zeros(blobs.back().size(), CV_32F); + + w = w.reshape(1, 1); + b = b.reshape(1, 1); + + w.convertTo(weights, CV_32F, inp_sc[0]/output_sc); + addWeighted(b, 1.0/output_sc, weights, -inp_zp[0], output_zp, bias, CV_32F); + } + else + { + // initialized during forward() + weights = Mat(); bias = Mat(); + } + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + bool setActivation(const Ptr& layer) CV_OVERRIDE + { + Ptr activ_int8 = layer.dynamicCast(); + if (!activ_int8.empty()) + { + return activ_int8->blobs.empty(); + } + return false; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + Mat &inpBlob = inputs[0]; + Mat &outBlob = outputs[0]; + + if (blobs.empty()) + { + CV_Assert(inp_sc.size() == 2 && inp_zp.size() == 2); + Mat inp_dequantized, w, b; + inputs[1].reshape(1, 1).convertTo(inp_dequantized, CV_32F, inp_sc[1], -(inp_sc[1]*inp_zp[1])); + w = hasWeights ? inp_dequantized : Mat::ones(inp_dequantized.size(), CV_32F); + b = hasBias ? inp_dequantized : Mat::zeros(inp_dequantized.size(), CV_32F); + + w.convertTo(weights, CV_32F, inp_sc[0]/output_sc); + addWeighted(b, 1.0/output_sc, weights, -inp_zp[0], output_zp, bias, CV_32F); + } + + MatShape inpShape = shape(inpBlob); + const int numWeights = weights.total(); + CV_Assert(numWeights != 0); + CV_CheckEQ(weights.total(), bias.total(), "Incompatible weights/bias blobs"); + + int endAxis; + for (endAxis = axis + 1; endAxis <= inpBlob.dims; ++endAxis) + { + if (total(inpShape, axis, endAxis) == numWeights) + break; + } + CV_Assert(total(inpShape, axis, endAxis) == numWeights); + CV_CheckTypeEQ(inpBlob.type(), CV_8SC1, ""); CV_CheckTypeEQ(outBlob.type(), CV_8SC1, ""); + + int numSlices = total(inpShape, 0, axis); + int8_t* inpData = (int8_t*)inpBlob.data; + int8_t* outData = (int8_t*)outBlob.data; + + if (endAxis != inpBlob.dims) + { + float* weightsData = (float*)weights.data; + float* biasesData = (float*)bias.data; + int spatialSize = total(inpShape, endAxis); // spatialSize != 1 + for (int i = 0; i < numSlices; ++i) + { + for (int j = 0; j < numWeights; ++j) + { + float w = weightsData[j]; + float b = biasesData[j]; + Mat inpSlice(1, spatialSize, CV_8S, inpData); + Mat outSlice(1, spatialSize, CV_8S, outData); + inpSlice.convertTo(outSlice, CV_8S, w, b); + inpData += spatialSize; + outData += spatialSize; + } + } + } + else + { + for (int i = 0; i < numSlices; ++i) + { + Mat inpSlice(1, numWeights, CV_8S, inpData); + Mat outSlice(1, numWeights, CV_8S, outData); + + multiply(inpSlice, weights, outSlice, 1.0, CV_8S); + add(outSlice, bias, outSlice, Mat(), CV_8S); + + inpData += numWeights; + outData += numWeights; + } + } + } + + void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + scale = (hasWeights && !blobs.empty()) ? blobs[0] : Mat(); + shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat(); + } + + void getScaleZeropoint(float& scale, int& zeropoint) const CV_OVERRIDE + { + scale = output_sc; + zeropoint = output_zp; + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + long flops = 0; + for(int i = 0; i < inputs.size(); i++) + { + flops += 2*total(inputs[i]); + } + return flops; + } + +private: + bool hasWeights; + std::vector inp_sc; + std::vector inp_zp; +}; + + +Ptr ScaleLayerInt8::create(const LayerParams& params) +{ + return Ptr(new ScaleLayerInt8Impl(params)); +} + +Ptr ShiftLayerInt8::create(const LayerParams& params) +{ + LayerParams scaleParams = params; + scaleParams.type = "ScaleInt8"; + scaleParams.set("bias_term", true); + scaleParams.set("axis", 0); + return Ptr(new ScaleLayerInt8Impl(scaleParams)); +} + +} // namespace dnn +} // namespace cv diff --git a/modules/dnn/src/int8layers/softmax_layer.cpp b/modules/dnn/src/int8layers/softmax_layer.cpp new file mode 100644 index 0000000000..7e3c82bc21 --- /dev/null +++ b/modules/dnn/src/int8layers/softmax_layer.cpp @@ -0,0 +1,176 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include +#include + +namespace cv +{ +namespace dnn +{ + +class SoftMaxLayerInt8Impl CV_FINAL : public SoftmaxLayerInt8 +{ +public: + + SoftMaxLayerInt8Impl(const LayerParams& params) + { + axisRaw = params.get("axis", 1); + logSoftMax = params.get("log_softmax", false); + output_sc = params.get("scales"); + output_zp = params.get("zeropoints"); + setParamsFrom(params); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + bool inplace = Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + MatShape shape = inputs[0]; + int cAxis = normalize_axis(axisRaw, shape.size()); + shape[cAxis] = 1; + internals.assign(1, shape); + return inplace; + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + Ptr dequantize_layer = top.dynamicCast(); + return !dequantize_layer.empty() && preferableTarget != DNN_TARGET_OPENCL_FP16; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs, internals; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + internals_arr.getMatVector(internals); + + const Mat &src = inputs[0]; + Mat &dst = outputs[0]; + + int axis = normalize_axis(axisRaw, src.dims); + size_t outerSize = src.total(0, axis), channels = src.size[axis], + innerSize = src.total(axis + 1); + + CV_Assert(src.type() == CV_8S && (dst.type() == CV_8S || dst.type() == CV_32F)); + CV_Assert(src.isContinuous() && dst.isContinuous()); + + size_t outerStep = src.total(axis); + size_t cnStep = src.total(axis + 1); + const int8_t *srcPtr = src.ptr(); + const float *expPtr = blobs[0].ptr(); + + if (dst.type() == CV_32F) + { + float *dstPtr = dst.ptr(); + for (size_t outerDim = 0; outerDim < outerSize; outerDim++) + { + size_t srcOffset = outerDim * outerStep; + std::vector expSum(innerSize, 0.f); + + // sum exp along axis + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + expSum[i] += expPtr[srcPtr[offset + i] + 128]; + } + + // divide by computed sum + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = expPtr[srcPtr[offset + i] + 128]/expSum[i]; + } + + if (logSoftMax) + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = log(dstPtr[offset + i]); + } + } + } + } + else + { + const float inv_scale = 1.f/output_sc; + int8_t *dstPtr = dst.ptr(); + for (size_t outerDim = 0; outerDim < outerSize; outerDim++) + { + size_t srcOffset = outerDim * outerStep; + std::vector expSum(innerSize, 0.f); + + // sum exp along axis + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + expSum[i] += expPtr[srcPtr[offset + i] + 128]; + } + + // divide by computed sum and quantize to int8 + if (logSoftMax) + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = saturate_cast(output_zp + std::round(inv_scale*log(expPtr[srcPtr[offset + i] + 128]/expSum[i]))); + } + } + else + { + for (size_t cnDim = 0; cnDim < channels; cnDim++) + { + const int offset = srcOffset + cnDim * cnStep; + for (size_t i = 0; i < innerSize; i++) + dstPtr[offset + i] = saturate_cast(output_zp + std::round(inv_scale*(expPtr[srcPtr[offset + i] + 128]/expSum[i]))); + } + } + } + } + } + + int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_UNUSED(outputs); // suppress unused variable warning + int64 flops = 0; + + for (int i = 0; i < inputs.size(); i++) + { + flops += 4*total(inputs[i]); + } + + return flops; + } + + int axisRaw; +}; + +Ptr SoftmaxLayerInt8::create(const LayerParams& params) +{ + return Ptr(new SoftMaxLayerInt8Impl(params)); +} + +} +} diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp index edd9948db1..57a95b15ee 100644 --- a/modules/dnn/src/layers/batch_norm_layer.cpp +++ b/modules/dnn/src/layers/batch_norm_layer.cpp @@ -35,6 +35,7 @@ namespace dnn class BatchNormLayerImpl CV_FINAL : public BatchNormLayer { public: + Mat origin_weights, origin_bias; Mat weights_, bias_; UMat umat_weight, umat_bias; mutable int dims; @@ -88,11 +89,11 @@ public: const float* weightsData = hasWeights ? blobs[weightsBlobIndex].ptr() : 0; const float* biasData = hasBias ? blobs[biasBlobIndex].ptr() : 0; - weights_.create(1, (int)n, CV_32F); - bias_.create(1, (int)n, CV_32F); + origin_weights.create(1, (int)n, CV_32F); + origin_bias.create(1, (int)n, CV_32F); - float* dstWeightsData = weights_.ptr(); - float* dstBiasData = bias_.ptr(); + float* dstWeightsData = origin_weights.ptr(); + float* dstBiasData = origin_bias.ptr(); for (size_t i = 0; i < n; ++i) { @@ -100,15 +101,12 @@ public: dstWeightsData[i] = w; dstBiasData[i] = (hasBias ? biasData[i] : 0.0f) - w * meanData[i] * varMeanScale; } - // We will use blobs to store origin weights and bias to restore them in case of reinitialization. - weights_.copyTo(blobs[0].reshape(1, 1)); - bias_.copyTo(blobs[1].reshape(1, 1)); } virtual void finalize(InputArrayOfArrays, OutputArrayOfArrays) CV_OVERRIDE { - blobs[0].reshape(1, 1).copyTo(weights_); - blobs[1].reshape(1, 1).copyTo(bias_); + origin_weights.reshape(1, 1).copyTo(weights_); + origin_bias.reshape(1, 1).copyTo(bias_); } void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE @@ -240,7 +238,7 @@ public: kernel.set(4, ocl::KernelArg::PtrReadOnly(umat_weight)); kernel.set(5, ocl::KernelArg::PtrReadOnly(umat_bias)); kernel.set(6, ocl::KernelArg::PtrWriteOnly(dst)); - bool ret = kernel.run(2, global, NULL, false); + bool ret = kernel.run_(2, global, NULL, false); if (!ret) return false; } @@ -411,6 +409,18 @@ public: } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + params.set("input_scale", scales[0][0]); + params.set("input_zeropoint", zeropoints[0][0]); + + params.blobs.clear(); + params.blobs.push_back(origin_weights); + params.blobs.push_back(origin_bias); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/blank_layer.cpp b/modules/dnn/src/layers/blank_layer.cpp index 5f93b45886..59548a9c0c 100644 --- a/modules/dnn/src/layers/blank_layer.cpp +++ b/modules/dnn/src/layers/blank_layer.cpp @@ -166,6 +166,11 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } }; Ptr BlankLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/concat_layer.cpp b/modules/dnn/src/layers/concat_layer.cpp index a950c56167..536114fcd7 100644 --- a/modules/dnn/src/layers/concat_layer.cpp +++ b/modules/dnn/src/layers/concat_layer.cpp @@ -70,6 +70,7 @@ public: setParamsFrom(params); axis = params.get("axis", 1); padding = params.get("padding", false); + paddingValue = params.get("padding_value", 0); } virtual bool getMemoryShapes(const std::vector &inputs, @@ -119,13 +120,14 @@ public: (backendId == DNN_BACKEND_VKCOM && haveVulkan() && !padding); } + template class ChannelConcatInvoker : public ParallelLoopBody { public: std::vector* inputs; Mat* output; int nstripes; - std::vector chptrs; + std::vector chptrs; static void run(std::vector& inputs, Mat& output, int nstripes) { @@ -139,14 +141,14 @@ public: for( i = 0; i < ninputs; i++ ) { Mat& inp = inputs[i]; - CV_Assert( inp.isContinuous() && (inp.type() == CV_32F || inp.type() == CV_16S) && + CV_Assert( inp.isContinuous() && (inp.type() == CV_32F || inp.type() == CV_16S || inp.type() == CV_8S) && inp.dims == 4 && inp.size[0] == output.size[0] && inp.size[2] == output.size[2] && inp.size[3] == output.size[3] ); nchannels += inp.size[1]; } CV_Assert( nchannels == output.size[1] ); - CV_Assert( output.isContinuous() && (output.type() == CV_32F || output.type() == CV_16S) ); + CV_Assert( output.isContinuous() && (output.type() == CV_32F || output.type() == CV_16S || output.type() == CV_8S) ); cc.chptrs.resize(nchannels*batchsz); @@ -157,7 +159,7 @@ public: for( int j = 0; j < batchsz; j++ ) for( int k = 0; k < inp.size[1]; k++ ) { - const float* ptr = inp.ptr(j, k); + const T* ptr = inp.ptr(j, k); cc.chptrs[ofs + j*nchannels + k] = ptr; } ofs += inp.size[1]; @@ -176,8 +178,8 @@ public: size_t stripeSize = (total + nstripes - 1)/nstripes; size_t stripeStart = r.start*stripeSize; size_t stripeEnd = std::min(total, r.end*stripeSize); - const float** ptrs = (const float**)&chptrs[0]; - float* outptr = output->ptr(); + const T** ptrs = (const T**)&chptrs[0]; + T* outptr = output->ptr(); size_t blockSize0 = 1 << 16; for( size_t ofs0 = stripeStart; ofs0 < stripeEnd; ) @@ -248,7 +250,8 @@ public: CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); - CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && + inputs_arr.depth() != CV_8S, forward_ocl(inputs_arr, outputs_arr, internals_arr)) std::vector inputs, outputs; @@ -259,12 +262,15 @@ public: Mat& outMat = outputs[0]; if (padding) - outMat.setTo(0); + outMat.setTo(paddingValue); if( cAxis == 1 && outMat.dims == 4 && !padding) { int nstripes = getNumThreads(); - ChannelConcatInvoker::run(inputs, outMat, nstripes); + if (outMat.type() == CV_8S) + ChannelConcatInvoker::run(inputs, outMat, nstripes); + else + ChannelConcatInvoker::run(inputs, outMat, nstripes); } else { @@ -394,6 +400,14 @@ public: return Ptr(new InfEngineNgraphNode(concat)); } #endif // HAVE_DNN_NGRAPH + + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (padding) + params.set("padding_value", zeropoints[1][0]); + return true; + } }; Ptr ConcatLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index bbea3e3f2c..18f190b36b 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -112,6 +112,15 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + Mat quantizedBlob; + blobs[0].convertTo(quantizedBlob, CV_8S, 1.f/scales[1][0], zeropoints[1][0]); + params.blobs.clear(); + params.blobs.push_back(quantizedBlob); + return true; + } }; Ptr ConstLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/layers/convolution_layer.cpp b/modules/dnn/src/layers/convolution_layer.cpp index fb57f26511..bb6fd5f5cc 100644 --- a/modules/dnn/src/layers/convolution_layer.cpp +++ b/modules/dnn/src/layers/convolution_layer.cpp @@ -48,6 +48,7 @@ #include "../ie_ngraph.hpp" #include "../op_vkcom.hpp" +#include #include #include "opencv2/core/hal/hal.hpp" @@ -914,11 +915,12 @@ public: bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; int blk_size_cn; ParallelConv() : input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0), - biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false) + biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false) , blk_size_cn(0) {} @@ -976,6 +978,7 @@ public: p.useAVX = checkHardwareSupport(CPU_AVX) && isConv2D; p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D; p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D; + p.useRVV = checkHardwareSupport(CPU_RVV) && isConv2D; int kernel_d = isConv3D? kernel_size[0] : 1; int kernel_h = isConv1D? 1 : kernel_size[kernel_size.size() - 2]; @@ -1176,6 +1179,13 @@ public: stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW); else + #endif + #if CV_TRY_RVV + if(useRVV) + opt_RVV::fastDepthwiseConv(wptr, kernel_h, kernel_w, + stride_h, stride_w, dilation_h, dilation_w, pad_t, pad_l, + biasptr, relu, inptr_, height, width, outptr_, out_d, outH, outW); + else #endif { const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], @@ -1546,6 +1556,12 @@ public: opt_AVX::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, outShape, bsz, vsz, vsz_a, relu, cn0 == 0); else + #endif + #if CV_TRY_RVV + if(useRVV) + opt_RVV::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0, + outShape, bsz, vsz, vsz_a, relu, cn0 == 0); + else #endif for( int i = 0; i < outCn; i += 2 ) { @@ -1721,7 +1737,26 @@ public: config.pad = pad; config.stride = stride; config.dilation = dilation; + if (inputs[0].dims != 4 && inputs[0].dims != umat_blobs[0].dims) + { + static bool bypassCheck = utils::getConfigurationParameterBool("OPENCV_OCL4DNN_CONVOLUTION_IGNORE_INPUT_DIMS_4_CHECK", false); + if (!bypassCheck) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Unsupported configuration: inputs[0].dims=" << inputs[0].dims << " umat_blobs[0].dims=" << umat_blobs[0].dims + << ". Consider reporting complete reproducer to https://github.com/opencv/opencv/issues/20833." + << " You can skip this check temporary through OPENCV_OCL4DNN_CONVOLUTION_IGNORE_INPUT_DIMS_4_CHECK=1" + ); + return false; + } + } config.group = inputs[0].size[1] / umat_blobs[0].size[1]; + if (config.group < 1) // config.group == 0 causes div by zero in ocl4dnn code + { + CV_LOG_WARNING(NULL, "DNN/OpenCL: Unsupported config.group=" << config.group + << ". Consider reporting complete reproducer to https://github.com/opencv/opencv/issues/20833" + ); + return false; + } config.bias_term = umat_blobs.size() == 2; config.use_half = use_half; @@ -2068,6 +2103,48 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + // References - https://arxiv.org/pdf/1712.05877.pdf + + // Quantized convolution with variable weights is not supported. + if (blobs.empty()) + return false; + + float inputScale = scales[0][0], outputScale = scales[1][0]; + int inputZp = zeropoints[0][0]; + params.set("input_zeropoint", inputZp); + + Mat weightsQuantized(weightsMat.rows, weightsMat.cols, CV_8S); + Mat biasQuantized(1, numOutput, CV_32S); + Mat outputMultiplier(1, numOutput, CV_32F); + double realMin, realMax, weightsScale; + + for( int i = 0; i < numOutput; i++ ) + { + // Quantize weights + cv::minMaxIdx(weightsMat.row(i), &realMin, &realMax); + realMin = std::min(realMin, 0.0); + realMax = std::max(realMax, 0.0); + weightsScale = (realMax == realMin) ? 1.0 : std::max(-realMin, realMax)/127; + weightsMat.row(i).convertTo(weightsQuantized.row(i), CV_8S, 1.f/weightsScale); + + // Quantize biases + float biasScale = inputScale * weightsScale; + biasQuantized.at(i) = (int)std::round(biasvec[i]/biasScale) - inputZp*(cv::sum(weightsQuantized.row(i))[0]); + + // Store multiplier + outputMultiplier.at(i) = biasScale / outputScale; + } + + params.blobs.clear(); + params.blobs.push_back(weightsQuantized.reshape(1, shape(blobs[0]))); + params.blobs.push_back(biasQuantized); + params.blobs.push_back(outputMultiplier); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { @@ -2297,6 +2374,7 @@ public: useAVX = checkHardwareSupport(CPU_AVX); useAVX2 = checkHardwareSupport(CPU_AVX2); useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + useRVV = checkHardwareSupport(CPU_RVV); } void operator()(const Range& range_) const CV_OVERRIDE @@ -2328,6 +2406,12 @@ public: if( useAVX ) opt_AVX::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax ); else + #endif + #if CV_TRY_RVV + if( useRVV ) { + opt_RVV::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax ); + } + else #endif for( m = 0; m < mmax; m += 2 ) { @@ -2427,6 +2511,7 @@ public: bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; }; class Col2ImInvoker : public cv::ParallelLoopBody diff --git a/modules/dnn/src/layers/cumsum_layer.cpp b/modules/dnn/src/layers/cumsum_layer.cpp new file mode 100644 index 0000000000..9c70f306d4 --- /dev/null +++ b/modules/dnn/src/layers/cumsum_layer.cpp @@ -0,0 +1,131 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "layers_common.hpp" + +#include + +namespace cv +{ +namespace dnn +{ + +class CumSumLayerImpl CV_FINAL : public CumSumLayer +{ +public: + CumSumLayerImpl(const LayerParams ¶ms) + { + axis_raw = params.get("axis", 0); + exclusive_raw = params.get("exclusive", 0); + reverse_raw = params.get("reverse", 0); + setParamsFrom(params); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + Layer::getMemoryShapes(inputs, requiredOutputs, outputs, internals); + return true; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector inputs, outputs, internals; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + // Get x tensor. + const auto &src_mat = inputs[0]; + const auto *src_ptr = src_mat.ptr(); + + // Get axis. + const int axis = normalize_axis(axis_raw, src_mat.dims); + + // Get y tensor. + auto &dst_mat = outputs[0]; + src_mat.copyTo(dst_mat); + auto *dst_ptr = dst_mat.ptr(); + + // Get flags. + const auto exclusive = exclusive_raw == 1; + const auto reverse = reverse_raw == 1; + + // Get parameters to iterate outer dimension. + const size_t outer_size = src_mat.total(0, axis); + const size_t outer_step_length = src_mat.total(axis); + + // Get parameters to iterate inner dimension. + const size_t inner_size = src_mat.size[axis]; + + if (!inner_size) + return; + + const size_t inner_step_length = src_mat.total(axis + 1); + const int inner_step = (reverse ? -1 : 1) * inner_step_length; + const int inner_start = reverse ? inner_size - 1 : 0; + const int inner_stop = reverse ? -1 : inner_size; + const int inner_delta = reverse ? -1 : 1; + + // Get parameters to populate channels. + const size_t num_channels = src_mat.total(axis + 1); + + for (size_t outer_dim = 0; outer_dim < outer_size; outer_dim++) + { + const size_t outer_offset = outer_dim * outer_step_length; + size_t src_offset = outer_offset + inner_start * inner_step_length; + + // Populate first element of inner dimension. + for (size_t channel = 0; channel < num_channels; channel++) + { + if (exclusive) + { + dst_ptr[src_offset + channel] = 0.0f; + } + else + { + dst_ptr[src_offset + channel] = src_ptr[src_offset + channel]; + src_offset += inner_step; + } + } + + // Populate remaining elements of inner dimension. + for (int inner_dim = inner_start + inner_delta; inner_dim != inner_stop; inner_dim += inner_delta) + { + const size_t dst_offset = outer_offset + inner_dim * inner_step_length; + + for (size_t channel = 0; channel < num_channels; channel++) + { + const size_t previous_dst_offset = dst_offset - inner_step; + dst_ptr[dst_offset + channel] = dst_ptr[previous_dst_offset + channel] + + src_ptr[src_offset + channel]; + src_offset += inner_step; + } + } + } + } + + int axis_raw; + int exclusive_raw; + int reverse_raw; +}; + +Ptr CumSumLayer::create(const LayerParams& params) +{ + return Ptr(new CumSumLayerImpl(params)); +} + +} +} diff --git a/modules/dnn/src/layers/detection_output_layer.cpp b/modules/dnn/src/layers/detection_output_layer.cpp index de97c873af..77d86d5652 100644 --- a/modules/dnn/src/layers/detection_output_layer.cpp +++ b/modules/dnn/src/layers/detection_output_layer.cpp @@ -462,7 +462,7 @@ public: // Retrieve all prior bboxes std::vector priorBBoxes; std::vector > priorVariances; - GetPriorBBoxes(priorData, numPriors, _bboxesNormalized, priorBBoxes, priorVariances); + GetPriorBBoxes(priorData, numPriors, _bboxesNormalized, _varianceEncodedInTarget, priorBBoxes, priorVariances); // Decode all loc predictions to bboxes util::NormalizedBBox clipBounds; @@ -756,7 +756,7 @@ public: CV_Assert(prior_bboxes.size() == prior_variances.size()); CV_Assert(prior_bboxes.size() == bboxes.size()); size_t num_bboxes = prior_bboxes.size(); - CV_Assert(num_bboxes == 0 || prior_variances[0].size() == 4); + CV_Assert(num_bboxes == 0 || prior_variances[0].size() == 4 || variance_encoded_in_target); decode_bboxes.clear(); decode_bboxes.resize(num_bboxes); if(variance_encoded_in_target) { @@ -808,12 +808,13 @@ public: } // Get prior bounding boxes from prior_data - // prior_data: 1 x 2 x num_priors * 4 x 1 blob. + // prior_data: 1 x 1 x num_priors * 4 x 1 blob or 1 x 2 x num_priors * 4 x 1 blob. // num_priors: number of priors. // prior_bboxes: stores all the prior bboxes in the format of util::NormalizedBBox. // prior_variances: stores all the variances needed by prior bboxes. static void GetPriorBBoxes(const float* priorData, const int& numPriors, - bool normalized_bbox, std::vector& priorBBoxes, + bool normalized_bbox, bool variance_encoded_in_target, + std::vector& priorBBoxes, std::vector >& priorVariances) { priorBBoxes.clear(); priorBBoxes.resize(numPriors); @@ -829,13 +830,16 @@ public: bbox.set_size(BBoxSize(bbox, normalized_bbox)); } - for (int i = 0; i < numPriors; ++i) + if (!variance_encoded_in_target) { - int startIdx = (numPriors + i) * 4; - // not needed here: priorVariances[i].clear(); - for (int j = 0; j < 4; ++j) + for (int i = 0; i < numPriors; ++i) { - priorVariances[i].push_back(priorData[startIdx + j]); + int startIdx = (numPriors + i) * 4; + // not needed here: priorVariances[i].clear(); + for (int j = 0; j < 4; ++j) + { + priorVariances[i].push_back(priorData[startIdx + j]); + } } } } diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 9bb5be342f..1b6483a5a0 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -255,6 +255,12 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return func.tryQuantize(scales, zeropoints, params); + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { @@ -288,6 +294,8 @@ struct BaseFunctor bool tryFuse(Ptr&) { return false; } void getScaleShift(Mat&, Mat&) const {} + + bool tryQuantize(const std::vector>&, const std::vector>&, LayerParams&) { return false; } }; struct ReLUFunctor : public BaseFunctor @@ -436,6 +444,29 @@ struct ReLUFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + if (slope != 0.f) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x >= 0.f ? x : slope*x; + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + } + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -559,6 +590,12 @@ struct ReLU6Functor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + return true; + } + int64 getFLOPSPerElement() const { return 2; } }; @@ -651,6 +688,26 @@ struct TanHFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = tanh(x); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -743,6 +800,26 @@ struct SwishFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x / (1.0f + exp(-x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -848,6 +925,28 @@ struct MishFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float eX = exp(x); + float n = (eX + 2) * eX; + float y = (x * n) / (n + 2); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -940,6 +1039,26 @@ struct SigmoidFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = 1.f/(1.f + exp(-x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 3; } }; @@ -1032,6 +1151,26 @@ struct ELUFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x >= 0.f ? x : exp(x) - 1; + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 2; } }; @@ -1130,6 +1269,26 @@ struct AbsValFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = abs(x); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 1; } }; @@ -1223,6 +1382,26 @@ struct BNLLFunctor : public BaseFunctor } #endif // HAVE_VULKAN + bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) + { + float inpScale = scales[0][0], outScale = scales[1][0]; + int inpZp = zeropoints[0][0], outZp = zeropoints[1][0]; + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - inpZp); + float y = x > 0 ? x + log(1. + exp(-x)) : log(1. + exp(x)); + int quantized = outZp + (int)std::round(y/outScale); + table[i+128] = saturate_cast(quantized); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPSPerElement() const { return 5; } }; @@ -1772,7 +1951,7 @@ Ptr ChannelsPReLULayer::create(const LayerParams& params) if (params.blobs[0].total() == 1) { LayerParams reluParams = params; - reluParams.set("negative_slope", params.blobs[0].at(0)); + reluParams.set("negative_slope", *params.blobs[0].ptr()); return ReLULayer::create(reluParams); } Ptr l(new ElementWiseLayer(ChannelsPReLUFunctor(params.blobs[0]))); diff --git a/modules/dnn/src/layers/eltwise_layer.cpp b/modules/dnn/src/layers/eltwise_layer.cpp index a337c48d9e..2c473ff412 100644 --- a/modules/dnn/src/layers/eltwise_layer.cpp +++ b/modules/dnn/src/layers/eltwise_layer.cpp @@ -71,7 +71,8 @@ public: PROD = 0, SUM = 1, MAX = 2, - DIV = 3 + DIV = 3, + MIN = 4, } op; std::vector coeffs; @@ -109,6 +110,8 @@ public: op = SUM; else if (operation == "max") op = MAX; + else if (operation == "min") + op = MIN; else if (operation == "div") op = DIV; else @@ -470,6 +473,13 @@ public: dstptr[j] = std::max(srcptr0[j], srcptrI[j]); } } + else if (op == MIN) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::min(srcptr0[j], srcptrI[j]); + } + } else if (op == SUM) { if (!coeffsptr || (coeffsptr[0] == 1.0f && coeffsptr[1] == 1.0f)) @@ -524,6 +534,13 @@ public: dstptr[j] = std::max(dstptr[j], srcptrI[j]); } } + else if (op == MIN) + { + for (int j = 0; j < blockSize; j++) + { + dstptr[j] = std::min(dstptr[j], srcptrI[j]); + } + } else if (op == SUM) { if (!coeffsptr || coeffsptr[inputIdx] == 1.0f) @@ -641,6 +658,11 @@ public: for (int i = 2; i < inputs.size(); ++i) max(inputs[i], outputs[0], outputs[0]); break; + case MIN: + min(inputs[0], inputs[1], outputs[0]); + for (int i = 2; i < inputs.size(); ++i) + min(inputs[i], outputs[0], outputs[0]); + break; default: return false; } @@ -745,6 +767,7 @@ public: auto op_ = [this] { switch (op) { case MAX: return cuda4dnn::EltwiseOpType::MAX; + case MIN: return cuda4dnn::EltwiseOpType::MIN; case SUM: return cuda4dnn::EltwiseOpType::SUM; case PROD: return cuda4dnn::EltwiseOpType::PRODUCT; case DIV: return cuda4dnn::EltwiseOpType::DIV; @@ -799,6 +822,12 @@ public: for (int i = 2; i < inputBuffers.size(); ++i) topExpr = max(topExpr, inputBuffers[i](x, y, c, n)); break; + case MIN: + topExpr = min(inputBuffers[0](x, y, c, n), + inputBuffers[1](x, y, c, n)); + for (int i = 2; i < inputBuffers.size(); ++i) + topExpr = min(topExpr, inputBuffers[i](x, y, c, n)); + break; default: return Ptr(); } @@ -823,6 +852,8 @@ public: ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::DIV); else if (op == MAX) ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::MAX); + else if (op == MIN) + ieLayer.setEltwiseType(InferenceEngine::Builder::EltwiseLayer::EltwiseType::MIN); else CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation"); @@ -857,6 +888,7 @@ public: case PROD: curr_node = std::make_shared(curr_node, next_node); break; case DIV: curr_node = std::make_shared(curr_node, next_node); break; case MAX: curr_node = std::make_shared(curr_node, next_node); break; + case MIN: curr_node = std::make_shared(curr_node, next_node); break; default: CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation"); } } @@ -864,6 +896,37 @@ public: } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (op == SUM) + { + std::vector newCoeffs; + float offset = zeropoints[1][0]; + float out_sc = scales[1][0]; + for (int i = 0; i < scales[0].size(); i++) + { + float coeff = coeffs.empty() ? 1.f : coeffs[i]; + float newcoeff = (scales[0][i] * coeff) / out_sc; + newCoeffs.push_back(newcoeff); + offset -= (newcoeff * zeropoints[0][i]); + } + params.set("coeff", DictValue::arrayReal(newCoeffs.data(), newCoeffs.size())); + params.set("offset", offset); + return true; + } + else if (op == PROD) + { + std::vector newCoeffs = scales[0]; + newCoeffs[0] /= scales[1][0]; + params.set("coeff", DictValue::arrayReal(newCoeffs.data(), newCoeffs.size())); + params.set("offset", zeropoints[1][0]); + params.set("input_zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + return true; + } + return op == MAX; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/flatten_layer.cpp b/modules/dnn/src/layers/flatten_layer.cpp index 7cf01a14fa..8ff862fab0 100644 --- a/modules/dnn/src/layers/flatten_layer.cpp +++ b/modules/dnn/src/layers/flatten_layer.cpp @@ -227,6 +227,11 @@ virtual Ptr initNgraph(const std::vector >& inp } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } int _startAxis; int _endAxis; diff --git a/modules/dnn/src/layers/fully_connected_layer.cpp b/modules/dnn/src/layers/fully_connected_layer.cpp index d9c1fa65c1..28ea7f347f 100644 --- a/modules/dnn/src/layers/fully_connected_layer.cpp +++ b/modules/dnn/src/layers/fully_connected_layer.cpp @@ -168,7 +168,7 @@ public: class FullyConnected : public ParallelLoopBody { public: - FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false) {} + FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false), useRVV(false) {} static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat, Mat& dstMat, const ActivationLayer* activ, int nstripes) @@ -191,6 +191,7 @@ public: p.useAVX = checkHardwareSupport(CPU_AVX); p.useAVX2 = checkHardwareSupport(CPU_AVX2); p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX; + p.useRVV = checkHardwareSupport(CPU_RVV); parallel_for_(Range(0, nstripes), p, nstripes); } @@ -239,6 +240,11 @@ public: if( useAVX ) opt_AVX::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); else + #endif + #if CV_TRY_RVV + if( useRVV ) + opt_RVV::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize); + else #endif { int i = 0; @@ -293,6 +299,7 @@ public: bool useAVX; bool useAVX2; bool useAVX512; + bool useRVV; }; #ifdef HAVE_OPENCL @@ -611,6 +618,45 @@ public: } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (blobs.empty()) + return false; + + int numOutput = blobs[0].size[0]; + float inputScale = scales[0][0], outputScale = scales[1][0]; + int inputZp = zeropoints[0][0]; + + Mat weightsQuantized(weightsMat.rows, weightsMat.cols, CV_8S); + Mat biasQuantized(1, numOutput, CV_32S); + Mat outputMultiplier(1, numOutput, CV_32F); + + double realMin, realMax, weightsScale; + for( int i = 0; i < numOutput; i++ ) + { + // Quantize weights + cv::minMaxIdx(weightsMat.row(i), &realMin, &realMax); + realMin = std::min(realMin, 0.0); + realMax = std::max(realMax, 0.0); + weightsScale = (realMax == realMin) ? 1.0 : std::max(-realMin, realMax)/127; + weightsMat.row(i).convertTo(weightsQuantized.row(i), CV_8S, 1.f/weightsScale); + + // Quantize biases + float biasScale = inputScale * weightsScale; + biasQuantized.at(i) = (int)std::round(biasMat.at(i)/biasScale) - inputZp*(cv::sum(weightsQuantized.row(i))[0]); + + // Store multiplier + outputMultiplier.at(i) = biasScale / outputScale; + } + + params.blobs.clear(); + params.blobs.push_back(weightsQuantized.reshape(1, shape(blobs[0]))); + params.blobs.push_back(biasQuantized); + params.blobs.push_back(outputMultiplier); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/layers_common.simd.hpp b/modules/dnn/src/layers/layers_common.simd.hpp index 706695a7b2..0a077e4631 100644 --- a/modules/dnn/src/layers/layers_common.simd.hpp +++ b/modules/dnn/src/layers/layers_common.simd.hpp @@ -737,5 +737,620 @@ void fastGEMM( const float* aptr, size_t astep, const float* bptr, #endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY +#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_RVV + +void fastGEMM( const float* aptr, size_t astep, const float* bptr, + size_t bstep, float* cptr, size_t cstep, + int ma, int na, int nb ) +{ + int n = 0; + int vl = vsetvlmax_e32m4(); + int mvl = vl; + for( ; n < nb; n += vl ) + { + if ( n + vl > nb) { + mvl = nb - n; + } + + for( int m = 0; m < ma; m += 7 ) + { + const float* aptr0 = aptr + astep*m; + const float* aptr1 = aptr + astep*std::min(m+1, ma-1); + const float* aptr2 = aptr + astep*std::min(m+2, ma-1); + const float* aptr3 = aptr + astep*std::min(m+3, ma-1); + const float* aptr4 = aptr + astep*std::min(m+4, ma-1); + const float* aptr5 = aptr + astep*std::min(m+5, ma-1); + const float* aptr6 = aptr + astep*std::min(m+6, ma-1); + + float* cptr0 = cptr + cstep*m; + float* cptr1 = cptr + cstep*std::min(m+1, ma-1); + float* cptr2 = cptr + cstep*std::min(m+2, ma-1); + float* cptr3 = cptr + cstep*std::min(m+3, ma-1); + float* cptr4 = cptr + cstep*std::min(m+4, ma-1); + float* cptr5 = cptr + cstep*std::min(m+5, ma-1); + float* cptr6 = cptr + cstep*std::min(m+6, ma-1); + + vfloat32m4_t d0 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d1 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d2 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d3 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d4 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d5 = vfmv_v_f_f32m4(0, vl); + vfloat32m4_t d6 = vfmv_v_f_f32m4(0, vl); + + for( int k = 0; k < na; k++ ) + { + float32_t a0 = aptr0[k]; + float32_t a1 = aptr1[k]; + float32_t a2 = aptr2[k]; + float32_t a3 = aptr3[k]; + float32_t a4 = aptr4[k]; + float32_t a5 = aptr5[k]; + float32_t a6 = aptr6[k]; + + vfloat32m4_t b = vle32_v_f32m4(bptr + k*bstep + n, mvl); + d0 = vfmacc_vf_f32m4(d0, a0, b, mvl); + d1 = vfmacc_vf_f32m4(d1, a1, b, mvl); + d2 = vfmacc_vf_f32m4(d2, a2, b, mvl); + d3 = vfmacc_vf_f32m4(d3, a3, b, mvl); + d4 = vfmacc_vf_f32m4(d4, a4, b, mvl); + d5 = vfmacc_vf_f32m4(d5, a5, b, mvl); + d6 = vfmacc_vf_f32m4(d6, a6, b, mvl); + } + vse32_v_f32m4(cptr0 + n, d0, mvl); + vse32_v_f32m4(cptr1 + n, d1, mvl); + vse32_v_f32m4(cptr2 + n, d2, mvl); + vse32_v_f32m4(cptr3 + n, d3, mvl); + vse32_v_f32m4(cptr4 + n, d4, mvl); + vse32_v_f32m4(cptr5 + n, d5, mvl); + vse32_v_f32m4(cptr6 + n, d6, mvl); + } + } +} + +void fastGEMM1T( const float* vec, const float* weights, + size_t wstep, const float* bias, + float* dst, int nvecs, int vecsize ) +{ + int vlm2 = vsetvlmax_e32m2(); + int i = 0; + for( ; i <= nvecs - 15; i += 15 ) + { + const float* wptr = weights + i*wstep; + vfloat32m2_t + vs0 = vfmv_v_f_f32m2(0, vlm2), vs1 = vfmv_v_f_f32m2(0, vlm2), vs2 = vfmv_v_f_f32m2(0, vlm2), + vs3 = vfmv_v_f_f32m2(0, vlm2), vs4 = vfmv_v_f_f32m2(0, vlm2), vs5 = vfmv_v_f_f32m2(0, vlm2), + vs6 = vfmv_v_f_f32m2(0, vlm2), vs7 = vfmv_v_f_f32m2(0, vlm2), vs8 = vfmv_v_f_f32m2(0, vlm2), + vs9 = vfmv_v_f_f32m2(0, vlm2), vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), + vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2), vs14 = vfmv_v_f_f32m2(0, vlm2); + int k = 0; + for( ; k < vecsize - vlm2; k += vlm2, wptr += vlm2 ) + { + vfloat32m2_t v = vle32_v_f32m2(vec + k, vlm2); + + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vlm2), v, vlm2); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep, vlm2), v, vlm2); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, vlm2), v, vlm2); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, vlm2), v, vlm2); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, vlm2), v, vlm2); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, vlm2), v, vlm2); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, vlm2), v, vlm2); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, vlm2), v, vlm2); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, vlm2), v, vlm2); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, vlm2), v, vlm2); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, vlm2), v, vlm2); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, vlm2), v, vlm2); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, vlm2), v, vlm2); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, vlm2), v, vlm2); + vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, vlm2), v, vlm2); + } + int kvl = vecsize - k; + if (kvl > 0) { + vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*1, kvl), v, kvl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*2, kvl), v, kvl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*3, kvl), v, kvl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*4, kvl), v, kvl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*5, kvl), v, kvl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*6, kvl), v, kvl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*7, kvl), v, kvl); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*8, kvl), v, kvl); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*9, kvl), v, kvl); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*10, kvl), v, kvl); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*11, kvl), v, kvl); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*12, kvl), v, kvl); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*13, kvl), v, kvl); + vs14 = vfmacc_vv_f32m2(vs14, vle32_v_f32m2(wptr + wstep*14, kvl), v, kvl); + } + + // Calculate the sum of each vector + float32_t sum[15]; + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); + sum[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); + sum[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); + sum[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); + sum[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); + sum[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); + sum[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); + sum[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); + sum[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); + sum[8] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); + sum[9] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); + sum[10] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); + sum[11] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); + sum[12] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); + sum[13] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); + sum[14] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs14, zero, vlm2)); + + vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, 15), vle32_v_f32m4(bias + i, 15), 15); + vse32_v_f32m4(dst + i, s0, 15); + } + int mvl = nvecs - i; + if (mvl > 0) + { + const float* wptr = weights + i*wstep; + vfloat32m2_t + vs0 = vfmv_v_f_f32m2(0, vlm2), vs1 = vfmv_v_f_f32m2(0, vlm2), vs2 = vfmv_v_f_f32m2(0, vlm2), + vs3 = vfmv_v_f_f32m2(0, vlm2), vs4 = vfmv_v_f_f32m2(0, vlm2), vs5 = vfmv_v_f_f32m2(0, vlm2), + vs6 = vfmv_v_f_f32m2(0, vlm2), vs7 = vfmv_v_f_f32m2(0, vlm2), vs8 = vfmv_v_f_f32m2(0, vlm2), + vs9 = vfmv_v_f_f32m2(0, vlm2), vs10 = vfmv_v_f_f32m2(0, vlm2), vs11 = vfmv_v_f_f32m2(0, vlm2), + vs12 = vfmv_v_f_f32m2(0, vlm2), vs13 = vfmv_v_f_f32m2(0, vlm2); + int k = 0; + for( ; k <= vecsize - vlm2; k += vlm2, wptr += vlm2 ) + { + vfloat32m2_t v = vle32_v_f32m2(vec + k, vlm2); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, vlm2), v, vlm2); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), vlm2), v, vlm2); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), vlm2), v, vlm2); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), vlm2), v, vlm2); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), vlm2), v, vlm2); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), vlm2), v, vlm2); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), vlm2), v, vlm2); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, mvl-1), vlm2), v, vlm2); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, mvl-1), vlm2), v, vlm2); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, mvl-1), vlm2), v, vlm2); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, mvl-1), vlm2), v, vlm2); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, mvl-1), vlm2), v, vlm2); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, mvl-1), vlm2), v, vlm2); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), vlm2), v, vlm2); + } + int kvl = vecsize - k; + if (kvl > 0) { + vfloat32m2_t v = vle32_v_f32m2(vec + k, kvl); + vs0 = vfmacc_vv_f32m2(vs0, vle32_v_f32m2(wptr, kvl), v, kvl); + vs1 = vfmacc_vv_f32m2(vs1, vle32_v_f32m2(wptr + wstep*std::min(1, mvl-1), kvl), v, kvl); + vs2 = vfmacc_vv_f32m2(vs2, vle32_v_f32m2(wptr + wstep*std::min(2, mvl-1), kvl), v, kvl); + vs3 = vfmacc_vv_f32m2(vs3, vle32_v_f32m2(wptr + wstep*std::min(3, mvl-1), kvl), v, kvl); + vs4 = vfmacc_vv_f32m2(vs4, vle32_v_f32m2(wptr + wstep*std::min(4, mvl-1), kvl), v, kvl); + vs5 = vfmacc_vv_f32m2(vs5, vle32_v_f32m2(wptr + wstep*std::min(5, mvl-1), kvl), v, kvl); + vs6 = vfmacc_vv_f32m2(vs6, vle32_v_f32m2(wptr + wstep*std::min(6, mvl-1), kvl), v, kvl); + vs7 = vfmacc_vv_f32m2(vs7, vle32_v_f32m2(wptr + wstep*std::min(7, mvl-1), kvl), v, kvl); + vs8 = vfmacc_vv_f32m2(vs8, vle32_v_f32m2(wptr + wstep*std::min(8, mvl-1), kvl), v, kvl); + vs9 = vfmacc_vv_f32m2(vs9, vle32_v_f32m2(wptr + wstep*std::min(9, mvl-1), kvl), v, kvl); + vs10 = vfmacc_vv_f32m2(vs10, vle32_v_f32m2(wptr + wstep*std::min(10, mvl-1), kvl), v, kvl); + vs11 = vfmacc_vv_f32m2(vs11, vle32_v_f32m2(wptr + wstep*std::min(11, mvl-1), kvl), v, kvl); + vs12 = vfmacc_vv_f32m2(vs12, vle32_v_f32m2(wptr + wstep*std::min(12, mvl-1), kvl), v, kvl); + vs13 = vfmacc_vv_f32m2(vs13, vle32_v_f32m2(wptr + wstep*std::min(13, mvl-1), kvl), v, kvl); + } + // Calculate the sum of each vector + float32_t sum[14]; + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm2); + sum[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs0, zero, vlm2)); + sum[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs1, zero, vlm2)); + sum[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs2, zero, vlm2)); + sum[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs3, zero, vlm2)); + sum[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs4, zero, vlm2)); + sum[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs5, zero, vlm2)); + sum[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs6, zero, vlm2)); + sum[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs7, zero, vlm2)); + sum[8] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs8, zero, vlm2)); + sum[9] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs9, zero, vlm2)); + sum[10] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs10, zero, vlm2)); + sum[11] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs11, zero, vlm2)); + sum[12] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs12, zero, vlm2)); + sum[13] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m2_f32m1(zero, vs13, zero, vlm2)); + + vfloat32m4_t s0 = vfadd_vv_f32m4(vle32_v_f32m4(sum, mvl), vle32_v_f32m4(bias + i, mvl), mvl); + vse32_v_f32m4(dst + i, s0, mvl); + } +} + +enum { FASCONV_BASE_VECSZ = 8 }; +void fastConv( const float* weights, size_t wstep, const float* bias, + const float* rowbuf, float* output, const int* outShape, + int blockSize, int vecsize, int vecsize_aligned, + const float* relu, bool initOutput ) +{ + int vl = FASCONV_BASE_VECSZ; + int vlm1Max = vsetvlmax_e32m1(); + int outCn = outShape[1]; + size_t outPlaneSize = outShape[2]*outShape[3]; + // now compute dot product of the weights + // and im2row-transformed part of the tensor + for( int i = 0; i < outCn; i += 3 ) + { + const float* wptr0 = weights + i*wstep; + const float* wptr1 = wptr0 + wstep; + const float* wptr2 = wptr1 + wstep; + float* outptr0 = output + i*outPlaneSize; + float* outptr1 = outptr0 + outPlaneSize; + float* outptr2 = outptr1 + outPlaneSize; + float bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2]; + + if( i+2 >= outCn ) + { + wptr2 = wptr1; + outptr2 = outptr1; + bias2 = bias1; + if( i+1 >= outCn ) + { + wptr2 = wptr1 = wptr0; + outptr2 = outptr1 = outptr0; + bias2 = bias1 = bias0; + } + } + + int j = 0; + for( ; j < blockSize; j += FASCONV_BASE_VECSZ ) + { + bool tail = false; + if (j + FASCONV_BASE_VECSZ > blockSize) + { + if (j == 0) { + vl = blockSize; + } + else { + j = blockSize - FASCONV_BASE_VECSZ; + tail = true; + } + } + int k = 0; + const float* rptr = rowbuf + j*vecsize_aligned; + int vlm1 = vsetvlmax_e32m1(); + vfloat32m1_t + vs00 = vfmv_v_f_f32m1(0, vlm1), vs10 = vfmv_v_f_f32m1(0, vlm1), vs20 = vfmv_v_f_f32m1(0, vlm1), + vs01 = vfmv_v_f_f32m1(0, vlm1), vs11 = vfmv_v_f_f32m1(0, vlm1), vs21 = vfmv_v_f_f32m1(0, vlm1), + vs02 = vfmv_v_f_f32m1(0, vlm1), vs12 = vfmv_v_f_f32m1(0, vlm1), vs22 = vfmv_v_f_f32m1(0, vlm1), + vs03 = vfmv_v_f_f32m1(0, vlm1), vs13 = vfmv_v_f_f32m1(0, vlm1), vs23 = vfmv_v_f_f32m1(0, vlm1), + vs04 = vfmv_v_f_f32m1(0, vlm1), vs14 = vfmv_v_f_f32m1(0, vlm1), vs24 = vfmv_v_f_f32m1(0, vlm1), + vs05 = vfmv_v_f_f32m1(0, vlm1), vs15 = vfmv_v_f_f32m1(0, vlm1), vs25 = vfmv_v_f_f32m1(0, vlm1), + vs06 = vfmv_v_f_f32m1(0, vlm1), vs16 = vfmv_v_f_f32m1(0, vlm1), vs26 = vfmv_v_f_f32m1(0, vlm1), + vs07 = vfmv_v_f_f32m1(0, vlm1), vs17 = vfmv_v_f_f32m1(0, vlm1), vs27 = vfmv_v_f_f32m1(0, vlm1); + + for (; k < vecsize; k += vlm1, rptr += vlm1 ) + { + if (k + vlm1 >= vecsize) { + vlm1 = vecsize - k; + } + vfloat32m1_t w0 = vle32_v_f32m1(wptr0 + k, vlm1); + vfloat32m1_t w1 = vle32_v_f32m1(wptr1 + k, vlm1); + vfloat32m1_t w2 = vle32_v_f32m1(wptr2 + k, vlm1); + vfloat32m1_t r0 = vle32_v_f32m1(rptr, vlm1); + + vs00 = vfmacc_vv_f32m1(vs00, w0, r0, vlm1); + vs10 = vfmacc_vv_f32m1(vs10, w1, r0, vlm1); + vs20 = vfmacc_vv_f32m1(vs20, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned, vlm1); + vs01 = vfmacc_vv_f32m1(vs01, w0, r0, vlm1); + vs11 = vfmacc_vv_f32m1(vs11, w1, r0, vlm1); + vs21 = vfmacc_vv_f32m1(vs21, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*2, vlm1); + vs02 = vfmacc_vv_f32m1(vs02, w0, r0, vlm1); + vs12 = vfmacc_vv_f32m1(vs12, w1, r0, vlm1); + vs22 = vfmacc_vv_f32m1(vs22, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*3, vlm1); + vs03 = vfmacc_vv_f32m1(vs03, w0, r0, vlm1); + vs13 = vfmacc_vv_f32m1(vs13, w1, r0, vlm1); + vs23 = vfmacc_vv_f32m1(vs23, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*4, vlm1); + vs04 = vfmacc_vv_f32m1(vs04, w0, r0, vlm1); + vs14 = vfmacc_vv_f32m1(vs14, w1, r0, vlm1); + vs24 = vfmacc_vv_f32m1(vs24, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*5, vlm1); + vs05 = vfmacc_vv_f32m1(vs05, w0, r0, vlm1); + vs15 = vfmacc_vv_f32m1(vs15, w1, r0, vlm1); + vs25 = vfmacc_vv_f32m1(vs25, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*6, vlm1); + vs06 = vfmacc_vv_f32m1(vs06, w0, r0, vlm1); + vs16 = vfmacc_vv_f32m1(vs16, w1, r0, vlm1); + vs26 = vfmacc_vv_f32m1(vs26, w2, r0, vlm1); + + r0 = vle32_v_f32m1(rptr + vecsize_aligned*7, vlm1); + vs07 = vfmacc_vv_f32m1(vs07, w0, r0, vlm1); + vs17 = vfmacc_vv_f32m1(vs17, w1, r0, vlm1); + vs27 = vfmacc_vv_f32m1(vs27, w2, r0, vlm1); + } + + // compute sum of each vs + vfloat32m1_t zero = vfmv_v_f_f32m1(0, vlm1Max); + // vl is required here to be at least FASCONV_BASE_VECSZ, aka 8. + float32_t sum0[FASCONV_BASE_VECSZ], sum1[FASCONV_BASE_VECSZ], sum2[FASCONV_BASE_VECSZ]; + sum0[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs00, zero, vlm1Max)); + sum0[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs01, zero, vlm1Max)); + sum0[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs02, zero, vlm1Max)); + sum0[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs03, zero, vlm1Max)); + sum0[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs04, zero, vlm1Max)); + sum0[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs05, zero, vlm1Max)); + sum0[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs06, zero, vlm1Max)); + sum0[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs07, zero, vlm1Max)); + sum1[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs10, zero, vlm1Max)); + sum1[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs11, zero, vlm1Max)); + sum1[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs12, zero, vlm1Max)); + sum1[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs13, zero, vlm1Max)); + sum1[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs14, zero, vlm1Max)); + sum1[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs15, zero, vlm1Max)); + sum1[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs16, zero, vlm1Max)); + sum1[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs17, zero, vlm1Max)); + sum2[0] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs20, zero, vlm1Max)); + sum2[1] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs21, zero, vlm1Max)); + sum2[2] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs22, zero, vlm1Max)); + sum2[3] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs23, zero, vlm1Max)); + sum2[4] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs24, zero, vlm1Max)); + sum2[5] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs25, zero, vlm1Max)); + sum2[6] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs26, zero, vlm1Max)); + sum2[7] = vfmv_f_s_f32m1_f32(vfredsum_vs_f32m1_f32m1(zero, vs27, zero, vlm1Max)); + + // if VLEN = 128, so LMUL = 2 for vl = 8. + // otherwise, VLEN >=256, we only use fist 8 element of the vReg. + vfloat32m2_t s0, s1, s2; + if( initOutput ) + { + s0 = vfmv_v_f_f32m2(bias0, vl); + s1 = vfmv_v_f_f32m2(bias1, vl); + s2 = vfmv_v_f_f32m2(bias2, vl); + } + else + { + s0 = vle32_v_f32m2(outptr0 + j, vl); + s1 = vle32_v_f32m2(outptr1 + j, vl); + s2 = vle32_v_f32m2(outptr2 + j, vl); + } + s0 = vfadd_vv_f32m2(vle32_v_f32m2(sum0, vl), s0, vl); + s1 = vfadd_vv_f32m2(vle32_v_f32m2(sum1, vl), s1, vl); + s2 = vfadd_vv_f32m2(vle32_v_f32m2(sum2, vl), s2, vl); + + if( relu ) + { + vfloat32m2_t vr0 = vfmv_v_f_f32m2(1, vl), vr1 = vfmv_v_f_f32m2(1, vl), vr2 = vfmv_v_f_f32m2(1, vl); + float r0 = relu[i], r1 = relu[i+1], r2 = relu[i+2]; + if( i+2 >= outCn ) + { + r2 = r1; + if( i+1 >= outCn ) + r2 = r1 = r0; + } + vr0 = vfmv_v_f_f32m2(r0, vl); + vr1 = vfmv_v_f_f32m2(r1, vl); + vr2 = vfmv_v_f_f32m2(r2, vl); + vbool16_t m0 = vmfgt_vf_f32m2_b16(s0, 0, vl); + vbool16_t m1 = vmfgt_vf_f32m2_b16(s1, 0, vl); + vbool16_t m2 = vmfgt_vf_f32m2_b16(s2, 0, vl); + s0 = vmerge_vvm_f32m2(m0, vfmul_vv_f32m2(s0, vr0, vl), s0, vl); + s1 = vmerge_vvm_f32m2(m1, vfmul_vv_f32m2(s1, vr1, vl), s1, vl); + s2 = vmerge_vvm_f32m2(m2, vfmul_vv_f32m2(s2, vr2, vl), s2, vl); + } + + if( tail ) + { + int maskbuf[FASCONV_BASE_VECSZ] = {0}; + int rsz = blockSize % FASCONV_BASE_VECSZ; + for( int i = 0; i < rsz; i++ ) + maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1; + vint32m2_t vmaskbuf = vle32_v_i32m2(maskbuf ,vl); + vbool16_t mask = vmslt_vx_i32m2_b16(vmaskbuf, 0, vl); // mask for tail + s0 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr0 + j, vl), s0, vl); + s1 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr1 + j, vl), s1, vl); + s2 = vmerge_vvm_f32m2(mask, vle32_v_f32m2(outptr2 + j, vl), s2, vl); + } + + vse32_v_f32m2(outptr0 + j, s0, vl); + vse32_v_f32m2(outptr1 + j, s1, vl); + vse32_v_f32m2(outptr2 + j, s2, vl); + } + } +} + +/* +Example for load_deinterleave: + input: ptr[16] = {1,2,3, ... ,14,15,16} + output: a = {1, 3, 5, 7, 9, 11, 13, 15} + output: b = {2, 4, 6, 8,10, 12, 14, 16} +*/ +static inline void vfloat32m2_load_deinterleave(const float* ptr, vfloat32m2_t& a, vfloat32m2_t& b, int vl) +{ + vuint64m4_t mask = vmv_v_x_u64m4(1,vl*2); + vuint32m4_t mask_re = vreinterpret_v_u64m4_u32m4(mask); + vbool8_t mask0 = vmseq_vx_u32m4_b8 (mask_re, 1, vl*2); + vbool8_t mask1 = vmseq_vx_u32m4_b8 (mask_re, 0, vl*2); + vfloat32m4_t tempa = vundefined_f32m4(), tempb = vundefined_f32m4(); + vfloat32m4_t vw = vle32_v_f32m4(ptr, vl*2); + tempa = vcompress_vm_f32m4(mask0, tempa, vw, vl*2); + tempb = vcompress_vm_f32m4(mask1, tempb, vw, vl*2); + /* The following instructions have not to be supported by the GNU toolchain. + So we temporarily use store and load instead. + // a = vlmul_trunc_v_f32m4_f32m2(tempa); + // b = vlmul_trunc_v_f32m4_f32m2(tempb); + */ + cv::AutoBuffer cvBuffer(sizeof(float32_t)*vl*2); + float* buffer = (float*)cvBuffer.data(); + vse32_v_f32m4(buffer, tempa, vl); + a = vle32_v_f32m2(buffer, vl); + vse32_v_f32m4(buffer, tempb, vl); + b = vle32_v_f32m2(buffer, vl); +} + +void fastDepthwiseConv( const float* wptr, + int kernel_h, int kernel_w, + int stride_h, int stride_w, + int dilation_h, int dilation_w, + int pad_t, int pad_l, + const float* biasptr, const float* relu, + const float* inptr_, + int height, int width, + float* outptr_, + int out_d, int outH, int outW ) +{ + int vl = vsetvlmax_e32m2(); + const float w00_ = wptr[0], w01_ = wptr[1], w02_ = wptr[2], + w10 = wptr[3], w11 = wptr[4], w12 = wptr[5], + w20_ = wptr[6], w21_ = wptr[7], w22_ = wptr[8]; + int outW1 = std::min(outW, (width - dilation_w*(kernel_w - 1) + pad_l)/stride_w); + float relu_coeff = relu ? relu[out_d] : 1.f, bias = biasptr[out_d]; + + for (int out_i = 0; out_i < outH; out_i++) + { + int in_i = out_i * stride_h - pad_t, out_j = 0; + const float* imgptr0 = inptr_ + in_i*width; + const float* imgptr1 = imgptr0 + dilation_h*width; + const float* imgptr2 = imgptr0 + (dilation_h*2)*width; + float out, w00 = w00_, w01 = w01_, w02 = w02_; + float w20 = w20_, w21 = w21_, w22 = w22_; + if (in_i < 0) + { + w00 = w01 = w02 = 0.f; + imgptr0 = imgptr1; + } + else if (in_i + dilation_h*(kernel_h-1) >= height) + { + w20 = w21 = w22 = 0.f; + imgptr2 = imgptr1; + } + float* outptr = outptr_ + out_i*outW; + if (pad_l > 0) + { + out = imgptr0[0]*w01 + imgptr0[dilation_w]*w02 + + imgptr1[0]*w11 + imgptr1[dilation_w]*w12 + + imgptr2[0]*w21 + imgptr2[dilation_w]*w22 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[0] = out; + out_j = 1; + } + + if (stride_w == 1 || (stride_w == 2 && dilation_w == 1)) + { + if( stride_w == 1 ) + for( ; out_j < outW1; out_j += vl ) + { + if (out_j + vl > outW1) + vl = outW1 - out_j; + int in_j = out_j * stride_w - pad_l; + vfloat32m2_t v00 = vle32_v_f32m2(imgptr0 + in_j, vl), + v01 = vle32_v_f32m2(imgptr0 + in_j + dilation_w, vl), + v02 = vle32_v_f32m2(imgptr0 + in_j + dilation_w*2, vl), + v10 = vle32_v_f32m2(imgptr1 + in_j, vl), + v11 = vle32_v_f32m2(imgptr1 + in_j + dilation_w, vl), + v12 = vle32_v_f32m2(imgptr1 + in_j + dilation_w*2, vl), + v20 = vle32_v_f32m2(imgptr2 + in_j, vl), + v21 = vle32_v_f32m2(imgptr2 + in_j + dilation_w, vl), + v22 = vle32_v_f32m2(imgptr2 + in_j + dilation_w*2, vl); + + vfloat32m2_t vout0 = vfmul_vf_f32m2(v00, w00, vl); + vfloat32m2_t vout1 = vfmul_vf_f32m2(v01, w01, vl); + vfloat32m2_t vout2 = vfmul_vf_f32m2(v02, w02, vl); + vout0 = vfadd_vf_f32m2(vout0, bias, vl); + + vout0 = vfmacc_vf_f32m2(vout0, w10, v10, vl); + vout1 = vfmacc_vf_f32m2(vout1, w11, v11, vl); + vout2 = vfmacc_vf_f32m2(vout2, w12, v12, vl); + + vout0 = vfmacc_vf_f32m2(vout0, w20, v20, vl); + vout1 = vfmacc_vf_f32m2(vout1, w21, v21, vl); + vout2 = vfmacc_vf_f32m2(vout2, w22, v22, vl); + + vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); + if (relu) + { + vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vf_f32m2(vout0, relu_coeff, vl), vout0, vl); + } + vse32_v_f32m2(outptr + out_j, vout0, vl); + } + else //stride_w == 2 && dilation_w == 1 + for( ; out_j < outW1; out_j += vl ) + { + if (out_j + vl > outW1) + vl = outW1 - out_j; + int in_j = out_j * stride_w - pad_l; + vfloat32m2_t v00, v01, v02, v10, v11, v12, v20, v21, v22, unused; + vfloat32m2_load_deinterleave(imgptr0 + in_j, v00, v01, vl); + vfloat32m2_load_deinterleave(imgptr0 + in_j + 2, v02, unused, vl); + vfloat32m2_load_deinterleave(imgptr1 + in_j, v10, v11, vl); + vfloat32m2_load_deinterleave(imgptr1 + in_j + 2, v12, unused, vl); + vfloat32m2_load_deinterleave(imgptr2 + in_j, v20, v21, vl); + vfloat32m2_load_deinterleave(imgptr2 + in_j + 2, v22, unused, vl); + + vfloat32m2_t vout0 = vfmul_vf_f32m2(v00, w00, vl); + vfloat32m2_t vout1 = vfmul_vf_f32m2(v01, w01, vl); + vfloat32m2_t vout2 = vfmul_vf_f32m2(v02, w02, vl); + vout0 = vfadd_vf_f32m2(vout0, bias, vl); + + vout0 = vfmacc_vf_f32m2(vout0, w10, v10, vl); + vout1 = vfmacc_vf_f32m2(vout1, w11, v11, vl); + vout2 = vfmacc_vf_f32m2(vout2, w12, v12, vl); + + vout0 = vfmacc_vf_f32m2(vout0, w20, v20, vl); + vout1 = vfmacc_vf_f32m2(vout1, w21, v21, vl); + vout2 = vfmacc_vf_f32m2(vout2, w22, v22, vl); + + vout0 = vfadd_vv_f32m2(vfadd_vv_f32m2(vout0, vout1, vl), vout2, vl); + if (relu) + { + vbool16_t m = vmfgt_vf_f32m2_b16(vout0, 0, vl); + vout0 = vmerge_vvm_f32m2(m, vfmul_vf_f32m2(vout0, relu_coeff, vl), vout0, vl); + } + vse32_v_f32m2(outptr + out_j, vout0, vl); + } + } + + for (; out_j < outW1; out_j++) + { + int in_j = out_j * stride_w - pad_l; + out = imgptr0[in_j]*w00 + imgptr0[in_j + dilation_w]*w01 + imgptr0[in_j + dilation_w*2]*w02 + + imgptr1[in_j]*w10 + imgptr1[in_j + dilation_w]*w11 + imgptr1[in_j + dilation_w*2]*w12 + + imgptr2[in_j]*w20 + imgptr2[in_j + dilation_w]*w21 + imgptr2[in_j + dilation_w*2]*w22 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[out_j] = out; + } + + for (; out_j < outW; out_j++ ) + { + int in_j0 = out_j * stride_w - pad_l, in_j1 = in_j0 + dilation_w, in_j2 = in_j0 + dilation_w*2; + float s0 = 1.f, s1 = 1.f, s2 = 1.f; + if (in_j0 >= width) + { + in_j0 = 0; + s0 = 0.f; + } + if (in_j1 >= width) + { + in_j1 = 0; + s1 = 0.f; + } + if (in_j2 >= width) + { + in_j2 = 0; + s2 = 0.f; + } + out = imgptr0[in_j0]*w00*s0 + imgptr0[in_j1]*w01*s1 + imgptr0[in_j2]*w02*s2 + + imgptr1[in_j0]*w10*s0 + imgptr1[in_j1]*w11*s1 + imgptr1[in_j2]*w12*s2 + + imgptr2[in_j0]*w20*s0 + imgptr2[in_j1]*w21*s1 + imgptr2[in_j2]*w22*s2 + bias; + if (relu) + out = out > 0.f ? out : out*relu_coeff; + outptr[out_j] = out; + } + } +} + +#endif // CV_RVV + CV_CPU_OPTIMIZATION_NAMESPACE_END }} // namespace diff --git a/modules/dnn/src/layers/mvn_layer.cpp b/modules/dnn/src/layers/mvn_layer.cpp index 783949d4cd..455f4997df 100644 --- a/modules/dnn/src/layers/mvn_layer.cpp +++ b/modules/dnn/src/layers/mvn_layer.cpp @@ -200,7 +200,7 @@ public: k1.set(argId++, ocl::KernelArg::PtrReadOnly(bnorm_weight)); k1.set(argId++, ocl::KernelArg::PtrReadOnly(bnorm_bias)); k1.set(argId++, ocl::KernelArg::PtrWriteOnly(outMat)); - ret = k1.run(1, globalsize, localsize, false); + ret = k1.run_(1, globalsize, localsize, false); if (!ret) return false; } diff --git a/modules/dnn/src/layers/normalize_bbox_layer.cpp b/modules/dnn/src/layers/normalize_bbox_layer.cpp index 24559543e1..236f2e43f1 100644 --- a/modules/dnn/src/layers/normalize_bbox_layer.cpp +++ b/modules/dnn/src/layers/normalize_bbox_layer.cpp @@ -338,7 +338,7 @@ public: std::iota(axes_data.begin(), axes_data.end(), 1); } auto axes = std::make_shared(ngraph::element::i64, ngraph::Shape{axes_data.size()}, axes_data); - auto norm = std::make_shared(ieInpNode, axes, epsilon, ngraph::op::EpsMode::ADD); + auto norm = std::make_shared(ieInpNode, axes, epsilon, ngraph::op::EpsMode::ADD); CV_Assert(blobs.empty() || numChannels == blobs[0].total()); std::vector shape(ieInpNode->get_shape().size(), 1); diff --git a/modules/dnn/src/layers/not_implemented_layer.cpp b/modules/dnn/src/layers/not_implemented_layer.cpp new file mode 100644 index 0000000000..c4b1343902 --- /dev/null +++ b/modules/dnn/src/layers/not_implemented_layer.cpp @@ -0,0 +1,194 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include "../dnn_common.hpp" + +namespace cv { namespace dnn { +CV__DNN_INLINE_NS_BEGIN + +namespace detail { + +class NotImplementedImpl CV_FINAL : public NotImplemented +{ +public: + NotImplementedImpl(const LayerParams& params) + { + setParamsFrom(params); + CV_Assert(params.has("type")); + std::stringstream ss; + ss << "Node for layer '" << params.name << "' of type '" << params.get("type") << "' wasn't initialized."; + msg = ss.str(); + } + + CV_DEPRECATED_EXTERNAL + virtual void finalize(const std::vector &input, std::vector &output) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void finalize(InputArrayOfArrays inputs, OutputArrayOfArrays outputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED_EXTERNAL + virtual void forward(std::vector &input, std::vector &output, std::vector &internals) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + void forward_fallback(InputArrayOfArrays inputs, OutputArrayOfArrays outputs, OutputArrayOfArrays internals) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED_EXTERNAL + void finalize(const std::vector &inputs, CV_OUT std::vector &outputs) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED std::vector finalize(const std::vector &inputs) + { + CV_Error(Error::StsNotImplemented, msg); + } + + CV_DEPRECATED void run(const std::vector &inputs, + CV_OUT std::vector &outputs, + CV_IN_OUT std::vector &internals) + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int inputNameToIndex(String inputName) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int outputNameToIndex(const String& outputName) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initHalide(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initInfEngine(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initNgraph(const std::vector > &inputs, + const std::vector >& nodes) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initVkCom(const std::vector > &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr initCUDA( + void *context, + const std::vector>& inputs, + const std::vector>& outputs + ) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void applyHalideScheduler(Ptr& node, + const std::vector &inputs, + const std::vector &outputs, + int targetId) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual Ptr tryAttach(const Ptr& node) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool setActivation(const Ptr& layer) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool tryFuse(Ptr& top) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual void unsetAttached() CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual int64 getFLOPS(const std::vector &inputs, + const std::vector &outputs) const CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + + virtual bool updateMemoryShapes(const std::vector &inputs) CV_OVERRIDE + { + CV_Error(Error::StsNotImplemented, msg); + } + +private: + std::string msg; +}; + +Ptr NotImplemented::create(const LayerParams& params) +{ + return makePtr(params); +} + +Ptr notImplementedRegisterer(LayerParams ¶ms) +{ + return detail::NotImplemented::create(params); +} + +void NotImplemented::Register() +{ + LayerFactory::registerLayer("NotImplemented", detail::notImplementedRegisterer); +} + +void NotImplemented::unRegister() +{ + LayerFactory::unregisterLayer("NotImplemented"); +} + +} // namespace detail + +CV__DNN_INLINE_NS_END +}} // namespace cv::dnn diff --git a/modules/dnn/src/layers/padding_layer.cpp b/modules/dnn/src/layers/padding_layer.cpp index d182568795..c1979ce701 100644 --- a/modules/dnn/src/layers/padding_layer.cpp +++ b/modules/dnn/src/layers/padding_layer.cpp @@ -134,6 +134,8 @@ public: cv::convertFp16(paddingValue_fp32, paddingValue_fp16); outputs[0].setTo(paddingValue_fp16[0]); } + else if (inputs_arr.depth() == CV_8S) + outputs[0].setTo(saturate_cast(paddingValue)); else outputs[0].setTo(paddingValue); inputs[0].copyTo(outputs[0](dstRanges)); @@ -264,6 +266,16 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + float outputScale = scales[1][0]; + int outputZp = zeropoints[1][0]; + float padValue = outputZp + std::round(params.get("value", 0)/outputScale); + params.set("value", padValue); + return true; + } + private: std::vector > paddings; // Pairs pad before, pad after. std::vector dstRanges; diff --git a/modules/dnn/src/layers/permute_layer.cpp b/modules/dnn/src/layers/permute_layer.cpp index c525c3f82f..77c2469c05 100644 --- a/modules/dnn/src/layers/permute_layer.cpp +++ b/modules/dnn/src/layers/permute_layer.cpp @@ -194,6 +194,7 @@ public: #endif } + template class PermuteInvoker : public ParallelLoopBody { public: @@ -229,7 +230,7 @@ public: size_t stripeStart = r.start*stripeSize; size_t stripeEnd = std::min(r.end*stripeSize, orows); - const size_t esz = sizeof(float); + const size_t esz = sizeof(T); size_t ostep0 = out->step[0]/esz, ostep1 = out->step[1]/esz, ostep2 = out->step[2]/esz; const size_t* ord = &order->at(0); size_t istep0 = inp->step[ord[0]]/esz, istep1 = inp->step[ord[1]]/esz, @@ -241,13 +242,13 @@ public: int i1 = (int)(val % n1); int i0 = (int)(val / n1); - const float* inptr_orig = inp->ptr(); - float* outptr_orig = out->ptr(); + const T* inptr_orig = inp->ptr(); + T* outptr_orig = out->ptr(); for( size_t ofs = stripeStart; ofs < stripeEnd; ofs++ ) { - const float* inptr = inptr_orig + i0*istep0 + i1*istep1 + i2*istep2; - float* outptr = outptr_orig + i0*ostep0 + i1*ostep1 + i2*ostep2; + const T* inptr = inptr_orig + i0*istep0 + i1*istep1 + i2*istep2; + T* outptr = outptr_orig + i0*ostep0 + i1*ostep1 + i2*ostep2; for( int i3 = 0; i3 < n3; i3++ ) outptr[i3] = inptr[i3*istep3]; @@ -321,7 +322,8 @@ public: CV_TRACE_FUNCTION(); CV_TRACE_ARG_VALUE(name, "name", name.c_str()); - CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget), + CV_OCL_RUN(IS_DNN_OPENCL_TARGET(preferableTarget) && + inputs_arr.depth() != CV_8S, forward_ocl(inputs_arr, outputs_arr, internals_arr)) if (inputs_arr.depth() == CV_16S) @@ -365,24 +367,48 @@ public: if( numAxes == 4 ) { int nstripes = getNumThreads(); - PermuteInvoker::run(inp, out, _order, nstripes); + if (inp.type() == CV_8S) + PermuteInvoker::run(inp, out, _order, nstripes); + else + PermuteInvoker::run(inp, out, _order, nstripes); } else { - const float *srcData = inp.ptr(); - float *dstData = out.ptr(); - - for (i = 0; i < count; ++i) + if (inp.type() == CV_8S) { - size_t oldPosition = 0; - size_t newPosition = i; + const int8_t *srcData = inp.ptr(); + int8_t *dstData = out.ptr(); - for (j = 0; j < numAxes; ++j) + for (i = 0; i < count; ++i) { - oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; - newPosition %= newStride[j]; + size_t oldPosition = 0; + size_t newPosition = i; + + for (j = 0; j < numAxes; ++j) + { + oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; + newPosition %= newStride[j]; + } + dstData[i] = srcData[oldPosition]; + } + } + else + { + const float *srcData = inp.ptr(); + float *dstData = out.ptr(); + + for (i = 0; i < count; ++i) + { + size_t oldPosition = 0; + size_t newPosition = i; + + for (j = 0; j < numAxes; ++j) + { + oldPosition += (newPosition / newStride[j]) * oldStride[order[j]]; + newPosition %= newStride[j]; + } + dstData[i] = srcData[oldPosition]; } - dstData[i] = srcData[oldPosition]; } } } @@ -436,6 +462,11 @@ public: } #endif // HAVE_VULKAN + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } size_t _count; std::vector _order; diff --git a/modules/dnn/src/layers/pooling_layer.cpp b/modules/dnn/src/layers/pooling_layer.cpp index b8e2cfdf8f..7653e53668 100644 --- a/modules/dnn/src/layers/pooling_layer.cpp +++ b/modules/dnn/src/layers/pooling_layer.cpp @@ -1327,6 +1327,23 @@ public: return true; } + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + if (type == MAX && !computeMaxIdx) + { + return true; + } + else if (type == AVE || type == SUM) + { + float multiplier = scales[0][0] / scales[1][0]; + params.set("multiplier", multiplier); + params.set("input_zeropoint", zeropoints[0][0]); + return true; + } + return false; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/recurrent_layers.cpp b/modules/dnn/src/layers/recurrent_layers.cpp index bdc46643fc..6bfa48adbf 100644 --- a/modules/dnn/src/layers/recurrent_layers.cpp +++ b/modules/dnn/src/layers/recurrent_layers.cpp @@ -80,12 +80,31 @@ static void sigmoid(const Mat &src, Mat &dst) cv::pow(1 + dst, -1, dst); } +typedef void (*ActivationFunction)(const Mat &src, Mat &dst); +static ActivationFunction get_activation_function(const String& activation) { + // most used activations for PyTorch and TF : Tanh, Sigmoid + // if you need to support more optional activations use std::map instead + if (activation == "Tanh") + { + return tanh; + } + else if (activation == "Sigmoid") + { + return sigmoid; + } + else + { + CV_Error(Error::StsNotImplemented, + cv::format("Activation function [%s] for layer LSTM is not supported", activation.c_str())); + } +} + class LSTMLayerImpl CV_FINAL : public LSTMLayer { int numTimeStamps, numSamples; bool allocated; - MatShape outTailShape; //shape of single output sample + MatShape outTailShape; //shape of single output sample MatShape outTsShape; //shape of N output samples bool useTimestampDim; @@ -95,6 +114,10 @@ class LSTMLayerImpl CV_FINAL : public LSTMLayer bool reverse; // If true, go in negative direction along the time axis bool bidirectional; // If true, produces both forward and reversed directions along time axis + ActivationFunction f_activation; + ActivationFunction g_activation; + ActivationFunction h_activation; + public: LSTMLayerImpl(const LayerParams& params) @@ -112,19 +135,24 @@ public: const Mat& Wh = blobs[0]; const Mat& Wx = blobs[1]; const Mat& bias = blobs[2]; + const Mat& hInternal = blobs[3]; + const Mat& cInternal = blobs[4]; CV_CheckEQ(Wh.dims, 2, ""); CV_CheckEQ(Wx.dims, 2, ""); CV_CheckEQ(Wh.rows, Wx.rows, ""); CV_CheckEQ(Wh.rows, (1 + static_cast(bidirectional))*4*Wh.cols, ""); CV_CheckEQ(Wh.rows, (int)bias.total(), ""); + CV_CheckEQ(hInternal.cols, Wh.cols, ""); + CV_CheckEQ(hInternal.cols, cInternal.cols, ""); + CV_CheckEQ(hInternal.rows, cInternal.rows, ""); CV_Assert(Wh.type() == Wx.type() && Wx.type() == bias.type()); // Peephole weights. - if (blobs.size() > 3) + if (blobs.size() > 5) { - CV_Assert(blobs.size() == 6); + CV_Assert(blobs.size() == 8); const int N = Wh.cols; - for (int i = 3; i < 6; ++i) + for (int i = 5; i < 8; ++i) { CV_Assert(blobs[i].rows == N && blobs[i].cols == N); CV_Assert(blobs[i].type() == bias.type()); @@ -140,6 +168,20 @@ public: reverse = params.get("reverse", false); CV_Assert(!reverse || !bidirectional); + // read activations + DictValue activations = params.get("activations", ""); + if (activations.size() == 1) // if activations wasn't specified use default + { + f_activation = sigmoid; + g_activation = tanh; + h_activation = tanh; + } else { + CV_Assert(activations.size() == 3); + f_activation = get_activation_function(activations.getStringValue(0)); + g_activation = get_activation_function(activations.getStringValue(1)); + h_activation = get_activation_function(activations.getStringValue(2)); + } + allocated = false; outTailShape.clear(); } @@ -181,7 +223,7 @@ public: std::vector &outputs, std::vector &internals) const CV_OVERRIDE { - CV_Assert((!usePeephole && blobs.size() == 3) || (usePeephole && blobs.size() == 6)); + CV_Assert((!usePeephole && blobs.size() == 5) || (usePeephole && blobs.size() == 8)); CV_Assert(inputs.size() == 1); const MatShape& inp0 = inputs[0]; @@ -228,7 +270,7 @@ public: std::vector input; inputs_arr.getMatVector(input); - CV_Assert((!usePeephole && blobs.size() == 3) || (usePeephole && blobs.size() == 6)); + CV_Assert((!usePeephole && blobs.size() == 5) || (usePeephole && blobs.size() == 8)); CV_Assert(input.size() == 1); const Mat& inp0 = input[0]; @@ -284,13 +326,14 @@ public: const Mat &Wh = blobs[0].rowRange(i * blobs[0].rows / numDirs, (i + 1) * blobs[0].rows / numDirs); const Mat &Wx = blobs[1].rowRange(i * blobs[1].rows / numDirs, (i + 1) * blobs[1].rows / numDirs); const Mat &bias = blobs[2].colRange(i * blobs[2].cols / numDirs, (i + 1) * blobs[2].cols / numDirs); + const Mat &h_0 = blobs[3].rowRange(i * blobs[3].rows / numDirs, (i + 1) * blobs[3].rows / numDirs); + const Mat &c_0 = blobs[4].rowRange(i * blobs[4].rows / numDirs, (i + 1) * blobs[4].rows / numDirs); int numOut = Wh.size[1]; - Mat hInternal = internals[0], cInternal = internals[1], dummyOnes = internals[2], gates = internals[3]; - hInternal.setTo(0.); - cInternal.setTo(0.); + h_0.copyTo(hInternal); + c_0.copyTo(cInternal); dummyOnes.setTo(1.); int numSamplesTotal = numTimeStamps*numSamples; @@ -331,17 +374,17 @@ public: if (usePeephole) { Mat gatesIF = gates.colRange(0, 2*numOut); - gemm(cInternal, blobs[3], 1, gateI, 1, gateI); - gemm(cInternal, blobs[4], 1, gateF, 1, gateF); - sigmoid(gatesIF, gatesIF); + gemm(cInternal, blobs[5], 1, gateI, 1, gateI); + gemm(cInternal, blobs[6], 1, gateF, 1, gateF); + f_activation(gatesIF, gatesIF); } else { Mat gatesIFO = gates.colRange(0, 3*numOut); - sigmoid(gatesIFO, gatesIFO); + f_activation(gatesIFO, gatesIFO); } - tanh(gateG, gateG); + g_activation(gateG, gateG); //compute c_t multiply(gateF, cInternal, gateF); // f_t (*) c_{t-1} @@ -355,12 +398,12 @@ public: } if (usePeephole) { - gemm(cInternal, blobs[5], 1, gateO, 1, gateO); - sigmoid(gateO, gateO); + gemm(cInternal, blobs[7], 1, gateO, 1, gateO); + f_activation(gateO, gateO); } //compute h_t - tanh(cInternal, hInternal); + h_activation(cInternal, hInternal); multiply(gateO, hInternal, hInternal); //save results in output blobs @@ -557,5 +600,214 @@ CV_EXPORTS_W Ptr RNNLayer::create(const LayerParams& params) return Ptr(new RNNLayerImpl(params)); } +class GRULayerImpl CV_FINAL : public GRULayer +{ + int numTimeStamps, numSamples; + bool allocated; + + MatShape outTailShape; //shape of single output sample + MatShape outTsShape; //shape of N output samples + bool bidirectional; // If true, produces both forward and reversed directions along time axis + +public: + + GRULayerImpl(const LayerParams& params) : numTimeStamps(0), numSamples(0) + { + setParamsFrom(params); + + bidirectional = params.get("bidirectional", false); + if (!blobs.empty()) + { + CV_Assert(blobs.size() >= 3); + + blobs[2] = blobs[2].reshape(1, 1); + + const Mat& Wh = blobs[0]; + const Mat& Wx = blobs[1]; + const Mat& bias = blobs[2]; + const Mat& hInternal = blobs[3]; + CV_CheckEQ(Wh.dims, 2, ""); + CV_CheckEQ(Wx.dims, 2, ""); + CV_CheckEQ(Wh.rows, Wx.rows, ""); + CV_CheckEQ(Wh.rows, (1 + static_cast(bidirectional)) * 3 * Wh.cols, ""); + CV_CheckEQ(Wh.rows * 2, (int)bias.total(), ""); + CV_CheckEQ(hInternal.cols, Wh.cols, ""); + CV_CheckTypeEQ(Wh.type(), Wx.type(), ""); + CV_CheckTypeEQ(Wx.type(), bias.type(), ""); + } + + allocated = false; + outTailShape.clear(); + } + + bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + CV_Assert(inputs.size() == 1); + const MatShape& inp0 = inputs[0]; + + const Mat &Wh = blobs[0], &Wx = blobs[1]; + int _numOut = Wh.size[1]; + int _numInp = Wx.size[1]; + MatShape outTailShape_(outTailShape), outResShape; + + if (!outTailShape_.empty()) + CV_Assert(total(outTailShape_) == _numOut); + else + outTailShape_.assign(1, _numOut); + + int _numSamples; + CV_Assert(inp0.size() >= 2 && total(inp0, 2) == _numInp); + _numSamples = inp0[1]; + outResShape.push_back(inp0[0]); + + outResShape.push_back(_numSamples); + outResShape.insert(outResShape.end(), outTailShape_.begin(), outTailShape_.end()); + outResShape.back() *= (1 + static_cast(bidirectional)); + + outputs.assign(1, outResShape); + + internals.assign(1, shape(_numSamples, _numOut)); // hInternal + internals.push_back(shape(_numSamples, 1)); // dummyOnes + internals.push_back(shape(_numSamples, 2 * _numOut)); // gates + internals.push_back(shape(_numSamples, 2 * _numOut)); // gates_b + internals.push_back(shape(_numSamples, 1 * _numOut)); // h_linear + internals.push_back(shape(_numSamples, _numOut)); // ones + + return false; + } + + void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays) CV_OVERRIDE + { + std::vector input; + inputs_arr.getMatVector(input); + + CV_Assert(input.size() == 1); + const Mat& inp0 = input[0]; + + Mat &Wh = blobs[0], &Wx = blobs[1]; + int numOut = Wh.size[1]; + int numInp = Wx.size[1]; + + if (!outTailShape.empty()) + CV_Assert(total(outTailShape) == numOut); + else + outTailShape.assign(1, numOut); + + CV_Assert(inp0.dims >= 2 && (int)inp0.total(2) == numInp); + numTimeStamps = inp0.size[0]; + numSamples = inp0.size[1]; + + outTsShape.clear(); + outTsShape.push_back(numSamples); + outTsShape.insert(outTsShape.end(), outTailShape.begin(), outTailShape.end()); + outTsShape.back() *= (1 + static_cast(bidirectional)); + + allocated = true; + } + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + if (inputs_arr.depth() == CV_16S) + { + forward_fallback(inputs_arr, outputs_arr, internals_arr); + return; + } + + std::vector input, output, internals; + inputs_arr.getMatVector(input); + outputs_arr.getMatVector(output); + internals_arr.getMatVector(internals); + + const int numDirs = 1 + static_cast(bidirectional); + for (int i = 0; i < numDirs; ++i) + { + const Mat &Wh = blobs[0].rowRange(i * blobs[0].rows / numDirs, (i + 1) * blobs[0].rows / numDirs); + const Mat &Wx = blobs[1].rowRange(i * blobs[1].rows / numDirs, (i + 1) * blobs[1].rows / numDirs); + const Mat &bias = blobs[2].colRange(i * blobs[2].cols / numDirs, (i + 1) * blobs[2].cols / numDirs); + const Mat &h_0 = blobs[3].rowRange(i * blobs[3].rows / numDirs, (i + 1) * blobs[3].rows / numDirs); + + const Mat &bx = bias.colRange(0, bias.cols / 2); + const Mat &bh = bias.colRange(bias.cols / 2, bias.cols); + + Mat hInternal = internals[0], dummyOnes = internals[1], gates = internals[2], + b_rz = internals[3], n_t = internals[4], ones = internals[5]; + h_0.copyTo(hInternal); + dummyOnes.setTo(1.); + ones.setTo(1.); + + int numOut = Wh.size[1]; + const Mat& wx_rz = Wx.rowRange(0, 2 * numOut); + const Mat& wh_rz = Wh.rowRange(0, 2 * numOut); + b_rz = bx.colRange(0, 2 * numOut) + bh.colRange(0, 2 * numOut); + const Mat& wx_n = Wx.rowRange(2 * numOut, 3 * numOut); + const Mat& wh_n = Wh.rowRange(2 * numOut, 3 * numOut); + const Mat& b_in = bx.colRange(2 * numOut, 3 * numOut); + const Mat& b_hn = bh.colRange(2 * numOut, 3 * numOut); + + int numSamplesTotal = numTimeStamps * numSamples; + Mat xTs = input[0].reshape(1, numSamplesTotal); + + Mat hOutTs = output[0].reshape(1, numSamplesTotal); + hOutTs = hOutTs.colRange(i * hOutTs.cols / numDirs, (i + 1) * hOutTs.cols / numDirs); + Mat cOutTs = Mat(); + + int tsStart, tsEnd, tsInc; + if (i == 1) { + tsStart = numTimeStamps - 1; + tsEnd = -1; + tsInc = -1; + } + else { + tsStart = 0; + tsEnd = numTimeStamps; + tsInc = 1; + } + for (int ts = tsStart; ts != tsEnd; ts += tsInc) + { + Range curRowRange(ts * numSamples, (ts + 1) * numSamples); + Mat xCurr = xTs.rowRange(curRowRange); + + // calculate r_t = sigmoid(x * Wx_r + h_(t-1) * Wh_r + b_r) + // calculate z_t = sigmoid(x * Wx_z + h_(t-1) * Wh_z + b_z) + gemm(xCurr, wx_rz, 1, gates, 0, gates, GEMM_2_T); // x * Wx_rz + gemm(hInternal, wh_rz, 1, gates, 1, gates, GEMM_2_T); // + h_(t-1) * Wh_rz + gemm(dummyOnes, b_rz, 1, gates, 1, gates); // + b_rz + sigmoid(gates, gates); // sigmoid() + + Mat z = gates.colRange(0, gates.cols / 2); + Mat r = gates.colRange(gates.cols / 2, gates.cols); + + // calculate n_t = tanh(r (*) (h_(t-1) * Wh_n + b_hn) + x * Wx_n + b_in) + gemm(hInternal, wh_n, 1, n_t, 0, n_t, GEMM_2_T); // h_(t-1) * Wh_n + gemm(dummyOnes, b_hn, 1, n_t, 1, n_t); // + b_hn + multiply(r, n_t, n_t); // r (*) (h_(t-1) * Wh_n + b_hn) + + gemm(xCurr, wx_n, 1, n_t, 1, n_t, GEMM_2_T); // + x * Wx_n + gemm(dummyOnes, b_in, 1, n_t, 1, n_t); // + b_in + tanh(n_t, n_t); // tanh() + + //compute next h_t = z (*) h_(t-1) + (1 - z) (*) n_t + multiply(z, hInternal, hInternal); // z (*) h_{t-1} + subtract(ones, z, z); // 1 - z + multiply(z, n_t, z); // (1 - z) * n + add(z, hInternal, hInternal); // z (*) h_(t-1) + (1 - z) (*) n_t + + //save results in output blobs + hInternal.copyTo(hOutTs.rowRange(curRowRange)); + } + } + } +}; + +Ptr GRULayer::create(const LayerParams ¶ms) { + return Ptr(new GRULayerImpl(params)); +} + } } diff --git a/modules/dnn/src/layers/region_layer.cpp b/modules/dnn/src/layers/region_layer.cpp index 7da211afb0..73ed53974f 100644 --- a/modules/dnn/src/layers/region_layer.cpp +++ b/modules/dnn/src/layers/region_layer.cpp @@ -70,6 +70,7 @@ class RegionLayerImpl CV_FINAL : public RegionLayer public: int coords, classes, anchors, classfix; float thresh, scale_x_y; + int new_coords; bool useSoftmax, useLogistic; #ifdef HAVE_OPENCL UMat blob_umat; @@ -89,6 +90,7 @@ public: useLogistic = params.get("logistic", false); nmsThreshold = params.get("nms_threshold", 0.4); scale_x_y = params.get("scale_x_y", 1.0); // Yolov4 + new_coords = params.get("new_coords", 0); // Yolov4x-mish CV_Assert(nmsThreshold >= 0.); CV_Assert(coords == 4); @@ -119,7 +121,7 @@ public: { #ifdef HAVE_DNN_NGRAPH if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2) && preferableTarget != DNN_TARGET_MYRIAD; + return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2) && preferableTarget != DNN_TARGET_MYRIAD && new_coords == 0; #endif #ifdef HAVE_CUDA if (backendId == DNN_BACKEND_CUDA) @@ -269,26 +271,28 @@ public: const float *srcData = inpBlob.ptr(); float *dstData = outBlob.ptr(); - // logistic activation for t0, for each grid cell (X x Y x Anchor-index) - for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { - int index = cell_size*i; - float x = srcData[index + 4]; - dstData[index + 4] = logistic_activate(x); // logistic activation - } - - if (useSoftmax) { // Yolo v2 + if (new_coords == 0) { + // logistic activation for t0, for each grid cell (X x Y x Anchor-index) for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { int index = cell_size*i; - softmax_activate(srcData + index + 5, classes, 1, dstData + index + 5); + float x = srcData[index + 4]; + dstData[index + 4] = logistic_activate(x); // logistic activation } - } - else if (useLogistic) { // Yolo v3 - for (int i = 0; i < batch_size*rows*cols*anchors; ++i){ - int index = cell_size*i; - const float* input = srcData + index + 5; - float* output = dstData + index + 5; - for (int c = 0; c < classes; ++c) - output[c] = logistic_activate(input[c]); + + if (useSoftmax) { // Yolo v2 + for (int i = 0; i < batch_size*rows*cols*anchors; ++i) { + int index = cell_size*i; + softmax_activate(srcData + index + 5, classes, 1, dstData + index + 5); + } + } + else if (useLogistic) { // Yolo v3 + for (int i = 0; i < batch_size*rows*cols*anchors; ++i){ + int index = cell_size*i; + const float* input = srcData + index + 5; + float* output = dstData + index + 5; + for (int c = 0; c < classes; ++c) + output[c] = logistic_activate(input[c]); + } } } for (int b = 0; b < batch_size; ++b) @@ -300,20 +304,46 @@ public: int index = (y*cols + x)*anchors + a; // index for each grid-cell & anchor int p_index = index_sample_offset + index * cell_size + 4; float scale = dstData[p_index]; - if (classfix == -1 && scale < .5) scale = 0; // if(t0 < 0.5) t0 = 0; + if (classfix == -1 && scale < .5) + { + scale = 0; // if(t0 < 0.5) t0 = 0; + } int box_index = index_sample_offset + index * cell_size; - float x_tmp = (logistic_activate(srcData[box_index + 0]) - 0.5f) * scale_x_y + 0.5f; - float y_tmp = (logistic_activate(srcData[box_index + 1]) - 0.5f) * scale_x_y + 0.5f; - dstData[box_index + 0] = (x + x_tmp) / cols; - dstData[box_index + 1] = (y + y_tmp) / rows; - dstData[box_index + 2] = exp(srcData[box_index + 2]) * biasData[2 * a] / wNorm; - dstData[box_index + 3] = exp(srcData[box_index + 3]) * biasData[2 * a + 1] / hNorm; + if (new_coords == 1) { + float x_tmp = (srcData[box_index + 0] - 0.5f) * scale_x_y + 0.5f; + float y_tmp = (srcData[box_index + 1] - 0.5f) * scale_x_y + 0.5f; + dstData[box_index + 0] = (x + x_tmp) / cols; + dstData[box_index + 1] = (y + y_tmp) / rows; + dstData[box_index + 2] = (srcData[box_index + 2]) * (srcData[box_index + 2]) * 4 * biasData[2 * a] / wNorm; + dstData[box_index + 3] = (srcData[box_index + 3]) * (srcData[box_index + 3]) * 4 * biasData[2 * a + 1] / hNorm; - int class_index = index_sample_offset + index * cell_size + 5; - for (int j = 0; j < classes; ++j) { - float prob = scale*dstData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability - dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + scale = srcData[p_index]; + if (classfix == -1 && scale < thresh) + { + scale = 0; // if(t0 < 0.5) t0 = 0; + } + + int class_index = index_sample_offset + index * cell_size + 5; + for (int j = 0; j < classes; ++j) { + float prob = scale*srcData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability + dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + } + } + else + { + float x_tmp = (logistic_activate(srcData[box_index + 0]) - 0.5f) * scale_x_y + 0.5f; + float y_tmp = (logistic_activate(srcData[box_index + 1]) - 0.5f) * scale_x_y + 0.5f; + dstData[box_index + 0] = (x + x_tmp) / cols; + dstData[box_index + 1] = (y + y_tmp) / rows; + dstData[box_index + 2] = exp(srcData[box_index + 2]) * biasData[2 * a] / wNorm; + dstData[box_index + 3] = exp(srcData[box_index + 3]) * biasData[2 * a + 1] / hNorm; + + int class_index = index_sample_offset + index * cell_size + 5; + for (int j = 0; j < classes; ++j) { + float prob = scale*dstData[class_index + j]; // prob = IoU(box, object) = t0 * class-probability + dstData[class_index + j] = (prob > thresh) ? prob : 0; // if (IoU < threshold) IoU = 0; + } } } if (nmsThreshold > 0) { @@ -407,11 +437,12 @@ public: config.scale_x_y = scale_x_y; - config.object_prob_cutoff = (classfix == -1) ? 0.5 : 0.0; + config.object_prob_cutoff = (classfix == -1) ? thresh : 0.f; config.class_prob_cutoff = thresh; config.nms_iou_threshold = nmsThreshold; + config.new_coords = (new_coords == 1); return make_cuda_node(preferableTarget, std::move(context->stream), blobs[0], config); } #endif diff --git a/modules/dnn/src/layers/reorg_layer.cpp b/modules/dnn/src/layers/reorg_layer.cpp index da1c61adac..797df4819d 100644 --- a/modules/dnn/src/layers/reorg_layer.cpp +++ b/modules/dnn/src/layers/reorg_layer.cpp @@ -231,6 +231,11 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE diff --git a/modules/dnn/src/layers/reshape_layer.cpp b/modules/dnn/src/layers/reshape_layer.cpp index ab8f41c7b6..4c10d155c8 100644 --- a/modules/dnn/src/layers/reshape_layer.cpp +++ b/modules/dnn/src/layers/reshape_layer.cpp @@ -343,6 +343,11 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } private: std::vector outShapes; diff --git a/modules/dnn/src/layers/resize_layer.cpp b/modules/dnn/src/layers/resize_layer.cpp index e872c7f6b0..47ff0719fe 100644 --- a/modules/dnn/src/layers/resize_layer.cpp +++ b/modules/dnn/src/layers/resize_layer.cpp @@ -49,6 +49,8 @@ public: alignCorners = params.get("align_corners", false); halfPixelCenters = params.get("half_pixel_centers", false); + if (interpolation == "opencv_linear") + halfPixelCenters = true; } bool getMemoryShapes(const std::vector &inputs, @@ -120,12 +122,22 @@ public: internals_arr.getMatVector(internals); if (outHeight == inputs[0].size[2] && outWidth == inputs[0].size[3]) + { + // outputs[0] = inputs[0] doesn't work due to BlobManager optimizations + if (inputs[0].data != outputs[0].data) + { + inputs[0].copyTo(outputs[0]); + } return; + } Mat& inp = inputs[0]; Mat& out = outputs[0]; - if ((interpolation == "nearest" && !alignCorners && !halfPixelCenters) || interpolation == "opencv_linear" || (interpolation == "bilinear" && halfPixelCenters)) + int depth = inp.depth(); + if ((interpolation == "nearest" && !alignCorners && !halfPixelCenters) || (interpolation == "opencv_linear" && depth != CV_8S) || + (interpolation == "bilinear" && halfPixelCenters && depth != CV_8S)) { + // INTER_LINEAR Resize mode does not support INT8 inputs InterpolationFlags mode = interpolation == "nearest" ? INTER_NEAREST : INTER_LINEAR; for (size_t n = 0; n < inputs[0].size[0]; ++n) { @@ -157,34 +169,66 @@ public: widthOffset = 0.5f * scaleWidth; } - for (int y = 0; y < outHeight; ++y) + if (depth == CV_8S) { - float input_y = y * scaleHeight + heightOffset; - int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); - y0 = std::min(y0, inpHeight - 1); - - const float* inpData_row = inpPlanes.ptr(y0); - - for (int x = 0; x < outWidth; ++x) + for (int y = 0; y < outHeight; ++y) { - float input_x = x * scaleWidth + widthOffset; - int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); - x0 = std::min(x0, inpWidth - 1); + float input_y = y * scaleHeight + heightOffset; + int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); + y0 = std::min(y0, inpHeight - 1); - float* outData = outPlanes.ptr(y, x); - const float* inpData_row_c = inpData_row; + const int8_t* inpData_row = inpPlanes.ptr(y0); - for (int c = 0; c < numPlanes; ++c) + for (int x = 0; x < outWidth; ++x) { - *outData = inpData_row_c[x0]; + float input_x = x * scaleWidth + widthOffset; + int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); + x0 = std::min(x0, inpWidth - 1); - inpData_row_c += inpSpatialSize; - outData += outSpatialSize; + int8_t* outData = outPlanes.ptr(y, x); + const int8_t* inpData_row_c = inpData_row; + + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row_c[x0]; + + inpData_row_c += inpSpatialSize; + outData += outSpatialSize; + } + } + } + } + else + { + for (int y = 0; y < outHeight; ++y) + { + float input_y = y * scaleHeight + heightOffset; + int y0 = halfPixelCenters ? std::floor(input_y) : lroundf(input_y); + y0 = std::min(y0, inpHeight - 1); + + const float* inpData_row = inpPlanes.ptr(y0); + + for (int x = 0; x < outWidth; ++x) + { + float input_x = x * scaleWidth + widthOffset; + int x0 = halfPixelCenters ? std::floor(input_x) : lroundf(input_x); + x0 = std::min(x0, inpWidth - 1); + + float* outData = outPlanes.ptr(y, x); + const float* inpData_row_c = inpData_row; + + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row_c[x0]; + + inpData_row_c += inpSpatialSize; + outData += outSpatialSize; + } } } } } - else if (interpolation == "bilinear") + else if (interpolation == "bilinear" || interpolation == "opencv_linear") { const int inpHeight = inp.size[2]; const int inpWidth = inp.size[3]; @@ -195,31 +239,65 @@ public: Mat inpPlanes = inp.reshape(1, numPlanes * inpHeight); Mat outPlanes = out.reshape(1, numPlanes * outHeight); - for (int y = 0; y < outHeight; ++y) + if (depth == CV_8S) { - float input_y = y * scaleHeight; - int y0 = static_cast(input_y); - const float* inpData_row0 = inpPlanes.ptr(y0); - const float* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); - for (int x = 0; x < outWidth; ++x) + for (int y = 0; y < outHeight; ++y) { - float input_x = x * scaleWidth; - int x0 = static_cast(input_x); - int x1 = std::min(x0 + 1, inpWidth - 1); - - float* outData = outPlanes.ptr(y, x); - const float* inpData_row0_c = inpData_row0; - const float* inpData_row1_c = inpData_row1; - for (int c = 0; c < numPlanes; ++c) + float input_y = halfPixelCenters ? std::max((y + 0.5f) * scaleHeight - 0.5f, 0.0f) : y * scaleHeight; + int y0 = static_cast(input_y); + const int8_t* inpData_row0 = inpPlanes.ptr(y0); + const int8_t* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); + for (int x = 0; x < outWidth; ++x) { - *outData = inpData_row0_c[x0] + - (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + - (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + - (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0])); + float input_x = halfPixelCenters ? std::max((x + 0.5f) * scaleWidth - 0.5f, 0.0f) : x * scaleWidth; + int x0 = static_cast(input_x); + int x1 = std::min(x0 + 1, inpWidth - 1); - inpData_row0_c += inpSpatialSize; - inpData_row1_c += inpSpatialSize; - outData += outSpatialSize; + int8_t* outData = outPlanes.ptr(y, x); + const int8_t* inpData_row0_c = inpData_row0; + const int8_t* inpData_row1_c = inpData_row1; + for (int c = 0; c < numPlanes; ++c) + { + *outData = static_cast(inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + + (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0]))); + + inpData_row0_c += inpSpatialSize; + inpData_row1_c += inpSpatialSize; + outData += outSpatialSize; + } + } + } + } + else + { + for (int y = 0; y < outHeight; ++y) + { + float input_y = y * scaleHeight; + int y0 = static_cast(input_y); + const float* inpData_row0 = inpPlanes.ptr(y0); + const float* inpData_row1 = inpPlanes.ptr(std::min(y0 + 1, inpHeight - 1)); + for (int x = 0; x < outWidth; ++x) + { + float input_x = x * scaleWidth; + int x0 = static_cast(input_x); + int x1 = std::min(x0 + 1, inpWidth - 1); + + float* outData = outPlanes.ptr(y, x); + const float* inpData_row0_c = inpData_row0; + const float* inpData_row1_c = inpData_row1; + for (int c = 0; c < numPlanes; ++c) + { + *outData = inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x0] - inpData_row0_c[x0]) + + (input_x - x0) * (inpData_row0_c[x1] - inpData_row0_c[x0] + + (input_y - y0) * (inpData_row1_c[x1] - inpData_row0_c[x1] - inpData_row1_c[x0] + inpData_row0_c[x0])); + + inpData_row0_c += inpSpatialSize; + inpData_row1_c += inpSpatialSize; + outData += outSpatialSize; + } } } } @@ -356,6 +434,11 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } protected: int outWidth, outHeight; diff --git a/modules/dnn/src/layers/scale_layer.cpp b/modules/dnn/src/layers/scale_layer.cpp index a5c268214e..001db24a2d 100644 --- a/modules/dnn/src/layers/scale_layer.cpp +++ b/modules/dnn/src/layers/scale_layer.cpp @@ -344,6 +344,14 @@ public: shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat(); } + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + params.set("input_scales", DictValue::arrayReal(scales[0].data(), scales[0].size())); + params.set("input_zeropoints", DictValue::arrayInt(zeropoints[0].data(), zeropoints[0].size())); + return true; + } + virtual int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/shuffle_channel_layer.cpp b/modules/dnn/src/layers/shuffle_channel_layer.cpp index 6db74d1abd..2a698d270f 100644 --- a/modules/dnn/src/layers/shuffle_channel_layer.cpp +++ b/modules/dnn/src/layers/shuffle_channel_layer.cpp @@ -147,6 +147,12 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + return true; + } + private: Ptr permute; std::vector permuteInpShape, permuteOutShape; diff --git a/modules/dnn/src/layers/slice_layer.cpp b/modules/dnn/src/layers/slice_layer.cpp index 54e2340387..20b493636a 100644 --- a/modules/dnn/src/layers/slice_layer.cpp +++ b/modules/dnn/src/layers/slice_layer.cpp @@ -64,6 +64,31 @@ namespace cv namespace dnn { +void sliceRangesFromShape(const MatShape& inpShape, int& axis, std::vector >& sliceRanges) +{ + CV_Assert(inpShape.size() > 0); + bool axisNeg = (axis < 0); + axis = (axis + static_cast(inpShape.size())) % inpShape.size(); + int n = inpShape[axis]; + + for (size_t i = 0; i < sliceRanges.size(); ++i){ + std::vector& ranges = sliceRanges[i]; + if (axisNeg) + { + ranges.insert(ranges.begin(), axis, Range::all()); + } + Range& range = ranges.back(); + + if (range.start >= 0) + { + continue; + } + + CV_Assert(n != 0); + range.start = (n + range.start) % n; + } +} + class SliceLayerImpl : public SliceLayer { public: @@ -75,20 +100,22 @@ public: num_split = params.get("num_split", 0); hasDynamicShapes = params.get("has_dynamic_shapes", false); shapesInitialized = !hasDynamicShapes; + if (params.has("slice_point")) { CV_Assert(!params.has("begin") && !params.has("size") && !params.has("end")); const DictValue &indicesValue = params.get("slice_point"); + int size = axis > 0 ? axis + 1 : 1; sliceRanges.resize(indicesValue.size() + 1, - std::vector(std::max(axis,0) + 1, Range::all())); + std::vector(size, Range::all())); int prevSlice = 0; for (int i = 0; i < indicesValue.size(); ++i) { - sliceRanges[i][axis].start = prevSlice; - sliceRanges[i][axis].end = indicesValue.get(i); - prevSlice = sliceRanges[i][axis].end; + sliceRanges[i][size - 1].start = prevSlice; + sliceRanges[i][size - 1].end = indicesValue.get(i); + prevSlice = sliceRanges[i][size - 1].end; } - sliceRanges.back()[axis].start = prevSlice; + sliceRanges.back()[size - 1].start = prevSlice; } else if (params.has("begin")) { @@ -103,7 +130,6 @@ public: { int start = begins.get(i); int sizeOrEnd = sizesOrEnds.get(i); // It may be negative to reverse indexation. - CV_Assert(start >= 0); sliceRanges[0][i].start = start; if (params.has("size")) @@ -164,16 +190,20 @@ public: CV_Assert(inputs.size() == 1); MatShape inpShape = inputs[0]; - if (!sliceRanges.empty()) + int axis_rw = axis; + std::vector > sliceRanges_rw = sliceRanges; + sliceRangesFromShape(inpShape, axis_rw, sliceRanges_rw); + + if (!sliceRanges_rw.empty()) { - outputs.resize(sliceRanges.size(), inpShape); + outputs.resize(sliceRanges_rw.size(), inpShape); for (int i = 0; i < outputs.size(); ++i) { - CV_Assert(sliceRanges[i].size() <= inpShape.size()); - for (int j = 0; j < sliceRanges[i].size(); ++j) + CV_Assert(sliceRanges_rw[i].size() <= inpShape.size()); + for (int j = 0; j < sliceRanges_rw[i].size(); ++j) { if (shapesInitialized || inpShape[j] > 0) - outputs[i][j] = normalize_axis_range(sliceRanges[i][j], inpShape[j]).size(); + outputs[i][j] = normalize_axis_range(sliceRanges_rw[i][j], inpShape[j]).size(); if (!sliceSteps.empty() && (i < sliceSteps.size()) && (j < sliceSteps[i].size()) && (sliceSteps[i][j] > 1)) outputs[i][j] = (outputs[i][j] + sliceSteps[i][j] - 1) / sliceSteps[i][j]; @@ -182,10 +212,10 @@ public: } else // Divide input blob on equal parts by axis. { - CV_Assert(0 <= axis && axis < inpShape.size()); + CV_Assert(0 <= axis_rw && axis_rw < inpShape.size()); int splits = num_split ? num_split : requiredOutputs; - CV_Assert(splits > 0 && inpShape[axis] % splits == 0); - inpShape[axis] /= splits; + CV_Assert(splits > 0 && inpShape[axis_rw] % splits == 0); + inpShape[axis_rw] /= splits; outputs.resize(splits, inpShape); } return false; @@ -210,6 +240,7 @@ public: CV_Assert(inputs.size() == 1); const MatSize& inpShape = inputs[0].size; + sliceRangesFromShape(shape(inputs[0]), axis, sliceRanges); finalSliceRanges = sliceRanges; if (sliceRanges.empty()) @@ -492,7 +523,7 @@ public: ocl::KernelArg::PtrReadOnly(input), ocl::KernelArg::PtrWriteOnly(output) ) - .run(2, (size_t*)ocl.global_size, (size_t*)ocl.local_size, false); + .run_(2, (size_t*)ocl.global_size, (size_t*)ocl.local_size, false); if (!ret) return false; } // for outputs.size() @@ -531,7 +562,12 @@ public: { std::vector inpIdx(dimsNum, 0); std::vector outIdx(dimsNum, 0); - getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + if (inpMat.type() == CV_16S) + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + else if (inpMat.type() == CV_8S) + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); + else + getSliceRecursive(inpMat, inpIdx, finalSliceRanges[i], sliceSteps[i], 0, dimsNum, outputs[i], outIdx); } } } @@ -647,8 +683,20 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + const int numOutputs = scales[1].size(); + for (int i = 0; i < numOutputs; i++) + { + if (scales[1][i] != scales[0][0]) + return false; + } + return true; + } private: + template void getSliceRecursive(const Mat &inpMat, std::vector &inpIdx, const std::vector &sliceRanges, const std::vector &sliceSteps, int dim, int dimsNum, @@ -658,8 +706,6 @@ private: int end = sliceRanges[dim].end; int step = !sliceSteps.empty() ? sliceSteps[dim] : 1; - const bool is32F = inpMat.depth() == CV_32F; - // TODO optimization is required (for 2D tail case at least) for (int k = begin, j = 0; k < end; k += step, j++) { @@ -667,14 +713,9 @@ private: outIdx[dim] = j; if (dim + 1 < dimsNum) - getSliceRecursive(inpMat, inpIdx, sliceRanges, sliceSteps, dim + 1, dimsNum, outputs, outIdx); + getSliceRecursive(inpMat, inpIdx, sliceRanges, sliceSteps, dim + 1, dimsNum, outputs, outIdx); else - { - if (is32F) - outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); - else - outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); // 16F emulation - } + outputs.at(outIdx.data()) = inpMat.at(inpIdx.data()); } } diff --git a/modules/dnn/src/layers/softmax_layer.cpp b/modules/dnn/src/layers/softmax_layer.cpp index 546c1017ad..e937e98f8c 100644 --- a/modules/dnn/src/layers/softmax_layer.cpp +++ b/modules/dnn/src/layers/softmax_layer.cpp @@ -374,6 +374,22 @@ public: } #endif // HAVE_DNN_NGRAPH + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + float inpScale = scales[0][0]; + Mat lookUpTable(1, 256, CV_32F); + float* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inpScale*(i - 127); // ensures exp(x) is always between (0, 1) + table[i+128] = std::exp(x); + } + params.blobs.clear(); + params.blobs.push_back(lookUpTable); + return true; + } + int64 getFLOPS(const std::vector &inputs, const std::vector &outputs) const CV_OVERRIDE { diff --git a/modules/dnn/src/layers/split_layer.cpp b/modules/dnn/src/layers/split_layer.cpp index b025d5ff1e..2a44176152 100644 --- a/modules/dnn/src/layers/split_layer.cpp +++ b/modules/dnn/src/layers/split_layer.cpp @@ -117,6 +117,17 @@ public: } #endif + virtual bool tryQuantize(const std::vector > &scales, + const std::vector > &zeropoints, LayerParams& params) CV_OVERRIDE + { + const int numOutputs = scales[1].size(); + for (int i = 0; i < numOutputs; i++) + { + if (scales[1][i] != scales[0][0]) + return false; + } + return true; + } }; Ptr SplitLayer::create(const LayerParams& params) diff --git a/modules/dnn/src/math_utils.hpp b/modules/dnn/src/math_utils.hpp new file mode 100644 index 0000000000..19ee474c73 --- /dev/null +++ b/modules/dnn/src/math_utils.hpp @@ -0,0 +1,83 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// Code is borrowed from https://github.com/kaldi-asr/kaldi/blob/master/src/base/kaldi-math.h + +// base/kaldi-math.h + +// Copyright 2009-2011 Ondrej Glembek; Microsoft Corporation; Yanmin Qian; +// Jan Silovsky; Saarland University +// +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef __OPENCV_DNN_MATH_UTILS_HPP__ +#define __OPENCV_DNN_MATH_UTILS_HPP__ + +#ifdef OS_QNX +#include +#else +#include +#endif + +#include + +#ifndef FLT_EPSILON +#define FLT_EPSILON 1.19209290e-7f +#endif + +namespace cv { namespace dnn { + +const float kNegativeInfinity = -std::numeric_limits::infinity(); + +const float kMinLogDiffFloat = std::log(FLT_EPSILON); + +#if !defined(_MSC_VER) || (_MSC_VER >= 1700) +inline float Log1p(float x) { return log1pf(x); } +#else +inline float Log1p(float x) { + const float cutoff = 1.0e-07; + if (x < cutoff) + return x - 2 * x * x; + else + return Log(1.0 + x); +} +#endif + +inline float Exp(float x) { return expf(x); } + +inline float LogAdd(float x, float y) { + float diff; + if (x < y) { + diff = x - y; + x = y; + } else { + diff = y - x; + } + // diff is negative. x is now the larger one. + + if (diff >= kMinLogDiffFloat) { + float res; + res = x + Log1p(Exp(diff)); + return res; + } else { + return x; // return the larger one. + } +} + +}} // namespace + +#endif // __OPENCV_DNN_MATH_UTILS_HPP__ diff --git a/modules/dnn/src/model.cpp b/modules/dnn/src/model.cpp index 0af8223a7f..bc8709d22e 100644 --- a/modules/dnn/src/model.cpp +++ b/modules/dnn/src/model.cpp @@ -3,8 +3,10 @@ // of this distribution and at http://opencv.org/license.html. #include "precomp.hpp" +#include "math_utils.hpp" #include #include +#include #include #include @@ -552,6 +554,9 @@ struct TextRecognitionModel_Impl : public Model::Impl std::string decodeType; std::vector vocabulary; + int beamSize = 10; + int vocPruneSize = 0; + TextRecognitionModel_Impl() { CV_TRACE_FUNCTION(); @@ -575,6 +580,13 @@ struct TextRecognitionModel_Impl : public Model::Impl decodeType = type; } + inline + void setDecodeOptsCTCPrefixBeamSearch(int beam, int vocPrune) + { + beamSize = beam; + vocPruneSize = vocPrune; + } + virtual std::string decode(const Mat& prediction) { @@ -586,44 +598,10 @@ struct TextRecognitionModel_Impl : public Model::Impl CV_Error(Error::StsBadArg, "TextRecognitionModel: vocabulary is not specified"); std::string decodeSeq; - if (decodeType == "CTC-greedy") - { - CV_CheckEQ(prediction.dims, 3, ""); - CV_CheckType(prediction.type(), CV_32FC1, ""); - const int vocLength = (int)(vocabulary.size()); - CV_CheckLE(prediction.size[1], vocLength, ""); - bool ctcFlag = true; - int lastLoc = 0; - for (int i = 0; i < prediction.size[0]; i++) - { - const float* pred = prediction.ptr(i); - int maxLoc = 0; - float maxScore = pred[0]; - for (int j = 1; j < vocLength + 1; j++) - { - float score = pred[j]; - if (maxScore < score) - { - maxScore = score; - maxLoc = j; - } - } - - if (maxLoc > 0) - { - std::string currentChar = vocabulary.at(maxLoc - 1); - if (maxLoc != lastLoc || ctcFlag) - { - lastLoc = maxLoc; - decodeSeq += currentChar; - ctcFlag = false; - } - } - else - { - ctcFlag = true; - } - } + if (decodeType == "CTC-greedy") { + decodeSeq = ctcGreedyDecode(prediction); + } else if (decodeType == "CTC-prefix-beam-search") { + decodeSeq = ctcPrefixBeamSearchDecode(prediction); } else if (decodeType.length() == 0) { CV_Error(Error::StsBadArg, "Please set decodeType"); } else { @@ -633,6 +611,200 @@ struct TextRecognitionModel_Impl : public Model::Impl return decodeSeq; } + virtual + std::string ctcGreedyDecode(const Mat& prediction) + { + std::string decodeSeq; + CV_CheckEQ(prediction.dims, 3, ""); + CV_CheckType(prediction.type(), CV_32FC1, ""); + const int vocLength = (int)(vocabulary.size()); + CV_CheckLE(prediction.size[1], vocLength, ""); + bool ctcFlag = true; + int lastLoc = 0; + for (int i = 0; i < prediction.size[0]; i++) + { + const float* pred = prediction.ptr(i); + int maxLoc = 0; + float maxScore = pred[0]; + for (int j = 1; j < vocLength + 1; j++) + { + float score = pred[j]; + if (maxScore < score) + { + maxScore = score; + maxLoc = j; + } + } + + if (maxLoc > 0) + { + std::string currentChar = vocabulary.at(maxLoc - 1); + if (maxLoc != lastLoc || ctcFlag) + { + lastLoc = maxLoc; + decodeSeq += currentChar; + ctcFlag = false; + } + } + else + { + ctcFlag = true; + } + } + return decodeSeq; + } + + struct PrefixScore + { + // blank ending score + float pB; + // none blank ending score + float pNB; + + PrefixScore() : pB(kNegativeInfinity), pNB(kNegativeInfinity) + { + + } + PrefixScore(float pB, float pNB) : pB(pB), pNB(pNB) + { + + } + }; + + struct PrefixHash + { + size_t operator()(const std::vector& prefix) const + { + // BKDR hash + unsigned int seed = 131; + size_t hash = 0; + for (size_t i = 0; i < prefix.size(); i++) + { + hash = hash * seed + prefix[i]; + } + return hash; + } + }; + + static + std::vector> TopK( + const float* predictions, int length, int k) + { + std::vector> results; + // No prune. + if (k <= 0) + { + for (int i = 0; i < length; ++i) + { + results.emplace_back(predictions[i], i); + } + return results; + } + + for (int i = 0; i < k; ++i) + { + results.emplace_back(predictions[i], i); + } + std::make_heap(results.begin(), results.end(), std::greater>{}); + + for (int i = k; i < length; ++i) + { + if (predictions[i] > results.front().first) + { + std::pop_heap(results.begin(), results.end(), std::greater>{}); + results.pop_back(); + results.emplace_back(predictions[i], i); + std::push_heap(results.begin(), results.end(), std::greater>{}); + } + } + return results; + } + + static inline + bool PrefixScoreCompare( + const std::pair, PrefixScore>& a, + const std::pair, PrefixScore>& b) + { + float probA = LogAdd(a.second.pB, a.second.pNB); + float probB = LogAdd(b.second.pB, b.second.pNB); + return probA > probB; + } + + virtual + std::string ctcPrefixBeamSearchDecode(const Mat& prediction) { + // CTC prefix beam seach decode. + // For more detail, refer to: + // https://distill.pub/2017/ctc/#inference + // https://gist.github.com/awni/56369a90d03953e370f3964c826ed4b0i + using Beam = std::vector, PrefixScore>>; + using BeamInDict = std::unordered_map, PrefixScore, PrefixHash>; + + CV_CheckType(prediction.type(), CV_32FC1, ""); + CV_CheckEQ(prediction.dims, 3, ""); + CV_CheckEQ(prediction.size[1], 1, ""); + CV_CheckEQ(prediction.size[2], (int)vocabulary.size() + 1, ""); // Length add 1 for ctc blank + + std::string decodeSeq; + Beam beam = {std::make_pair(std::vector(), PrefixScore(0.0, kNegativeInfinity))}; + for (int i = 0; i < prediction.size[0]; i++) + { + // Loop over time + BeamInDict nextBeam; + const float* pred = prediction.ptr(i); + std::vector> topkPreds = + TopK(pred, vocabulary.size() + 1, vocPruneSize); + for (const auto& each : topkPreds) + { + // Loop over vocabulary + float prob = each.first; + int token = each.second; + for (const auto& it : beam) + { + const std::vector& prefix = it.first; + const PrefixScore& prefixScore = it.second; + if (token == 0) // 0 stands for ctc blank + { + PrefixScore& nextScore = nextBeam[prefix]; + nextScore.pB = LogAdd(nextScore.pB, + LogAdd(prefixScore.pB + prob, prefixScore.pNB + prob)); + continue; + } + + std::vector nPrefix(prefix); + nPrefix.push_back(token); + PrefixScore& nextScore = nextBeam[nPrefix]; + if (prefix.size() > 0 && token == prefix.back()) + { + nextScore.pNB = LogAdd(nextScore.pNB, prefixScore.pB + prob); + PrefixScore& mScore = nextBeam[prefix]; + mScore.pNB = LogAdd(mScore.pNB, prefixScore.pNB + prob); + } + else + { + nextScore.pNB = LogAdd(nextScore.pNB, + LogAdd(prefixScore.pB + prob, prefixScore.pNB + prob)); + } + } + } + // Beam prune + Beam newBeam(nextBeam.begin(), nextBeam.end()); + int newBeamSize = std::min(static_cast(newBeam.size()), beamSize); + std::nth_element(newBeam.begin(), newBeam.begin() + newBeamSize, + newBeam.end(), PrefixScoreCompare); + newBeam.resize(newBeamSize); + std::sort(newBeam.begin(), newBeam.end(), PrefixScoreCompare); + beam = std::move(newBeam); + } + + CV_Assert(!beam.empty()); + for (int token : beam[0].first) + { + CV_Check(token, token > 0 && token <= vocabulary.size(), ""); + decodeSeq += vocabulary.at(token - 1); + } + return decodeSeq; + } + virtual std::string recognize(InputArray frame) { @@ -698,6 +870,12 @@ const std::string& TextRecognitionModel::getDecodeType() const return TextRecognitionModel_Impl::from(impl).decodeType; } +TextRecognitionModel& TextRecognitionModel::setDecodeOptsCTCPrefixBeamSearch(int beamSize, int vocPruneSize) +{ + TextRecognitionModel_Impl::from(impl).setDecodeOptsCTCPrefixBeamSearch(beamSize, vocPruneSize); + return *this; +} + TextRecognitionModel& TextRecognitionModel::setVocabulary(const std::vector& inputVoc) { TextRecognitionModel_Impl::from(impl).setVocabulary(inputVoc); diff --git a/modules/dnn/src/nms.cpp b/modules/dnn/src/nms.cpp index 26489dc7e9..d321cfed88 100644 --- a/modules/dnn/src/nms.cpp +++ b/modules/dnn/src/nms.cpp @@ -58,6 +58,83 @@ void NMSBoxes(const std::vector& bboxes, const std::vector& NMSFast_(bboxes, scores, score_threshold, nms_threshold, eta, top_k, indices, rotatedRectIOU); } +void softNMSBoxes(const std::vector& bboxes, + const std::vector& scores, + std::vector& updated_scores, + const float score_threshold, + const float nms_threshold, + std::vector& indices, + size_t top_k, + const float sigma, + SoftNMSMethod method) +{ + CV_Assert_N(bboxes.size() == scores.size(), score_threshold >= 0, + nms_threshold >= 0, sigma >= 0); + + indices.clear(); + updated_scores.clear(); + + std::vector > score_index_vec(scores.size()); + for (size_t i = 0; i < scores.size(); i++) + { + score_index_vec[i].first = scores[i]; + score_index_vec[i].second = i; + } + + const auto score_cmp = [](const std::pair& a, const std::pair& b) + { + return a.first == b.first ? a.second > b.second : a.first < b.first; + }; + + top_k = top_k == 0 ? scores.size() : std::min(top_k, scores.size()); + ptrdiff_t start = 0; + while (indices.size() < top_k) + { + auto it = std::max_element(score_index_vec.begin() + start, score_index_vec.end(), score_cmp); + + float bscore = it->first; + size_t bidx = it->second; + + if (bscore < score_threshold) + { + break; + } + + indices.push_back(static_cast(bidx)); + updated_scores.push_back(bscore); + std::swap(score_index_vec[start], *it); // first start elements are chosen + + for (size_t i = start + 1; i < scores.size(); ++i) + { + float& bscore_i = score_index_vec[i].first; + const size_t bidx_i = score_index_vec[i].second; + + if (bscore_i < score_threshold) + { + continue; + } + + float overlap = rectOverlap(bboxes[bidx], bboxes[bidx_i]); + + switch (method) + { + case SoftNMSMethod::SOFTNMS_LINEAR: + if (overlap > nms_threshold) + { + bscore_i *= 1.f - overlap; + } + break; + case SoftNMSMethod::SOFTNMS_GAUSSIAN: + bscore_i *= exp(-(overlap * overlap) / sigma); + break; + default: + CV_Error(Error::StsBadArg, "Not supported SoftNMS method."); + } + } + ++start; + } +} + CV__DNN_INLINE_NS_END }// dnn }// cv diff --git a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp index 7bb277d102..bf5fba71a1 100644 --- a/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp +++ b/modules/dnn/src/ocl4dnn/include/ocl4dnn.hpp @@ -222,8 +222,6 @@ class OCL4DNNConvSpatial bool createDWConvKernel(int32_t blockWidth, int32_t blockHeight, int32_t blockDepth); - void CreateSubBuffer(const UMat& buffer, UMat& sub_buffer, - int32_t offset, int32_t size, bool write_only); bool convolve(const UMat &bottom, UMat &top, const UMat &weight, const UMat &bias, int32_t numImages, @@ -269,7 +267,7 @@ class OCL4DNNConvSpatial void generate_idlf_tuneritems(std::vector< cv::Ptr > &tunerItems, int blockM, int blockK, int simd_size); void setFusionDefine(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise); - void setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, ocl::Kernel &kernel, cl_uint &argIdx); + void setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, int fused_eltwise_offset, ocl::Kernel &kernel, cl_uint &argIdx); int32_t group_; bool bias_term_; diff --git a/modules/dnn/src/ocl4dnn/src/math_functions.cpp b/modules/dnn/src/ocl4dnn/src/math_functions.cpp index 855a21e08f..c924d66b12 100644 --- a/modules/dnn/src/ocl4dnn/src/math_functions.cpp +++ b/modules/dnn/src/ocl4dnn/src/math_functions.cpp @@ -116,6 +116,7 @@ ocl::Image2D ocl4dnnGEMMCopyBufferToImage(UMat buffer, int offset, .args( ocl::KernelArg::PtrReadOnly(buffer), image, offset, + padded_width, padded_height, width, height, ld) .run(2, global_copy, NULL, false); diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp index 059fc8f402..beb5fcd96d 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_conv_spatial.cpp @@ -167,6 +167,7 @@ OCL4DNNConvSpatial::OCL4DNNConvSpatial(OCL4DNNConvConfig config) channels_ = config.in_shape[dims - spatial_dims - 1]; num_output_ = config.out_shape[dims - spatial_dims - 1]; group_ = config.group; + CV_CheckGT(group_, 0, ""); // avoid div by zero below fused_activ_ = OCL4DNN_CONV_FUSED_ACTIV_NONE; fused_eltwise_ = false; @@ -218,14 +219,7 @@ OCL4DNNConvSpatial::OCL4DNNConvSpatial(OCL4DNNConvConfig config) #endif if (!use_cache_path_) { - static int warn_ = 0; - if (!warn_) - { - std::cerr - << "OpenCV(ocl4dnn): Kernel configuration cache directory doesn't exist: " << cache_path_ << std::endl - << std::endl; - warn_ = true; - } + CV_LOG_ONCE_ERROR(NULL, "OpenCV(ocl4dnn): Kernel configuration cache directory doesn't exist: " << cache_path_); } } @@ -270,17 +264,21 @@ void OCL4DNNConvSpatial::setFusionDefine(ocl4dnnFusedActiv_t fused_activ, } template -void OCL4DNNConvSpatial::setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, ocl::Kernel &kernel, cl_uint &argIdx) +void OCL4DNNConvSpatial::setFusionArg(ocl4dnnFusedActiv_t fused_activ, bool fused_eltwise, int fused_eltwise_offset, ocl::Kernel &kernel, cl_uint &argIdx) { if (fused_eltwise) - kernel.set(argIdx++, (cl_mem)bottom_data2_.handle(ACCESS_READ)); + { + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom_data2_)); + if (fused_eltwise_offset >= 0) + kernel.set(argIdx++, fused_eltwise_offset); + } switch (fused_activ) { case OCL4DNN_CONV_FUSED_ACTIV_RELU: kernel.set(argIdx++, (float)negative_slope_); break; case OCL4DNN_CONV_FUSED_ACTIV_PRELU: - kernel.set(argIdx++, (cl_mem)negative_slope_umat_.handle(ACCESS_READ)); + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(negative_slope_umat_)); break; case OCL4DNN_CONV_FUSED_ACTIV_POWER: kernel.set(argIdx++, (float)power_); @@ -414,7 +412,6 @@ void OCL4DNNConvSpatial::setupKernelDetails(int32_t kernelType, addDef("CHANNELS", channels_ / group_); addDef("APPLY_BIAS", bias_term_); addDef("OUTPUT_Z", M_); - addDef("ZPAR", 1); setFusionDefine(fused_activ_, fused_eltwise_); src_ = cv::ocl::dnn::conv_layer_spatial_oclsrc; @@ -668,8 +665,7 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, int r, int c, int interleavedRows, int nonInterleavedRows, int blockWidth, int rowAlignment ) { - CHECK_EQ(interleavedRows % 2, 0) << - "interleaveMatrix only supports even values for interleavedRows."; + CV_Check(interleavedRows, interleavedRows % 2 == 0, "interleaveMatrix only supports even values for interleavedRows."); size_t memSize = r * c * sizeof(float); size_t dstSize = memSize * @@ -681,9 +677,12 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, const int yStride = c * 2; const Dtype *pSrc = mem; Dtype* pDst = mem_dst; - for (int y = 0; y < r;) { - for (int rows = 0; rows < interleavedRows; rows += 2) { - if ( y >= r ) break; + for (int y = 0; y < r;) + { + for (int rows = 0; rows < interleavedRows; rows += 2) + { + if (y >= r) + break; if ((c % xStride) == 0) { for (int x = 0; x < c / xStride; x++) { memcpy(pDst + x * xStride * 2, // NOLINT @@ -708,11 +707,14 @@ void interleaveMatrix(Dtype* mem_dst, const Dtype *mem, y += 2; } - for (int rows = 0; rows < nonInterleavedRows; rows++) { - if (y >= r) break; + for (int rows = 0; rows < nonInterleavedRows; rows++) + { + if (y >= r) + break; const int stride = rowAlignment; int remaining = c; - for (int x = 0; x < c; x += stride) { + for (int x = 0; x < c; x += stride) + { if (remaining >= stride) { memcpy(pDst + x * 2, pSrc + x, stride * sizeof(Dtype)); // NOLINT remaining -=stride; @@ -765,12 +767,11 @@ bool OCL4DNNConvSpatial::swizzleWeight(const UMat &weight, swizzled_factor ); - size_t global_work_size_copy[3] = { - (size_t) (alignSize(num_output_, swizzled_factor) * channels * kernel_w_ * kernel_h_), 1, 1 }; + size_t global_work_size_copy[1] = { (size_t)(alignSize(num_output_, swizzled_factor) * channels * kernel_w_ * kernel_h_) }; - if (!oclk_copy_weight.run(3, global_work_size_copy, NULL, false)) + if (!oclk_copy_weight.run_(1, global_work_size_copy, NULL, false)) { - std::cout << "Swizzle kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: Swizzle kernel run failed"); return false; } } else { @@ -849,34 +850,6 @@ bool OCL4DNNConvSpatial::createBasicKernel(int32_t blockWidth, return false; } -template<> -void OCL4DNNConvSpatial::CreateSubBuffer(const UMat& buffer, UMat& sub_buffer, - int32_t offset, int32_t size, bool write_only) -{ - cl_mem sub_mem; - cl_buffer_region region; - cl_int err; - size_t element_size = (use_half_) ? sizeof(short) : sizeof(float); - - region.origin = offset * element_size + buffer.offset; - region.size = size * element_size; - sub_mem = clCreateSubBuffer((cl_mem)buffer.handle(ACCESS_READ), - write_only ? CL_MEM_WRITE_ONLY : CL_MEM_READ_ONLY, - CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); - if (err) - { - std::cout << "Failed to create sub buffer." << std::endl; - return; - } - - int step = element_size, rows = size, cols = 1; - ocl::convertFromBuffer(sub_mem, step, rows, cols, - (use_half_) ? CV_16SC1 : CV_32FC1, sub_buffer); - - //decrease ocl mem refcount - clReleaseMemObject(sub_mem); -} - template<> bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, const UMat &weight, const UMat &bias, @@ -895,10 +868,12 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, if (config->kernelType == KERNEL_TYPE_INTEL_IDLF) { if (!swizzleWeight(weight, config->workItem_output[2], false)) return false; +#if 0 size_t total_bottom_size = bottom_dim_ * numImages; size_t total_kernel_size = kernel_h_ * kernel_w_ * channels_ * M_; size_t total_bias_size = M_ * group_; size_t total_top_size = top_dim_ * numImages; +#endif for (int32_t g = 0; g < group_; ++g) { bias_offset = M_ * g; int32_t image_offset = width_ * height_ * (channels_ / group_) * g; @@ -910,89 +885,41 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, output_image_offset, kernel, argIdx); - UMat img_buffer; - if (image_offset) - { - CreateSubBuffer(bottom, img_buffer, image_offset, - total_bottom_size - image_offset, false); - if (img_buffer.empty()) - return false; + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); + kernel.set(argIdx++, image_offset); - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(img_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); + kernel.set(argIdx++, kernel_offset); - UMat kernel_buffer; - if (kernel_offset) - { - CreateSubBuffer(swizzled_weights_umat, kernel_buffer, kernel_offset, - total_kernel_size - kernel_offset, false); - if (kernel_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(kernel_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); - } - - UMat bias_buffer; if (bias_term_) { - if (bias_offset) - { - CreateSubBuffer(bias, bias_buffer, bias_offset, - total_bias_size - bias_offset, false); - if (bias_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); + kernel.set(argIdx++, bias_offset); } - UMat out_buffer; - if (output_image_offset) - { - CreateSubBuffer(top, out_buffer, output_image_offset, - total_top_size - output_image_offset, true); - if (out_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(out_buffer)); - kernel.set(argIdx++, (int)(out_buffer.offset / element_size)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); - kernel.set(argIdx++, (int)(top.offset / element_size)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); + kernel.set(argIdx++, (int)(top.offset / element_size) + output_image_offset); kernel.set(argIdx++, (uint16_t)width_); kernel.set(argIdx++, (uint16_t)height_); kernel.set(argIdx++, (uint16_t)output_w_); kernel.set(argIdx++, (uint16_t)output_h_); - if (!kernel.run(3, config->global_work_size, config->local_work_size, false)) + if (!kernel.run_(3, config->global_work_size, config->local_work_size, false)) { - std::cout << "IDLF kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: IDLF kernel run failed"); return false; } } } else if (config->kernelType == KERNEL_TYPE_GEMM_LIKE) { if (!swizzleWeight(weight, config->workItem_output[1], true)) return false; +#if 0 size_t total_bottom_size = bottom_dim_ * numImages; size_t total_kernel_size = kernel_h_ * kernel_w_ * channels_ * M_; size_t total_bias_size = M_ * group_; +#endif size_t total_top_size = top_dim_ * numImages; for (int32_t g = 0; g < group_; ++g) { bias_offset = M_ * g; @@ -1005,72 +932,25 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, output_image_offset, kernel, argIdx); - UMat img_buffer; - if (image_offset) - { - CreateSubBuffer(bottom, img_buffer, image_offset, - total_bottom_size - image_offset, false); - if (img_buffer.empty()) - return false; + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); + kernel.set(argIdx++, (int)image_offset); + kernel.set(argIdx++, (int)(bottom.total() - image_offset)); - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(img_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); + kernel.set(argIdx++, (int)kernel_offset); + kernel.set(argIdx++, (int)(swizzled_weights_umat.total() - kernel_offset)); - UMat kernel_buffer; - if (kernel_offset) - { - CreateSubBuffer(swizzled_weights_umat, kernel_buffer, kernel_offset, - total_kernel_size - kernel_offset, false); - if (kernel_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(kernel_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(swizzled_weights_umat)); - } - - UMat bias_buffer; if (bias_term_) { - if (bias_offset) - { - CreateSubBuffer(bias, bias_buffer, bias_offset, - total_bias_size - bias_offset, false); - if (bias_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias_buffer)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bias)); + kernel.set(argIdx++, (int)bias_offset); } - UMat out_buffer; - if (output_image_offset) - { - CreateSubBuffer(top, out_buffer, output_image_offset, - total_top_size - output_image_offset, true); - if (out_buffer.empty()) - return false; - - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(out_buffer)); - kernel.set(argIdx++, (int)(out_buffer.offset / element_size)); - } - else - { - kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); - kernel.set(argIdx++, (int)(top.offset / element_size)); - } + kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); + kernel.set(argIdx++, (int)(top.offset / element_size) + output_image_offset); + kernel.set(argIdx++, (int)total_top_size - (int)(top.offset / element_size)); kernel.set(argIdx++, (uint16_t)width_); kernel.set(argIdx++, (uint16_t)height_); @@ -1100,9 +980,9 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, gy = alignSize(gy, blockK); size_t global_size[3] = { gx, gy, config->global_work_size[2] }; - if (!kernel.run(3, global_size, config->local_work_size, false)) + if (!kernel.run_(3, global_size, config->local_work_size, false)) { - std::cout << "GEMM like kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: GEMM like kernel run failed"); return false; } } @@ -1112,7 +992,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(weight)); if (bias_term_) @@ -1124,14 +1004,17 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_w_); kernel.set(argIdx++, (uint16_t)output_h_); - size_t global_size[3]; - global_size[0] = output_w_; - global_size[1] = output_h_; - global_size[2] = num_output_ * num_; - - if (!kernel.run(3, global_size, NULL, false)) + size_t wgs = kernel.workGroupSize(); + if (!wgs) { - std::cout << "DWCONV kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: Can't query workGroupSize of DWCONV kernel"); + return false; + } + size_t lws[1] = { wgs }; + size_t gws[1] = { roundUp((size_t)output_w_ * output_h_ * num_output_ * num_, (unsigned)lws[0]) }; + if (!kernel.run_(1, gws, lws, false)) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: DWCONV kernel run failed"); return false; } } else { @@ -1152,7 +1035,7 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, return false; cl_uint argIdx = 0; - setFusionArg(fused_activ_, fused_eltwise_, kernel, argIdx); + setFusionArg(fused_activ_, fused_eltwise_, -1, kernel, argIdx); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); kernel.set(argIdx++, image_offset); kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(weight)); @@ -1171,11 +1054,18 @@ bool OCL4DNNConvSpatial::convolve(const UMat &bottom, UMat &top, kernel.set(argIdx++, (uint16_t)output_h_); kernel.set(argIdx++, (uint16_t)pad_w_); kernel.set(argIdx++, (uint16_t)pad_h_); - if (!kernel.run(3, config->global_work_size, - (config->use_null_local) ? NULL : config->local_work_size, - false)) + + size_t wgs = kernel.workGroupSize(); + if (!wgs) { - std::cout << "Basic kernel run failed." << std::endl; + CV_LOG_ERROR(NULL, "DNN/OpenCL: Can't query workGroupSize of Basic kernel"); + return false; + } + size_t lws[1] = { wgs }; + size_t gws[1] = { roundUp((size_t)output_w_ * output_h_ * M_, (unsigned)lws[0]) }; + if (!kernel.run_(1, gws, lws, false)) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Basic kernel run failed"); return false; } } @@ -1195,14 +1085,9 @@ float OCL4DNNConvSpatial::timedConvolve(const UMat &bottom, UMat &top, { queue = cv::ocl::Queue::getDefault(); } - catch (const cv::Exception&) + catch (const std::exception& e) { - static int warn_ = 0; - if (!warn_) - { - std::cout << "OpenCV(ocl4dnn): Can't get OpenCL default queue for auto-tuning." << std::endl; - warn_ = true; - } + CV_LOG_ONCE_ERROR(NULL, "OpenCV(ocl4dnn): Can't get OpenCL default queue for auto-tuning: " << e.what()); return 1e6; } @@ -1257,8 +1142,11 @@ bool OCL4DNNConvSpatial::verifyResult(const UMat &bottom, else if (config->tested) return false; - int32_t sz[4] = {numImages, num_output_, output_h_, output_w_}; - top.zeros(4, sz, (use_half_) ? CV_16SC1 : CV_32FC1); + //int32_t sz[4] = {numImages, num_output_, output_h_, output_w_}; + CV_CheckEQ(top.total(), (size_t)numImages * num_output_ * output_h_ * output_w_, ""); + CV_CheckTypeEQ(top.type(), (use_half_) ? CV_16SC1 : CV_32FC1, ""); + top.setTo(Scalar::all(0)); + bool saved_tuned = tuned_; tuned_ = false; convolve(bottom, top, weight, bias, numImages, config); @@ -1403,9 +1291,9 @@ ocl::Program OCL4DNNConvSpatial::compileKernel() phash.insert(std::pair(kernel_name_, program)); if (!program.ptr()) { - std::cout << "Failed to compile kernel: " << kernel_name_ - << ", buildflags: " << options - << ", errmsg: " << errmsg << std::endl; + CV_LOG_WARNING(NULL, "DNN/OpenCL: Failed to compile kernel: " << kernel_name_ + << ", buildflags: '" << options << "', errmsg: '" << errmsg << "'" + ); } return program; } @@ -1434,26 +1322,13 @@ bool OCL4DNNConvSpatial::createGEMMLikeConvKernel(int32_t blockM, ocl::Program program = compileKernel(); if (program.ptr()) { - size_t workgroupSize_used; ocl::Kernel kernel(kernel_name_.c_str(), program); if (kernel.empty()) return false; - workgroupSize_used = kernel.preferedWorkGroupSizeMultiple(); - if (workgroupSize_used != simd_size) - { - std::cerr << "OpenCV(ocl4dnn): The OpenCL compiler chose a simd size (" << workgroupSize_used << ") that " << std::endl; - std::cerr << " does not equal the size (" << simd_size << ") kernel source required." << std::endl; - std::cerr << " Skip this kernel " << kernel_name_ << std::endl; - unloadProgram(kernel_name_); - return false; - } - else - { - kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], - true, KERNEL_TYPE_GEMM_LIKE)); - return true; - } + kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], + true, KERNEL_TYPE_GEMM_LIKE)); + return true; } else return false; @@ -1499,26 +1374,13 @@ bool OCL4DNNConvSpatial::createIDLFKernel(int32_t blockWidth, ocl::Program program = compileKernel(); if (program.ptr()) { - size_t workgroupSize_used; ocl::Kernel kernel(kernel_name_.c_str(), program); if (kernel.empty()) return false; - workgroupSize_used = kernel.preferedWorkGroupSizeMultiple(); - if (workgroupSize_used != simd_size) - { - std::cerr << "OpenCV(ocl4dnn): The OpenCL compiler chose a simd size (" << workgroupSize_used << ") that " << std::endl; - std::cerr << " does not equal the size (" << simd_size << ") kernel source required." << std::endl; - std::cerr << " Skip this kernel " << kernel_name_ << std::endl; - unloadProgram(kernel_name_); - return false; - } - else - { - kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], - true, KERNEL_TYPE_INTEL_IDLF)); - return true; - } + kernelQueue.push_back(makePtr(kernel_name_, &global_size[0], &local_size[0], &workItemOutput[0], + true, KERNEL_TYPE_INTEL_IDLF)); + return true; } else return false; @@ -1857,7 +1719,8 @@ void OCL4DNNConvSpatial::setupConvolution(const UMat &bottom, fastestTime = kernelQueue[x]->executionTime; } } - if (fastestKernel < 0) break; + if (fastestKernel < 0) + break; // Test fastest kernel bool verified = verifyResult(bottom, top, weight, bias, numImages, kernelQueue[fastestKernel], verifyTop); if (verified == true) { @@ -2016,17 +1879,18 @@ bool OCL4DNNConvSpatial::setupKernelByConfig(int x, int y, int z, int typ { if (z == 1) z = 16; - CHECK_EQ(z == 16 || z == 8, true) << "invalid SIMD size" << std::endl; + CV_Check(z, z == 16 || z == 8, "DNN/OpenCL: IDLF - invalid SIMD size"); } kernelQueue.clear(); createConvolutionKernel(type, x, y, z); - if (kernelQueue.size() != 1) { - std::cerr << "Failed setup kernel by config:" + if (kernelQueue.size() != 1) + { + CV_LOG_ERROR(NULL, "DNN/OpenCL: Failed setup kernel by config: " << " x = " << x << " y = " << y << " z = " << z << " type = " << type - << std::endl; + ); return false; } bestKernelConfig = kernelQueue[0]; @@ -2058,13 +1922,9 @@ bool OCL4DNNConvSpatial::loadTunedConfig() { if (cache_path_.empty()) { - static int warn_ = 0; - if (!warn_) - { - std::cout << "OpenCV(ocl4dnn): consider to specify kernel configuration cache directory " << std::endl - << " via OPENCV_OCL4DNN_CONFIG_PATH parameter." << std::endl; - warn_ = true; - } + CV_LOG_ONCE_WARNING(NULL, "OpenCV(ocl4dnn): consider to specify kernel configuration cache directory " + "through OPENCV_OCL4DNN_CONFIG_PATH parameter." + ); } return false; } diff --git a/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp b/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp index 78576711a7..7b32189fdc 100644 --- a/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp +++ b/modules/dnn/src/ocl4dnn/src/ocl4dnn_softmax.cpp @@ -127,7 +127,7 @@ bool OCL4DNNSoftmax::Forward(const UMat& bottom, UMat& top) oclk_softmax_forward_kernel.set(argIdx++, ocl::KernelArg::PtrReadOnly(bottom)); oclk_softmax_forward_kernel.set(argIdx++, ocl::KernelArg::PtrWriteOnly(top)); } - ret = oclk_softmax_forward_kernel.run(3, global_size, local_size, false); + ret = oclk_softmax_forward_kernel.run_(3, global_size, local_size, false); } return ret; } diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 7826f2b0ca..c4ebcd1c63 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -234,6 +234,27 @@ public: } }; +class NormalizeSubgraph2_2 : public NormalizeSubgraphBase +{ +public: + NormalizeSubgraph2_2() + { + int input = addNodeToMatch(""); + int norm = addNodeToMatch("ReduceL2", input); + + int min = addNodeToMatch(""); + int max = addNodeToMatch(""); + int clip = addNodeToMatch("Clip", norm, min, max); + + int shape = addNodeToMatch(""); + int expand = addNodeToMatch("Expand", clip, shape); + + addNodeToMatch("Div", input, expand); + + setFusedNode("Normalize", input); + } +}; + class NormalizeSubgraph3 : public NormalizeSubgraphBase { public: @@ -558,6 +579,7 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); + subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); subgraphs.push_back(makePtr()); @@ -572,7 +594,8 @@ void simplifySubgraphs(opencv_onnx::GraphProto& net) Mat getMatFromTensor(opencv_onnx::TensorProto& tensor_proto) { if (tensor_proto.raw_data().empty() && tensor_proto.float_data().empty() && - tensor_proto.double_data().empty() && tensor_proto.int64_data().empty()) + tensor_proto.double_data().empty() && tensor_proto.int64_data().empty() && + tensor_proto.int32_data().empty()) return Mat(); opencv_onnx::TensorProto_DataType datatype = tensor_proto.data_type(); @@ -641,6 +664,24 @@ Mat getMatFromTensor(opencv_onnx::TensorProto& tensor_proto) convertInt64ToInt32(src, dst, blob.total()); } } + else if (datatype == opencv_onnx::TensorProto_DataType_INT8 || + datatype == opencv_onnx::TensorProto_DataType_UINT8) + { + // TODO : Add support for uint8 weights and acitvations. For now, converting uint8 tensors to int8. + int offset = datatype == opencv_onnx::TensorProto_DataType_INT8 ? 0 : -128; + int depth = datatype == opencv_onnx::TensorProto_DataType_INT8 ? CV_8S : CV_8U; + + if (!tensor_proto.int32_data().empty()) + { + const ::google::protobuf::RepeatedField field = tensor_proto.int32_data(); + Mat(sizes, CV_32SC1, (void*)field.data()).convertTo(blob, CV_8S, 1.0, offset); + } + else + { + char* val = const_cast(tensor_proto.raw_data().c_str()); + Mat(sizes, depth, val).convertTo(blob, CV_8S, 1.0, offset); + } + } else { std::string errorMsg = "Unsupported data type: " + diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 98714bbd5c..4a2153d5a7 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -41,6 +41,8 @@ CV__DNN_INLINE_NS_BEGIN extern bool DNN_DIAGNOSTICS_RUN; +class ONNXLayerHandler; + class ONNXImporter { opencv_onnx::ModelProto model_proto; @@ -56,65 +58,25 @@ class ONNXImporter Mat getBlob(const std::string& input_name); LayerParams getLayerParams(const opencv_onnx::NodeProto& node_proto); - bool isCeilMode(const LayerParams& layerParams); void addConstant(const std::string& name, const Mat& blob); void addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); - static const std::set& getSupportedTypes(); + void handleQuantizedNode(LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto); + void expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, + const std::string& input, size_t n); + void addNegation(const LayerParams& layerParams, opencv_onnx::NodeProto& node_proto, int input_id); public: - - ONNXImporter(Net& net, const char *onnxFile) - : dstNet(net), utilNet() - { - hasDynamicShapes = false; - CV_Assert(onnxFile); - CV_LOG_DEBUG(NULL, "DNN/ONNX: processing ONNX model from file: " << onnxFile); - - std::fstream input(onnxFile, std::ios::in | std::ios::binary); - if (!input) - { - CV_Error(Error::StsBadArg, cv::format("Can't read ONNX file: %s", onnxFile)); - } - - if (!model_proto.ParseFromIstream(&input)) - { - CV_Error(Error::StsUnsupportedFormat, cv::format("Failed to parse ONNX model: %s", onnxFile)); - } - - populateNet(); - } - - ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) - : dstNet(net), utilNet() - { - hasDynamicShapes = false; - CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); - - struct _Buf : public std::streambuf - { - _Buf(const char* buffer, size_t sizeBuffer) - { - char* p = const_cast(buffer); - setg(p, p, p + sizeBuffer); - } - }; - - _Buf buf(buffer, sizeBuffer); - std::istream input(&buf); - - if (!model_proto.ParseFromIstream(&input)) - CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array."); - - populateNet(); - } + ONNXImporter(Net& net, const char *onnxFile); + ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer); void populateNet(); protected: + std::unique_ptr layerHandler; Net& dstNet; - Net utilNet; opencv_onnx::GraphProto graph_proto; std::string framework_name; @@ -129,8 +91,145 @@ protected: typedef std::map::iterator IterLayerId_t; void handleNode(const opencv_onnx::NodeProto& node_proto); + +private: + friend class ONNXLayerHandler; + typedef void (ONNXImporter::*ONNXImporterNodeParser)(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + typedef std::map DispatchMap; + + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); + + void parseMaxPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseAveragePool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSlice (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSplit (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBias (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePow (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMinMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseNeg (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLSTM (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGRU (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseImageScaler (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseClip (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseElu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTanh (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseLRN (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseBatchNormalization (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConvTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseFlatten (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUnsqueeze (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseExpand (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseReshape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parsePad (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseShape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCast (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConstantFill (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseGather (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseUpsample (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseSoftMax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQuantDequant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQEltwise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQSigmoid (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQAvgPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseQConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + + void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); }; +class ONNXLayerHandler : public detail::LayerHandler +{ +public: + explicit ONNXLayerHandler(ONNXImporter* importer_); + + void fillRegistry(const opencv_onnx::GraphProto& net); + +protected: + ONNXImporter* importer; +}; + +ONNXLayerHandler::ONNXLayerHandler(ONNXImporter* importer_) : importer(importer_){} + +void ONNXLayerHandler::fillRegistry(const opencv_onnx::GraphProto &net) +{ + int layersSize = net.node_size(); + for (int li = 0; li < layersSize; li++) { + const opencv_onnx::NodeProto &node_proto = net.node(li); + const std::string& name = node_proto.output(0); + const std::string& type = node_proto.op_type(); + if (importer->dispatch.find(type) == importer->dispatch.end()) + { + addMissing(name, type); + } + } + printMissing(); +} + +ONNXImporter::ONNXImporter(Net& net, const char *onnxFile) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) +{ + hasDynamicShapes = false; + CV_Assert(onnxFile); + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing ONNX model from file: " << onnxFile); + + std::fstream input(onnxFile, std::ios::in | std::ios::binary); + if (!input) + { + CV_Error(Error::StsBadArg, cv::format("Can't read ONNX file: %s", onnxFile)); + } + + if (!model_proto.ParseFromIstream(&input)) + { + CV_Error(Error::StsUnsupportedFormat, cv::format("Failed to parse ONNX model: %s", onnxFile)); + } + + populateNet(); +} + +ONNXImporter::ONNXImporter(Net& net, const char* buffer, size_t sizeBuffer) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new ONNXLayerHandler(this) : nullptr), dstNet(net), dispatch(buildDispatchMap()) +{ + hasDynamicShapes = false; + CV_LOG_DEBUG(NULL, "DNN/ONNX: processing in-memory ONNX model (" << sizeBuffer << " bytes)"); + + struct _Buf : public std::streambuf + { + _Buf(const char* buffer, size_t sizeBuffer) + { + char* p = const_cast(buffer); + setg(p, p, p + sizeBuffer); + } + }; + + _Buf buf(buffer, sizeBuffer); + std::istream input(&buf); + + if (!model_proto.ParseFromIstream(&input)) + CV_Error(Error::StsUnsupportedFormat, "Failed to parse onnx model from in-memory byte array."); + + populateNet(); +} + inline void replaceLayerParam(LayerParams& layerParams, const String& oldKey, const String& newKey) { if (layerParams.has(oldKey)) { @@ -153,7 +252,7 @@ void runLayer(LayerParams& params, const std::vector& inputs, CV_Assert((bool)layer); std::vector inpShapes(inputs.size()); - int ddepth = CV_32F; + int ddepth = params.get("depth", CV_32F); for (size_t i = 0; i < inputs.size(); ++i) { inpShapes[i] = shape(inputs[i]); @@ -202,6 +301,10 @@ static DictValue parse(const ::google::protobuf::RepeatedField< ::google::protob return DictValue::arrayInt(&dst[0], src.size()); } +static DictValue parseStr(const ::google::protobuf::RepeatedPtrField< ::std::string>& src) { + return DictValue::arrayString(src.begin(), static_cast(src.size())); +} + LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_proto) { LayerParams lp; @@ -261,6 +364,10 @@ LayerParams ONNXImporter::getLayerParams(const opencv_onnx::NodeProto& node_prot CV_Assert(attribute_proto.ints_size() == 1 || attribute_proto.ints_size() == 2 || attribute_proto.ints_size() == 3); lp.set("dilation", parse(attribute_proto.ints())); } + else if(attribute_name == "activations" && node_proto.op_type() == "LSTM") + { + lp.set(attribute_name, parseStr(attribute_proto.strings())); + } else if (attribute_proto.has_i()) { ::google::protobuf::int64 src = attribute_proto.i(); @@ -361,11 +468,8 @@ Mat ONNXImporter::getBlob(const std::string& input_name) void ONNXImporter::addLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(layerParams.name, layerParams.type, layerParams); - else - id = dstNet.addLayer(layerParams.name, layerParams.type, layerParams); + int depth = layerParams.get("depth", CV_32F); + int id = dstNet.addLayer(layerParams.name, layerParams.type, depth, layerParams); for (int i = 0; i < node_proto.output_size(); ++i) { layer_id.insert(std::make_pair(node_proto.output(i), LayerInfo(id, i))); @@ -378,10 +482,7 @@ void ONNXImporter::addLayer(LayerParams& layerParams, const std::string& input_name = node_proto.input(j); IterLayerId_t layerId = layer_id.find(input_name); if (layerId != layer_id.end()) { - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, inpNum); ++inpNum; // Collect input shapes. IterShape_t shapeIt = outShapes.find(input_name); @@ -390,11 +491,7 @@ void ONNXImporter::addLayer(LayerParams& layerParams, } } // Compute shape of output blob for this layer. - Ptr layer; - if (DNN_DIAGNOSTICS_RUN) - layer = utilNet.getLayer(id); - else - layer = dstNet.getLayer(id); // FIXIT: avoid instantiation of layers during the import stage + Ptr layer = dstNet.getLayer(id); // FIXIT: avoid instantiation of layers during the import stage layer->getMemoryShapes(layerInpShapes, 0, layerOutShapes, layerInternalShapes); for (int i = 0; i < node_proto.output_size() && i < (int)layerOutShapes.size(); ++i) { @@ -402,12 +499,114 @@ void ONNXImporter::addLayer(LayerParams& layerParams, } } +/** @brief Make N copies of input layer and set them as input to node_proto. + * @param prefix prefix of new layers' names + * @param node_proto node which will contain all copies as inputs + * @param input name of the node to copy + * @param n number of copies + */ +void ONNXImporter::expandMid(const std::string& prefix, opencv_onnx::NodeProto& node_proto, + const std::string& input, size_t n) +{ + std::vector input_names; + input_names.reserve(n); + for (size_t j = 0; j < n; j++) + { + LayerParams copyLP; + copyLP.name = format("%s/copy_%zu", prefix.c_str(), j); + copyLP.type = "Identity"; + CV_Assert((layer_id.find(copyLP.name) == layer_id.end()) && + "Couldn't copy the node: generated name already exists in the graph."); + input_names.push_back(copyLP.name); + + node_proto.set_input(0, input); + node_proto.set_output(0, copyLP.name); + addLayer(copyLP, node_proto); + } + node_proto.clear_input(); + for (size_t i = 0; i < input_names.size(); i++) + { + node_proto.add_input(input_names[i]); + } +} + +/** @brief Multiply one of node_proto inputs by -1 + * @param layerParams parameters of the node + * @param node_proto node which input will be replaced + * @param input_id id of input to be multiplied by -1 + */ +void ONNXImporter::addNegation(const LayerParams& layerParams, opencv_onnx::NodeProto& node_proto, int input_id) +{ + LayerParams powerParams; + powerParams.name = layerParams.name + "/neg"; + powerParams.type = "Power"; + powerParams.set("scale", -1.f); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(input_id)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(input_id)]; + + //Replace input to Power + node_proto.set_input(input_id, powerParams.name); +} + void ONNXImporter::addConstant(const std::string& name, const Mat& blob) { constBlobs.insert(std::make_pair(name, blob)); outShapes.insert(std::make_pair(name, shape(blob))); } +void ONNXImporter::handleQuantizedNode(LayerParams& layerParams, + const opencv_onnx::NodeProto& node_proto) +{ + // Quantized nodes have output names ending with 'quantized' + std::string outName = node_proto.output(0); + int len = outName.length(); + if (len <= 9) + return; + + if (outName.substr(len - 9) == "quantized") + { + outName = outName.substr(0, len - 9); + Mat scale, zeropoint; + + if (constBlobs.find(outName + "scale") != constBlobs.end() && + constBlobs.find(outName + "zero_point") != constBlobs.end()) + { + scale = getBlob(outName + "scale"); + zeropoint = getBlob(outName + "zero_point"); + } + else + { + std::string inpName = node_proto.input(0); + inpName = inpName.substr(0, inpName.length() - 9); + scale = getBlob(inpName + "scale"); + zeropoint = getBlob(inpName + "zero_point"); + + for (int i = 0; i < node_proto.output_size(); i++) + { + std::string out = node_proto.output(i); + out = out.substr(0, out.length() - 9); + addConstant(out + "scale", scale); + addConstant(out + "zero_point", zeropoint); + } + } + + if (scale.total() != 1 || zeropoint.total() != 1) + CV_Error(Error::StsNotImplemented, "Per-channel scales/zeropoints are not supported"); + + layerParams.set("depth", CV_8S); + layerParams.set("scales", DictValue::arrayReal(scale.ptr(), 1)); + layerParams.set("zeropoints", DictValue::arrayInt(zeropoint.ptr(), 1)); + } +} + void ONNXImporter::populateNet() { CV_Assert(model_proto.has_graph()); @@ -450,7 +649,8 @@ void ONNXImporter::populateNet() for (int j = 0; j < inpShape.size(); ++j) { inpShape[j] = tensorShape.dim(j).dim_value(); - if (!tensorShape.dim(j).dim_param().empty()) + // NHW, NCHW(NHWC), NCDHW(NDHWC); do not set this flag if only N is dynamic + if (!tensorShape.dim(j).dim_param().empty() && !(j == 0 && inpShape.size() >= 3)) hasDynamicShapes = true; } if (!inpShape.empty() && !hasDynamicShapes) @@ -471,35 +671,11 @@ void ONNXImporter::populateNet() layer_id.insert(std::make_pair(name, LayerInfo(0, netInputs.size() - 1))); } } - utilNet.setInputsNames(netInputs); dstNet.setInputsNames(netInputs); if (DNN_DIAGNOSTICS_RUN) { - auto &supportedTypes = getSupportedTypes(); - for (int li = 0; li < layersSize; li++) { - const opencv_onnx::NodeProto &node_proto = graph_proto.node(li); - std::string name = node_proto.output(0); - std::string layer_type = node_proto.op_type(); - auto registered = supportedTypes.find(layer_type); - if (registered == supportedTypes.end()) { - CV_LOG_ERROR(NULL, "DNN/ONNX: NOTE: Potential problem with creating node " << name<< " with type " << layer_type << ".\n Type " - << layer_type << " IS NOT SUPPORTED!\n" - ); - } - } - auto oldConstBlobs = constBlobs; - auto oldOutShapes = outShapes; - auto oldLayerId = layer_id; CV_LOG_INFO(NULL, "DNN/ONNX: start diagnostic run!"); - for (int li = 0; li < layersSize; li++) { - const opencv_onnx::NodeProto &node_proto = graph_proto.node(li); - handleNode(node_proto); - } - CV_LOG_INFO(NULL, "DNN/ONNX: diagnostic run completed!"); - constBlobs = oldConstBlobs; - outShapes = oldOutShapes; - layer_id = oldLayerId; - enableModelDiagnostics(false); + layerHandler->fillRegistry(graph_proto); } for(int li = 0; li < layersSize; li++) @@ -508,90 +684,14 @@ void ONNXImporter::populateNet() handleNode(node_proto); } - CV_LOG_DEBUG(NULL, "DNN/ONNX: import completed!"); + CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN ? "DNN/ONNX: diagnostic run completed!" : "DNN/ONNX: import completed!")); } -const std::set& ONNXImporter::getSupportedTypes() +void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto) { - static const std::set layerTypes = { - "MaxPool", - "AveragePool", - "GlobalAveragePool", - "GlobalMaxPool", - "ReduceMean", - "ReduceSum", - "ReduceMax", - "Slice", - "Split", - "Add", - "Sum", - "Sub", - "Pow", - "Max", - "Neg", - "Constant", - "LSTM", - "ImageScaler", - "Clip", - "LeakyRelu", - "Relu", - "Elu", - "Tanh", - "PRelu", - "LRN", - "InstanceNormalization", - "BatchNormalization", - "Gemm", - "MatMul", - "Mul", - "Div", - "Conv", - "ConvTranspose", - "Transpose", - "Squeeze", - "Flatten", - "Unsqueeze", - "Expand", - "Reshape", - "Pad", - "Shape", - "Cast", - "ConstantOfShape", - "ConstantFill", - "Gather", - "Concat", - "Resize", - "Upsample", - "SoftMax", - "Softmax", - "LogSoftmax", - "DetectionOutput", - "Interp", - "CropAndResize", - "ROIPooling", - "PSROIPooling", - "ChannelsPReLU", - "Sigmoid", - "Swish", - "Mish", - "AbsVal", - "BNLL", - "MaxUnpool", - "Dropout", - "Identity", - "Crop", - "Normalize" - }; - return layerTypes; -} - -void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) -{ - opencv_onnx::NodeProto node_proto = node_proto_; // TODO FIXIT - CV_Assert(node_proto.output_size() >= 1); std::string name = node_proto.output(0); - std::string layer_type = node_proto.op_type(); + const std::string& layer_type = node_proto.op_type(); CV_LOG_DEBUG(NULL, "DNN/ONNX: processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) ); @@ -605,1482 +705,17 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) layerParams.type = layer_type; layerParams.set("has_dynamic_shapes", hasDynamicShapes); - if (layer_type == "MaxPool") + handleQuantizedNode(layerParams, node_proto); + + DispatchMap::const_iterator iter = dispatch.find(layer_type); + if (iter != dispatch.end()) { - layerParams.type = "Pooling"; - layerParams.set("pool", "MAX"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); - } - else if (layer_type == "AveragePool") - { - layerParams.type = "Pooling"; - layerParams.set("pool", "AVE"); - layerParams.set("ceil_mode", layerParams.has("pad_mode")); - layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); - } - else if (layer_type == "GlobalAveragePool" || layer_type == "GlobalMaxPool" || - layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax") - { - CV_Assert(node_proto.input_size() == 1); - layerParams.type = "Pooling"; - String pool; - if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") - pool = "MAX"; - else if (layer_type == "ReduceSum") - pool = "SUM"; - else - pool = "AVE"; - layerParams.set("pool", pool); - layerParams.set("global_pooling", !layerParams.has("axes")); - if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) - { - MatShape inpShape = outShapes[node_proto.input(0)]; - DictValue axes = layerParams.get("axes"); - bool keepdims = layerParams.get("keepdims"); - MatShape targetShape; - std::vector shouldDelete(inpShape.size(), false); - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - shouldDelete[axis] = true; - } - for (int axis = 0; axis < inpShape.size(); ++axis){ - if (!shouldDelete[axis]) - targetShape.push_back(inpShape[axis]); - else if (keepdims) - targetShape.push_back(1); - } - - if (inpShape.size() == 3 && axes.size() <= 2) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - CV_CheckNE(axis, 0, ""); - - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("axis", 0); - reshapeLp.set("num_axes", 1); - int newShape[] = {1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - - LayerParams avgLp; - avgLp.name = layerParams.name + "/avg"; - avgLp.type = "Pooling"; - CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); - avgLp.set("pool", pool); - if (axes.size() == 2) - { - CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); - CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); - avgLp.set("global_pooling", true); - } - else - { - avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); - avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); - } - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, avgLp.name); - addLayer(avgLp, node_proto); - } - else - { - if (inpShape.size() != 4 && inpShape.size() != 5) - CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); - - CV_Assert(axes.size() <= inpShape.size() - 2); - std::vector kernel_size(inpShape.size() - 2, 1); - if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) - { - int axis = normalize_axis(axes.get(0), inpShape.size()); - MatShape newShape = inpShape; - newShape[axis + 1] = total(newShape, axis + 1); - newShape.resize(axis + 2); - newShape.insert(newShape.begin(), 2 - axis, 1); - - LayerParams reshapeLp; - reshapeLp.type = "Reshape"; - reshapeLp.name = layerParams.name + "/reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); - - node_proto.set_output(0, reshapeLp.name); - addLayer(reshapeLp, node_proto); - - kernel_size.resize(2); - kernel_size[0] = inpShape[axis]; - node_proto.set_input(0, node_proto.output(0)); - } - else - { - for (int i = 0; i < axes.size(); i++) { - int axis = normalize_axis(axes.get(i), inpShape.size()); - CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); - kernel_size[axis - 2] = inpShape[axis]; - } - } - - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/avg"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); - - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); - } - - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); - - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); - } - else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) - { - CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - int newShape[] = {1, 1, 1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - - LayerParams poolLp = layerParams; - poolLp.name = layerParams.name + "/pool"; - CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); - - node_proto.set_input(0, reshapeLp.name); - node_proto.set_output(0, poolLp.name); - addLayer(poolLp, node_proto); - - layerParams.type = "Reshape"; - int targetShape[] = {1}; - layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); - - node_proto.set_input(0, node_proto.output(0)); - node_proto.set_output(0, layerParams.name); - } - } - else if (layer_type == "Slice") - { - int axis = 0; - std::vector begin; - std::vector end; - std::vector steps; - int inp_size = node_proto.input_size(); - - if (inp_size == 1) - { - if (layerParams.has("axes")) { - DictValue axes = layerParams.get("axes"); - for (int i = 1; i < axes.size(); ++i) { - CV_Assert(axes.get(i - 1) == axes.get(i) - 1); - } - axis = axes.get(0); - } - - DictValue starts = layerParams.get("starts"); - DictValue ends = layerParams.get("ends"); - CV_Assert(starts.size() == ends.size()); - - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - for (int i = 0; i < starts.size(); ++i) - { - begin.push_back(starts.get(i)); - int finish = ends.get(i); - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } - } else { // inp_size > 1 - CV_Assert(inp_size >= 3); - for (int i = 1; i < inp_size; i++) { - CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); - } - Mat start_blob = getBlob(node_proto, 1); - Mat end_blob = getBlob(node_proto, 2); - CV_Assert(start_blob.total() == end_blob.total()); - - if (inp_size > 3) { - Mat axes_blob = getBlob(node_proto, 3); - const int* axes = (int*)axes_blob.data; - for (int i = 1; i < axes_blob.total(); ++i) { - CV_Assert(axes[i - 1] == axes[i] - 1); - } - axis = axes[0]; - } - - const int* starts = start_blob.ptr(); - const int* ends = end_blob.ptr(); - if (axis > 0) { - begin.resize(axis, 0); - end.resize(axis, -1); - } - std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); - for (int i = 0; i < end_blob.total(); ++i) - { - int finish = ends[i]; - end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim - } - - if (inp_size == 5) { - CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); - Mat step_blob = getBlob(node_proto, 4); - const int* steps_ptr = step_blob.ptr(); - - if (axis > 0) - steps.resize(axis, 1); - - std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); - - // Very strange application for Slice op with tensor reversing. - // We just workaround it for 2d constants. - if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && - axis == 0 && - start_blob.at(0) == -1 && step_blob.at(0) == -1 && - end_blob.at(0) == std::numeric_limits::min()) - { - Mat inp = getBlob(node_proto, 0); - if (inp.dims == 2) - { - Mat flipped; - flip(inp, flipped, 0); - addConstant(layerParams.name, flipped); - return; - } - } - } - } - layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); - layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); - layerParams.set("axis", axis); - - if (!steps.empty()) - layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); - - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inp = getBlob(node_proto, 0); - std::vector inputs, sliced; - inputs.push_back(inp); - runLayer(layerParams, inputs, sliced); - CV_Assert(sliced.size() == 1); - addConstant(layerParams.name, sliced[0]); - return; - } - } - else if (layer_type == "Split") - { - if (layerParams.has("split")) - { - DictValue splits = layerParams.get("split"); - const int numSplits = splits.size(); - CV_Assert(numSplits > 1); - - std::vector slicePoints(numSplits - 1, splits.get(0)); - for (int i = 1; i < splits.size() - 1; ++i) - { - slicePoints[i] = slicePoints[i - 1] + splits.get(i - 1); - } - layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); - } - else - { - layerParams.set("num_split", node_proto.output_size()); - } - layerParams.type = "Slice"; - } - else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub") - { - bool isSub = layer_type == "Sub"; - CV_CheckEQ(node_proto.input_size(), 2, ""); - bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); - bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); - if (is_const_0 && is_const_1) - { - Mat blob_0 = getBlob(node_proto, 0); - Mat blob_1 = getBlob(node_proto, 1); - CV_Assert(blob_0.size == blob_1.size); - Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); - addConstant(layerParams.name, output); - return; - } - else if (is_const_0 || is_const_1) - { - int const_blob_id = is_const_0 ? 0 : 1; - Mat blob = getBlob(node_proto, const_blob_id); - int blob_total = blob.total(); - if (blob_total == 1) { - layerParams.type = "Power"; - layerParams.set("shift", (isSub ? -1 : 1) * blob.at(0)); - } - else { - MatShape inpShape = outShapes[node_proto.input(1 - const_blob_id)]; - if (shape(blob) == inpShape) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - constParams.type = "Const"; - constParams.blobs.push_back((isSub ? -1 : 1) * blob); - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(constParams.name, constParams.type, constParams); - else - id = dstNet.addLayer(constParams.name, constParams.type, constParams); - layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); - outShapes[constParams.name] = shape(blob); - - layerParams.type = "Eltwise"; - node_proto.set_input(const_blob_id, constParams.name); - } - else - { - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - int axis = 1; - for (int i = 0; i < graph_proto.initializer_size(); i++) - { - opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); - if (tensor_proto.name() == node_proto.input(const_blob_id)) - { - axis = inpShape.size() - tensor_proto.dims_size(); - break; - } - } - layerParams.set("axis", axis); - blob = blob.reshape(1, 1); - layerParams.blobs.push_back((isSub ? -1 : 1) * blob); - } - } - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - if (isSub) - { - static float subCoeffs[] = {1.f, -1.f}; - layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); - } - } - else - { - if (isSub) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/neg"; - powerParams.type = "Power"; - powerParams.set("scale", -1); - - int id; - //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - layerParams.set("bias_term", true); - } - } - else if (layer_type == "Pow") - { - if (layer_id.find(node_proto.input(1)) != layer_id.end()) - CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); - - Mat blob = getBlob(node_proto, 1); - if (blob.total() != 1) - CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); - - blob.convertTo(blob, CV_32F); - layerParams.type = "Power"; - layerParams.set("power", blob.at(0)); - } - else if (layer_type == "Max") - { - layerParams.type = "Eltwise"; - layerParams.set("operation", "max"); - } - else if (layer_type == "Neg") - { - layerParams.type = "Power"; - layerParams.set("scale", -1); - } - else if (layer_type == "Constant") - { - CV_Assert(node_proto.input_size() == 0); - CV_Assert(layerParams.blobs.size() == 1); - addConstant(layerParams.name, layerParams.blobs[0]); - return; - } - else if (layer_type == "LSTM") - { - LayerParams lstmParams = layerParams; - lstmParams.name += "/lstm"; - - // https://pytorch.org/docs/stable/nn.html#lstm - CV_Assert(node_proto.input_size() == 7); - Mat Wx = getBlob(node_proto, 1); - Mat Wh = getBlob(node_proto, 2); - Mat b = getBlob(node_proto, 3); - CV_CheckEQ(countNonZero(getBlob(node_proto, 5)), 0, "Unsupported non zero initial_h"); - CV_CheckEQ(countNonZero(getBlob(node_proto, 6)), 0, "Unsupported non zero initial_c"); - b = b.reshape(1, b.size[0]); - - const int numHidden = lstmParams.get("hidden_size"); - const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. - const int numFeatures = Wx.size[2]; - Mat bx = b.colRange(0, b.cols / 2); - Mat bh = b.colRange(b.cols / 2, b.cols); - b = bx + bh; - - // IFGO->IGFO - for (int k = 0; k < numDirs; ++k) - { - float* WxData = Wx.ptr(k); - float* WhData = Wh.ptr(k); - float* biasData = b.ptr(k); - for (int j = 0; j < numHidden; ++j) - { - for (int i = 0; i < numFeatures; ++i) - { - std::swap(WxData[(numHidden + j) * numFeatures + i], - WxData[(numHidden * 2 + j) * numFeatures + i]); - } - for (int i = 0; i < numHidden; ++i) - { - std::swap(WhData[(numHidden + j) * numHidden + i], - WhData[(numHidden * 2 + j) * numHidden + i]); - } - std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); - } - } - Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); - Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); - - lstmParams.blobs.resize(3); - lstmParams.blobs[0] = Wh; - lstmParams.blobs[1] = Wx; - lstmParams.blobs[2] = b; - lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); - - node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name - addLayer(lstmParams, node_proto); - - MatShape lstmShape = outShapes[node_proto.output(0)]; - - // Add fake 1 as it is done in ONNX - lstmShape.insert(lstmShape.begin() + 1, 1); - - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); - node_proto.set_input(0, lstmParams.name); // redirect input to LSTM - node_proto.set_output(0, layerParams.name); // keep origin LSTM's name - } - else if (layer_type == "ImageScaler") - { - const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; - layerParams.erase("scale"); - - if (layerParams.has("bias")) - { - layerParams.type = "Scale"; - layerParams.blobs.push_back( - Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); - - layerParams.set("bias_term", true); - Mat bias(1, layerParams.get("bias").size(), CV_32FC1); - for (int j = 0; j < bias.total(); j++) { - bias.at(0, j) = layerParams.get("bias").getRealValue(j); - } - layerParams.blobs.push_back(bias); - layerParams.erase("bias"); - } - else { - layerParams.set("scale", scale); - layerParams.type = "Power"; - } - } - else if (layer_type == "Clip") - { - layerParams.type = "ReLU6"; - replaceLayerParam(layerParams, "min", "min_value"); - replaceLayerParam(layerParams, "max", "max_value"); - - } - else if (layer_type == "LeakyRelu") - { - layerParams.type = "ReLU"; - replaceLayerParam(layerParams, "alpha", "negative_slope"); - } - else if (layer_type == "Relu") - { - layerParams.type = "ReLU"; - } - else if (layer_type == "Elu") - { - layerParams.type = "ELU"; - } - else if (layer_type == "Tanh") - { - layerParams.type = "TanH"; - } - else if (layer_type == "PRelu") - { - layerParams.type = "PReLU"; - layerParams.blobs.push_back(getBlob(node_proto, 1)); - } - else if (layer_type == "LRN") - { - replaceLayerParam(layerParams, "size", "local_size"); - } - else if (layer_type == "InstanceNormalization") - { - if (node_proto.input_size() != 3) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias"); - - layerParams.blobs.resize(4); - layerParams.blobs[2] = getBlob(node_proto, 1); // weightData - layerParams.blobs[3] = getBlob(node_proto, 2); // biasData - layerParams.set("has_bias", true); - layerParams.set("has_weight", true); - - // Get number of channels in input - int size = layerParams.blobs[2].total(); - layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean - layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std - - LayerParams mvnParams; - mvnParams.name = layerParams.name + "/MVN"; - mvnParams.type = "MVN"; - mvnParams.set("eps", layerParams.get("epsilon")); - layerParams.erase("epsilon"); - - //Create MVN layer - int id; - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); - else - id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(0)); - CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); - outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; - - //Replace Batch Norm's input to MVN - node_proto.set_input(0, mvnParams.name); - layerParams.type = "BatchNorm"; - } - else if (layer_type == "BatchNormalization") - { - if (node_proto.input_size() != 5) - CV_Error(Error::StsNotImplemented, - "Expected input, scale, bias, mean and var"); - - layerParams.type = "BatchNorm"; - replaceLayerParam(layerParams, "epsilon", "eps"); - replaceLayerParam(layerParams, "spatial", "use_global_stats"); - - Mat meanData = getBlob(node_proto, 3); - Mat stdData = getBlob(node_proto, 4); - - layerParams.blobs.push_back(meanData); - layerParams.blobs.push_back(stdData); - - if (!node_proto.input(1).empty()) { - layerParams.set("has_weight", true); - layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData - } else { - layerParams.set("has_weight", false); - } - - if (!node_proto.input(2).empty()) { - layerParams.set("has_bias", true); - layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData - } else { - layerParams.set("has_bias", false); - } - } - else if (layer_type == "Gemm") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "InnerProduct"; - Mat weights = getBlob(node_proto, 1); - int ind_num_out = 0; - if (layerParams.has("transB") && !layerParams.get("transB")) { - transpose(weights, weights); - ind_num_out = 1; - } - layerParams.blobs.push_back(weights); - - if (node_proto.input_size() == 3) { - Mat bias = getBlob(node_proto, 2); - layerParams.blobs.push_back(bias); - } - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inputBuf = getBlob(node_proto, 0); - - LayerParams constParams; - constParams.name = node_proto.input(0); - constParams.type = "Const"; - constParams.blobs.push_back(inputBuf); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - } - - layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); - layerParams.set("bias_term", node_proto.input_size() == 3); - } - else if (layer_type == "MatMul") - { - CV_Assert(node_proto.input_size() == 2); - layerParams.type = "InnerProduct"; - layerParams.set("bias_term", false); - CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); - int firstInpDims = outShapes[node_proto.input(0)].size(); - int secondInpDims; - - if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 1); - secondInpDims = blob.dims; - layerParams.blobs.push_back(blob.t()); - layerParams.set("num_output", layerParams.blobs[0].size[0]); - } else { - secondInpDims = outShapes[node_proto.input(1)].size(); - } - layerParams.set("axis", firstInpDims - secondInpDims + 1); - } - else if (layer_type == "Mul" || layer_type == "Div") - { - CV_Assert(node_proto.input_size() == 2); - - bool isDiv = layer_type == "Div"; - int constId = -1; - bool haveVariables = false; - for (int i = 0; i < 2; ++i) - { - if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) - constId = i; - else - haveVariables = true; - } - if (constId != -1 && haveVariables) - { - Mat blob = getBlob(node_proto, constId); - blob = blob.reshape(1, 1); - if (blob.total() == 1) { - float coeff = isDiv ? 1.0 / blob.at(0) : blob.at(0); - layerParams.set("scale", coeff); - layerParams.type = "Power"; - } - else { - if (isDiv) - divide(1.0, blob, blob); - layerParams.blobs.push_back(blob); - layerParams.type = "Scale"; - } - } - else if (!haveVariables) - { - Mat inp0 = getBlob(node_proto, 0); - Mat inp1 = getBlob(node_proto, 1); - - if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) - CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); - - if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) - { - if (inp0.dims < inp1.dims) - { - inp0 = inp0.reshape(1, inp1.dims, inp1.size); - inp0.dims = inp1.dims; - } - else - { - inp1 = inp1.reshape(1, inp0.dims, inp0.size); - inp1.dims = inp0.dims; - } - } - - Mat out; - if (inp0.total() != inp1.total()) - { - if (inp0.total() == 1) - { - float coeff = isDiv ? 1.0 / inp0.at(0) : inp0.at(0); - multiply(inp1, coeff, out); - } - else - { - float coeff = isDiv ? 1.0 / inp1.at(0) : inp1.at(0); - multiply(inp0, coeff, out); - } - - } - else - { - out = isDiv ? inp0 / inp1 : inp0.mul(inp1); - } - - if (inp0.dims == 1 && inp1.dims == 1) - out.dims = 1; // to workaround dims == 1 - addConstant(layerParams.name, out); - return; - } - else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) - { - layerParams.type = "Eltwise"; - layerParams.set("operation", isDiv ? "div" : "prod"); - } - else - { - // Scale layer allocate output with the first input shape - if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(1)); - proto.add_input(node_proto.input(0)); - proto.add_output(layerParams.name); - node_proto = proto; - } - - if (isDiv) - { - LayerParams powerParams; - powerParams.name = layerParams.name + "/inv"; - powerParams.type = "Power"; - powerParams.set("power", -1); - - int id; - //Create Power layer - if (DNN_DIAGNOSTICS_RUN) - id = utilNet.addLayer(powerParams.name, powerParams.type, powerParams); - else - id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); - //Connect to input - IterLayerId_t layerId = layer_id.find(node_proto.input(1)); - CV_Assert(layerId != layer_id.end()); - if (DNN_DIAGNOSTICS_RUN) - utilNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - else - dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); - //Add shape - layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); - outShapes[powerParams.name] = outShapes[node_proto.input(1)]; - - //Replace input to Power - node_proto.set_input(1, powerParams.name); - } - layerParams.type = "Scale"; - } - } - else if (layer_type == "Conv") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Convolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) - { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - } - int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; - layerParams.set("num_output", outCn); - } - else if (layer_type == "ConvTranspose") - { - CV_Assert(node_proto.input_size() >= 2); - layerParams.type = "Deconvolution"; - for (int j = 1; j < node_proto.input_size(); j++) { - layerParams.blobs.push_back(getBlob(node_proto, j)); - } - layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); - layerParams.set("bias_term", node_proto.input_size() == 3); - - if (!layerParams.has("kernel_size")) - CV_Error(Error::StsNotImplemented, - "Required attribute 'kernel_size' is not present."); - - if (layerParams.has("output_shape")) - { - const DictValue& outShape = layerParams.get("output_shape"); - DictValue strides = layerParams.get("stride"); - DictValue kernel = layerParams.get("kernel_size"); - - String padMode; - std::vector adjust_pads; - if (layerParams.has("pad_mode")) - { - padMode = toUpperCase(layerParams.get("pad_mode")); - if (padMode != "SAME" && padMode != "VALID") - CV_Error(Error::StsError, "Unsupported padding mode " + padMode); - - for (int i = 0; i < strides.size(); i++) - { - int sz = outShape.get(2 + i); - int stride = strides.get(i); - adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : - (sz - kernel.get(i)) % stride); - } - layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); - } - } - else if (layerParams.has("output_padding")) - { - replaceLayerParam(layerParams, "output_padding", "adj"); - } - } - else if (layer_type == "Transpose") - { - layerParams.type = "Permute"; - replaceLayerParam(layerParams, "perm", "order"); - - CV_Assert(node_proto.input_size() == 1); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - std::vector inputs(1, getBlob(node_proto, 0)), transposed; - runLayer(layerParams, inputs, transposed); - CV_Assert(transposed.size() == 1); - addConstant(layerParams.name, transposed[0]); - return; - } - } - else if (layer_type == "Squeeze") - { - CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); - DictValue axes_dict = layerParams.get("axes"); - MatShape inpShape = outShapes[node_proto.input(0)]; - - std::vector maskedAxes(inpShape.size(), false); - for (int i = 0; i < axes_dict.size(); ++i) - { - int axis = axes_dict.getIntValue(i); - CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); - maskedAxes[axis] = inpShape[axis] == 1; - } - MatShape outShape; - for (int i = 0; i < inpShape.size(); ++i) - { - if (!maskedAxes[i]) - outShape.push_back(inpShape[i]); - } - if (outShape.size() != inpShape.size()) - { - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - { - if (!maskedAxes[index]) - inputIndices.push_back(index); - } - for (int index = 0; index < outShape.size(); ++index) - dynamicAxes.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - } - else - layerParams.type = "Identity"; - - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat inp = getBlob(node_proto, 0); - Mat out = inp.reshape(1, outShape); - out.dims = outShape.size(); // to workaround dims == 1 - addConstant(layerParams.name, out); - return; - } - } - else if (layer_type == "Flatten") - { - CV_CheckEQ(node_proto.input_size(), 1, ""); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat input = getBlob(node_proto, 0); - int axis = normalize_axis(layerParams.get("axis", 1), input.dims); - - std::vector out_size(&input.size[0], &input.size[0] + axis); - out_size.push_back(input.total(axis)); - Mat output = input.reshape(1, out_size); - addConstant(layerParams.name, output); - return; - } - } - else if (layer_type == "Unsqueeze") - { - CV_Assert(node_proto.input_size() == 1); - DictValue axes = layerParams.get("axes"); - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - // Constant input. - Mat input = getBlob(node_proto, 0); - - std::vector dims; - for (int j = 0; j < input.dims; j++) { - dims.push_back(input.size[j]); - } - CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); - for (int j = 0; j < axes.size(); j++) { - dims.insert(dims.begin() + axes.getIntValue(j), 1); - } - - Mat out = input.reshape(0, dims); - addConstant(layerParams.name, out); - return; - } - - // Variable input. - if (axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); - - MatShape inpShape = outShapes[node_proto.input(0)]; - int axis = axes.getIntValue(0); - CV_Assert(0 <= axis && axis <= inpShape.size()); - std::vector outShape = inpShape; - outShape.insert(outShape.begin() + axis, 1); - layerParams.type = "Reshape"; - layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < outShape.size(); ++index) { - if (index != axis) - dynamicAxes.push_back(index); - } - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - } - else if (layer_type == "Expand") - { - CV_CheckEQ(node_proto.input_size(), 2, ""); - const std::string& input0 = node_proto.input(0); - const std::string& input1 = node_proto.input(1); - Mat newShapeMat = getBlob(input1); - MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); - - MatShape inpShape; - bool haveVariables = constBlobs.find(input0) == constBlobs.end(); - if (haveVariables) - { - IterShape_t shapeIt = outShapes.find(input0); - CV_Assert(shapeIt != outShapes.end()); - inpShape = shapeIt->second; - } - else - { - inpShape = shape(getBlob(input0)); - } - - String srcName = input0; - // Unsqueeze and repeat along new axis - if (targetShape.size() == inpShape.size() + 1) - { - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] == -1 && i < inpShape.size()) - targetShape[i] = inpShape[i]; - else if (i < inpShape.size() && targetShape[i] != inpShape[i]) - inpShape.insert(inpShape.begin() + i, 1); - } - if (haveVariables) - { - LayerParams reshapeLp; - reshapeLp.name = layerParams.name + "/reshape"; - reshapeLp.type = "Reshape"; - CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); - reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(reshapeLp.name); - addLayer(reshapeLp, proto); - srcName = reshapeLp.name; - } - } - CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); - - std::vector broadcast_axes; - for (int i = 0; i < targetShape.size(); i++) - { - if (targetShape[i] != inpShape[i]) - { - if (inpShape[i] == 1) - broadcast_axes.push_back(i); - else - CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); - } - } - - if (!haveVariables) - { - if (broadcast_axes.size() != 1) - CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); - - Mat input = getBlob(node_proto, 0); - input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); - Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); - output = output.reshape(0, targetShape); - addConstant(layerParams.name, output); - return; - } - - if (broadcast_axes.size() == 2 && - broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) - { - LayerParams constParams; - constParams.name = layerParams.name + "/const"; - CV_Assert(layer_id.find(constParams.name) == layer_id.end()); - constParams.type = "Const"; - - Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); - constParams.blobs.push_back(inp); - - opencv_onnx::NodeProto proto; - proto.add_output(constParams.name); - addLayer(constParams, proto); - - layerParams.type = "Scale"; - layerParams.set("bias_term", false); - node_proto.set_input(0, constParams.name); - node_proto.set_input(1, srcName); - } - else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) - { - String base_name = layerParams.name + "/copy_"; - std::vector input_names; - for (int j = 0; j < targetShape[broadcast_axes[0]]; j++) - { - std::ostringstream ss; - ss << j; - LayerParams copyLP; - copyLP.name = base_name + ss.str(); - copyLP.type = "Identity"; - CV_Assert(layer_id.find(copyLP.name) == layer_id.end()); - input_names.push_back(copyLP.name); - - node_proto.set_input(0, srcName); - node_proto.set_output(0, copyLP.name); - addLayer(copyLP, node_proto); - } - node_proto.clear_input(); - for (int i = 0; i < input_names.size(); i++) - { - node_proto.add_input(input_names[i]); - } - layerParams.set("axis", broadcast_axes[0]); - layerParams.type = "Concat"; - node_proto.set_output(0, layerParams.name); - } - else - CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); - } - else if (layer_type == "Reshape") - { - CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); - - if (node_proto.input_size() == 2) { - Mat blob = getBlob(node_proto, 1); - CV_Assert(blob.type() == CV_32SC1); - - layerParams.set("dim", DictValue::arrayInt( - blob.ptr(), blob.total() )); - - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - std::vector inputs(1, getBlob(node_proto, 0)), outputs; - runLayer(layerParams, inputs, outputs); - addConstant(layerParams.name, outputs[0]); - return; - } - } - else { - DictValue shape = layerParams.get("shape"); - std::vector dim; - for (int j = 0; j < shape.size(); j++) { - dim.push_back(shape.getIntValue(j)); - } - - if (layer_id.find(node_proto.input(0)) == layer_id.end()) { - Mat input = getBlob(node_proto, 0); - Mat out = input.reshape(0, dim); - addConstant(layerParams.name, out); - return; - } - replaceLayerParam(layerParams, "shape", "dim"); - } - } - else if (layer_type == "Pad") - { - layerParams.type = "Padding"; - replaceLayerParam(layerParams, "mode", "type"); - if (node_proto.input_size() == 3 || node_proto.input_size() == 2) - { - // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. - // We need to shuffle it to begin0, end0, begin1, end1, ... - Mat paddings = getBlob(node_proto, 1).reshape(1, 2); - paddings = paddings.t(); - layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); - - if (node_proto.input_size() == 3) - { - Mat value = getBlob(node_proto, 2); - layerParams.set("value", value.at(0)); - } - } - } - else if (layer_type == "Shape") - { - CV_Assert(node_proto.input_size() == 1); - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - const MatShape& inpShape = shapeIt->second; - - Mat shapeMat(inpShape.size(), 1, CV_32S); - for (int j = 0; j < inpShape.size(); ++j) - shapeMat.at(j) = inpShape[j]; - shapeMat.dims = 1; - - addConstant(layerParams.name, shapeMat); - return; - } - else if (layer_type == "Cast") - { - if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) - { - Mat blob = getBlob(node_proto, 0); - int type; - switch (layerParams.get("to")) - { - case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; - case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; - case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; - case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; - case opencv_onnx::TensorProto_DataType_INT8: - case opencv_onnx::TensorProto_DataType_INT16: - case opencv_onnx::TensorProto_DataType_INT32: - case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; - default: type = blob.type(); - } - Mat dst; - blob.convertTo(dst, type); - dst.dims = blob.dims; - addConstant(layerParams.name, dst); - return; - } - else - layerParams.type = "Identity"; - } - else if (layer_type == "ConstantOfShape" || layer_type == "ConstantFill") - { - int depth = CV_32F; - float fill_value; - if (!layerParams.blobs.empty()) - { - CV_Assert(!layerParams.has("value")); - depth = layerParams.blobs[0].depth(); - Mat floats; - layerParams.blobs[0].convertTo(floats, CV_32F); - fill_value = floats.at(0, 0); - } - else - fill_value = layerParams.get("value", 0); - - MatShape inpShape = getBlob(node_proto, 0); - for (int i = 0; i < inpShape.size(); i++) - CV_CheckGT(inpShape[i], 0, ""); - Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); - addConstant(layerParams.name, tensor); - return; - } - else if (layer_type == "Gather") - { - CV_Assert(node_proto.input_size() == 2); - Mat indexMat = getBlob(node_proto, 1); - CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); - int index = indexMat.at(0); - int axis = layerParams.get("axis", 0); - - if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) - { - Mat input = getBlob(node_proto, 0); - Mat out; - std::vector ranges(input.dims, Range::all()); - ranges[axis] = Range(index, index + 1); - - out = input(ranges); - MatShape outShape = shape(out); - if (outShape.size() > 1) - { - outShape.erase(outShape.begin() + axis); - out.reshape(0, outShape); - } else { - out.dims = 1; - } - addConstant(layerParams.name, out); - return; - } - else - { - IterShape_t shapeIt = outShapes.find(node_proto.input(0)); - CV_Assert(shapeIt != outShapes.end()); - MatShape inpShape = shapeIt->second; - - LayerParams sliceLp; - sliceLp.type = "Slice"; - sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; - std::vector begin(inpShape.size(), 0); - std::vector end(inpShape.size(), -1); - begin[axis] = index; - end[axis] = index + 1; - - cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); - cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); - sliceLp.set("begin", paramBegin); - sliceLp.set("end", paramEnd); - sliceLp.set("has_dynamic_shapes", hasDynamicShapes); - - if (inpShape.size() > 1) - { - opencv_onnx::NodeProto proto; - proto.add_input(node_proto.input(0)); - proto.add_output(sliceLp.name); - addLayer(sliceLp, proto); - - inpShape.erase(inpShape.begin() + axis); - layerParams.type = "Reshape"; - layerParams.set("axis", 0); - layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); - if (hasDynamicShapes) - { - std::vector dynamicAxes; - std::vector inputIndices; - for (int index = 0; index < inpShape.size(); ++index) - dynamicAxes.push_back(index); - for (int index = 0; index < inpShape.size(); ++index) - inputIndices.push_back(index); - layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); - layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); - } - node_proto.set_input(0, sliceLp.name); - } - else - { - layerParams = sliceLp; - } - } - } - else if (layer_type == "Concat") - { - bool hasVariableInps = false; - for (int i = 0; i < node_proto.input_size(); ++i) - { - if (layer_id.find(node_proto.input(i)) != layer_id.end()) - { - hasVariableInps = true; - break; - } - } - - if (!hasVariableInps) - { - std::vector inputs(node_proto.input_size()), concatenated; - // Due constant folding we can get inputs with different number of dimensions - // Insert the missing dimension to inputs - MatShape inputShape; - for (size_t i = 0; i < inputs.size(); ++i) - { - inputs[i] = getBlob(node_proto, i); - if (inputs[i].size.dims() > inputShape.size()) - { - inputShape = shape(inputs[i]); - } - } - - // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 - int axis = layerParams.get("axis", 1); - for (size_t i = 0; i < inputs.size(); ++i) - { - MatShape targetShape = inputShape; - targetShape[axis] = shape(inputs[i])[axis]; - CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); - inputs[i] = inputs[i].reshape(0, targetShape); - } - runLayer(layerParams, inputs, concatenated); - - CV_Assert(concatenated.size() == 1); - addConstant(layerParams.name, concatenated[0]); - return; - } - } - else if (layer_type == "Resize") - { - for (int i = 1; i < node_proto.input_size(); i++) - CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); - - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); - - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); - - // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] - int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 - : node_proto.input_size() > 2 ? 2 : 1; - - Mat scales = getBlob(node_proto, foundScaleId); - if (scales.total() == 4) - { - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - else - { - const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); - if (constBlobs.find(inputLast) != constBlobs.end()) - { - Mat shapes = getBlob(inputLast); - CV_CheckEQ(shapes.size[0], 4, ""); - CV_CheckEQ(shapes.size[1], 1, ""); - CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); - if (shapes.depth() == CV_32F) - shapes.convertTo(shapes, CV_32S); - layerParams.set("width", shapes.at(3)); - layerParams.set("height", shapes.at(2)); - } - } - replaceLayerParam(layerParams, "mode", "interpolation"); - } - else if (layer_type == "Upsample") - { - //fused from Resize Subgraph - if (layerParams.has("coordinate_transformation_mode")) - { - String interp_mode = layerParams.get("coordinate_transformation_mode"); - CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); - - layerParams.set("align_corners", interp_mode == "align_corners"); - if (layerParams.get("mode") == "linear") - { - layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? - "opencv_linear" : "bilinear"); - } - } - if (layerParams.get("mode") == "linear" && framework_name == "pytorch") - layerParams.set("mode", "opencv_linear"); - - layerParams.type = "Resize"; - if (layerParams.has("scales")) - { - // Pytorch layer - DictValue scales = layerParams.get("scales"); - CV_Assert(scales.size() == 4); - layerParams.set("zoom_factor_y", scales.getIntValue(2)); - layerParams.set("zoom_factor_x", scales.getIntValue(3)); - } - else if (layerParams.has("height_scale") && layerParams.has("width_scale")) - { - // Caffe2 layer - replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); - replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); - } - else - { - // scales as input - const std::string& input1 = node_proto.input(1); - if (constBlobs.find(input1) != constBlobs.end()) - { - Mat scales = getBlob(input1); - CV_Assert(scales.total() == 4); - layerParams.set("zoom_factor_y", scales.at(2)); - layerParams.set("zoom_factor_x", scales.at(3)); - } - } - replaceLayerParam(layerParams, "mode", "interpolation"); - } - else if (layer_type == "SoftMax" || layer_type == "LogSoftmax") - { - layerParams.type = "Softmax"; - layerParams.set("log_softmax", layer_type == "LogSoftmax"); - } - else if (layer_type == "DetectionOutput") - { - CV_CheckEQ(node_proto.input_size(), 3, ""); - if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) - { - Mat priors = getBlob(node_proto, 2); - - LayerParams constParams; - constParams.name = layerParams.name + "/priors"; - constParams.type = "Const"; - constParams.blobs.push_back(priors); - - opencv_onnx::NodeProto priorsProto; - priorsProto.add_output(constParams.name); - addLayer(constParams, priorsProto); - - node_proto.set_input(2, constParams.name); - } + CALL_MEMBER_FN(*this, iter->second)(layerParams, node_proto); } else { - for (int j = 0; j < node_proto.input_size(); j++) { - if (layer_id.find(node_proto.input(j)) == layer_id.end()) - layerParams.blobs.push_back(getBlob(node_proto, j)); - } + parseCustomLayer(layerParams, node_proto); } - addLayer(layerParams, node_proto); } catch (const cv::Exception& e) { @@ -2089,6 +724,7 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) CV_LOG_ERROR(NULL, "DNN/ONNX: Potential problem during processing node with " << node_proto.input_size() << " inputs and " << node_proto.output_size() << " outputs: " << cv::format("[%s]:(%s)", layer_type.c_str(), name.c_str()) << "\n" << e.msg ); + cv::AutoLock lock(getLayerFactoryMutex()); auto registeredLayers = getLayerFactoryImpl(); if (registeredLayers.find(layerParams.type) != registeredLayers.end()) { @@ -2130,18 +766,2225 @@ void ONNXImporter::handleNode(const opencv_onnx::NodeProto& node_proto_) } } +void ONNXImporter::parseMaxPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PoolingInt8" : "Pooling"; + layerParams.set("pool", "MAX"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseAveragePool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Pooling"; + layerParams.set("pool", "AVE"); + layerParams.set("ceil_mode", layerParams.has("pad_mode")); + layerParams.set("ave_pool_padded_area", framework_name == "pytorch"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseReduce(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + + CV_Assert(node_proto.input_size() == 1); + layerParams.type = "Pooling"; + String pool; + if (layer_type == "GlobalMaxPool" || layer_type == "ReduceMax") + pool = "MAX"; + else if (layer_type == "ReduceSum") + pool = "SUM"; + else + pool = "AVE"; + layerParams.set("pool", pool); + layerParams.set("global_pooling", !layerParams.has("axes")); + if (layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + MatShape inpShape = outShapes[node_proto.input(0)]; + DictValue axes = layerParams.get("axes"); + bool keepdims = layerParams.get("keepdims"); + MatShape targetShape; + std::vector shouldDelete(inpShape.size(), false); + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + shouldDelete[axis] = true; + } + for (int axis = 0; axis < inpShape.size(); ++axis){ + if (!shouldDelete[axis]) + targetShape.push_back(inpShape[axis]); + else if (keepdims) + targetShape.push_back(1); + } + + if (inpShape.size() == 3 && axes.size() <= 2) + { + int axis = normalize_axis(axes.get(0), inpShape.size()); + CV_CheckNE(axis, 0, ""); + + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("axis", 0); + reshapeLp.set("num_axes", 1); + int newShape[] = {1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 2)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams avgLp; + avgLp.name = layerParams.name + "/avg"; + avgLp.type = "Pooling"; + CV_Assert(layer_id.find(avgLp.name) == layer_id.end()); + avgLp.set("pool", pool); + if (axes.size() == 2) + { + CV_CheckEQ(normalize_axis(axes.get(0), inpShape.size()), 1, "Unsupported mode"); + CV_CheckEQ(normalize_axis(axes.get(1), inpShape.size()), 2, "Unsupported mode"); + avgLp.set("global_pooling", true); + } + else + { + avgLp.set(axis == 2 ? "global_pooling_w" : "global_pooling_h", true); + avgLp.set(axis == 2 ? "kernel_h" : "kernel_w", 1); + } + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, avgLp.name); + addLayer(avgLp, node_proto); + } + else + { + if (inpShape.size() != 4 && inpShape.size() != 5) + CV_Error(Error::StsNotImplemented, "Unsupported input shape of " + layer_type + " operation."); + + CV_Assert(axes.size() <= inpShape.size() - 2); + std::vector kernel_size(inpShape.size() - 2, 1); + if (axes.size() == 1 && (normalize_axis(axes.get(0), inpShape.size()) <= 1)) + { + int axis = normalize_axis(axes.get(0), inpShape.size()); + MatShape newShape = inpShape; + newShape[axis + 1] = total(newShape, axis + 1); + newShape.resize(axis + 2); + newShape.insert(newShape.begin(), 2 - axis, 1); + + LayerParams reshapeLp; + reshapeLp.type = "Reshape"; + reshapeLp.name = layerParams.name + "/reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], newShape.size())); + + node_proto.set_output(0, reshapeLp.name); + addLayer(reshapeLp, node_proto); + + kernel_size.resize(2); + kernel_size[0] = inpShape[axis]; + node_proto.set_input(0, node_proto.output(0)); + } + else + { + for (int i = 0; i < axes.size(); i++) { + int axis = normalize_axis(axes.get(i), inpShape.size()); + CV_Assert_N(axis >= 2 + i, axis < inpShape.size()); + kernel_size[axis - 2] = inpShape[axis]; + } + } + + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/avg"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + poolLp.set("kernel_size", DictValue::arrayInt(&kernel_size[0], kernel_size.size())); + + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + } + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], targetShape.size())); + + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + else if (!layerParams.has("axes") && (layer_type == "ReduceMean" || layer_type == "ReduceSum" || layer_type == "ReduceMax")) + { + CV_CheckEQ(layerParams.get("keepdims"), 0, "layer only supports keepdims = false"); + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + int newShape[] = {1, 1, 1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 4)); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + + LayerParams poolLp = layerParams; + poolLp.name = layerParams.name + "/pool"; + CV_Assert(layer_id.find(poolLp.name) == layer_id.end()); + + node_proto.set_input(0, reshapeLp.name); + node_proto.set_output(0, poolLp.name); + addLayer(poolLp, node_proto); + + layerParams.type = "Reshape"; + int targetShape[] = {1}; + layerParams.set("dim", DictValue::arrayInt(&targetShape[0], 1)); + + node_proto.set_input(0, node_proto.output(0)); + node_proto.set_output(0, layerParams.name); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSlice(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int axis = 0; + std::vector begin; + std::vector end; + std::vector steps; + int inp_size = node_proto.input_size(); + + if (inp_size == 1) + { + if (layerParams.has("axes")) { + DictValue axes = layerParams.get("axes"); + for (int i = 1; i < axes.size(); ++i) { + CV_Assert(axes.get(i - 1) == axes.get(i) - 1); + } + axis = axes.get(0); + } + + DictValue starts = layerParams.get("starts"); + DictValue ends = layerParams.get("ends"); + CV_Assert(starts.size() == ends.size()); + + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); + } + for (int i = 0; i < starts.size(); ++i) + { + begin.push_back(starts.get(i)); + int finish = ends.get(i); + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim + } + } else { // inp_size > 1 + CV_Assert(inp_size >= 3); + for (int i = 1; i < inp_size; i++) { + CV_Assert(constBlobs.find(node_proto.input(i)) != constBlobs.end()); + } + Mat start_blob = getBlob(node_proto, 1); + Mat end_blob = getBlob(node_proto, 2); + CV_Assert(start_blob.total() == end_blob.total()); + + if (inp_size > 3) { + Mat axes_blob = getBlob(node_proto, 3); + const int* axes = (int*)axes_blob.data; + for (int i = 1; i < axes_blob.total(); ++i) { + CV_Assert(axes[i - 1] == axes[i] - 1); + } + axis = axes[0]; + } + + const int* starts = start_blob.ptr(); + const int* ends = end_blob.ptr(); + if (axis > 0) { + begin.resize(axis, 0); + end.resize(axis, -1); + } + std::copy(starts, starts + start_blob.total(), std::back_inserter(begin)); + for (int i = 0; i < end_blob.total(); ++i) + { + int finish = ends[i]; + end.push_back((finish < 0) ? --finish : finish); // numpy doesn't include last dim + } + + if (inp_size == 5) { + CV_Assert(constBlobs.find(node_proto.input(4)) != constBlobs.end()); + Mat step_blob = getBlob(node_proto, 4); + const int* steps_ptr = step_blob.ptr(); + + if (axis > 0) + steps.resize(axis, 1); + + std::copy(steps_ptr, steps_ptr + step_blob.total(), std::back_inserter(steps)); + + // Very strange application for Slice op with tensor reversing. + // We just workaround it for 2d constants. + if (constBlobs.find(node_proto.input(0)) != constBlobs.end() && + axis == 0 && + start_blob.at(0) == -1 && step_blob.at(0) == -1 && + end_blob.at(0) == std::numeric_limits::min()) + { + Mat inp = getBlob(node_proto, 0); + if (inp.dims == 2) + { + Mat flipped; + flip(inp, flipped, 0); + addConstant(layerParams.name, flipped); + return; + } + } + } + } + layerParams.set("begin", DictValue::arrayInt(&begin[0], begin.size())); + layerParams.set("end", DictValue::arrayInt(&end[0], end.size())); + layerParams.set("axis", axis); + + if (!steps.empty()) + layerParams.set("steps", DictValue::arrayInt(&steps[0], steps.size())); + + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + std::vector inputs, sliced; + inputs.push_back(inp); + runLayer(layerParams, inputs, sliced); + CV_Assert(sliced.size() == 1); + addConstant(layerParams.name, sliced[0]); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSplit(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layerParams.has("split")) + { + DictValue splits = layerParams.get("split"); + const int numSplits = splits.size(); + CV_Assert(numSplits > 1); + + std::vector slicePoints(numSplits - 1, splits.get(0)); + for (int i = 1; i < splits.size() - 1; ++i) + { + slicePoints[i] = slicePoints[i - 1] + splits.get(i); + } + layerParams.set("slice_point", DictValue::arrayInt(&slicePoints[0], slicePoints.size())); + } + else + { + layerParams.set("num_split", node_proto.output_size()); + } + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "SliceInt8" : "Slice"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseBias(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + bool isSub = layer_type == "Sub"; + CV_Assert((node_proto.input_size() == 2) || (layer_type == "Sum" && node_proto.input_size() > 2)); + + if (layer_type == "Sum" && node_proto.input_size() > 2) + { + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (layer_id.find(node_proto.input(i)) == layer_id.end()) + { + CV_Error(Error::StsNotImplemented, "Sum of constants is not implemented for inputs > 2"); + } + } + } + + bool is_const_0 = layer_id.find(node_proto.input(0)) == layer_id.end(); + bool is_const_1 = layer_id.find(node_proto.input(1)) == layer_id.end(); + if (is_const_0 && is_const_1) + { + Mat blob_0 = getBlob(node_proto, 0); + Mat blob_1 = getBlob(node_proto, 1); + CV_Assert(blob_0.size == blob_1.size); + Mat output = isSub ? (blob_0 - blob_1) : (blob_0 + blob_1); + addConstant(layerParams.name, output); + return; + } + else if (is_const_0 || is_const_1) + { + int const_blob_id = is_const_0 ? 0 : 1; + int input_id = 1 - const_blob_id; + Mat blob = getBlob(node_proto, const_blob_id); + int blob_total = blob.total(); + + const float inputScale = isSub && is_const_0 ? -1.f : 1.f; + const float constScale = isSub && is_const_1 ? -1.f : 1.f; + + if (blob_total == 1) { + layerParams.type = "Power"; + layerParams.set("scale", inputScale); + layerParams.set("shift", constScale * blob.ptr()[0]); + } + else { + MatShape inpShape = outShapes[node_proto.input(input_id)]; + if (shape(blob) == inpShape) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + constParams.type = "Const"; + constParams.blobs.push_back(blob); + int id = dstNet.addLayer(constParams.name, constParams.type, constParams); + layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); + outShapes[constParams.name] = shape(blob); + + layerParams.type = "Eltwise"; + float coeffs[] = {1., isSub ? -1.f : 1.f}; + layerParams.set("coeff", DictValue::arrayReal(coeffs, 2)); + node_proto.set_input(const_blob_id, constParams.name); + } + else + { + if (inputScale < 0.f) + { + addNegation(layerParams, node_proto, input_id); + } + + layerParams.type = "Scale"; + layerParams.set("bias_term", true); + int axis = 1; + for (int i = 0; i < graph_proto.initializer_size(); i++) + { + opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); + if (tensor_proto.name() == node_proto.input(const_blob_id)) + { + axis = inpShape.size() - tensor_proto.dims_size(); + break; + } + } + layerParams.set("axis", axis); + blob = blob.reshape(1, 1); + layerParams.blobs.push_back(constScale * blob); + } + } + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + if (isSub) + { + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); + } + } + else + { + if (isSub) + { + addNegation(layerParams, node_proto, 1); + } + layerParams.type = "Scale"; + layerParams.set("bias_term", true); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePow(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (layer_id.find(node_proto.input(1)) != layer_id.end()) + CV_Error(Error::StsNotImplemented, "Unsupported Pow op with variable power"); + + Mat blob = getBlob(node_proto, 1); + if (blob.total() != 1) + CV_Error(Error::StsNotImplemented, "Pow op supports only scalar power"); + + blob.convertTo(blob, CV_32F); + layerParams.type = "Power"; + layerParams.set("power", blob.ptr()[0]); + addLayer(layerParams, node_proto); +} + +// "Min" "Max" +void ONNXImporter::parseMinMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const std::string& layer_type = node_proto.op_type(); + layerParams.type = "Eltwise"; + layerParams.set("operation", layer_type == "Max" ? "max" : "min"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseNeg(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "Power"; + layerParams.set("scale", -1); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConstant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 0); + CV_Assert(layerParams.blobs.size() == 1); + addConstant(layerParams.name, layerParams.blobs[0]); +} + +void ONNXImporter::parseLSTM(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + LayerParams lstmParams = layerParams; + lstmParams.name += "/lstm"; + + // https://pytorch.org/docs/stable/nn.html#lstm + CV_Assert(node_proto.input_size() >= 7); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + + const int numHidden = lstmParams.get("hidden_size"); + const int numDirs = Wx.size[0]; // Is 1 for forward only and 2 for bidirectional LSTM. + const int numFeatures = Wx.size[2]; + + Mat h0, c0; + if (!node_proto.input(5).empty()) { + h0 = getBlob(node_proto, 5); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + } else { + // initial_h attribute can be empty in case of keras2onnx producer. fill it with zeros + h0 = Mat::zeros(numDirs * numFeatures, numHidden, CV_32FC1); + } + if (!node_proto.input(6).empty()) { + c0 = getBlob(node_proto, 6); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + } else { + // initial_c attribute can be empty in case of keras2onnx producer. fill it with zeros + c0 = Mat::zeros(numDirs * numFeatures, numHidden, CV_32FC1); + } + + b = b.reshape(1, b.size[0]); + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) + { + for (int i = 0; i < numFeatures; ++i) + { + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); + } + for (int i = 0; i < numHidden; ++i) + { + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); + } + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + } + } + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + + + lstmParams.blobs.resize(5); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; + + // read direction attribute + lstmParams.set("reverse", lstmParams.get("direction", "") == "reverse"); + lstmParams.set("bidirectional", lstmParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, lstmParams.name); // set different name so output shapes will be registered on that name + addLayer(lstmParams, node_proto); + + MatShape lstmShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + lstmShape.insert(lstmShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&lstmShape[0], lstmShape.size())); + node_proto.set_input(0, lstmParams.name); // redirect input to LSTM + node_proto.set_output(0, layerParams.name); // keep origin LSTM's name + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseGRU(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + LayerParams gruParams = layerParams; + gruParams.name += "/gru"; + + // https://pytorch.org/docs/stable/generated/torch.nn.GRU.html?highlight=gru# + CV_Assert(node_proto.input_size() == 6); + Mat Wx = getBlob(node_proto, 1); + Mat Wh = getBlob(node_proto, 2); + Mat b = getBlob(node_proto, 3); + Mat h0 = getBlob(node_proto, 5); + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + b = b.reshape(1, b.size[0]); + + gruParams.blobs.resize(4); + gruParams.blobs[0] = Wh; + gruParams.blobs[1] = Wx; + gruParams.blobs[2] = b; + gruParams.blobs[3] = h0; + gruParams.set("bidirectional", gruParams.get("direction", "") == "bidirectional"); + + node_proto.set_output(0, gruParams.name); // set different name so output shapes will be registered on that name + addLayer(gruParams, node_proto); + + MatShape gruShape = outShapes[node_proto.output(0)]; + + // Add fake 1 as it is done in ONNX + gruShape.insert(gruShape.begin() + 1, 1); + + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&gruShape[0], gruShape.size())); + node_proto.set_input(0, gruParams.name); // redirect input to GRU + node_proto.set_output(0, layerParams.name); // keep origin GRU's name + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseImageScaler(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const float scale = layerParams.has("scale") ? layerParams.get("scale") : 1.0f; + layerParams.erase("scale"); + + if (layerParams.has("bias")) + { + layerParams.type = "Scale"; + layerParams.blobs.push_back( + Mat(Size(1, layerParams.get("bias").size()), CV_32FC1, scale)); + + layerParams.set("bias_term", true); + Mat bias(1, layerParams.get("bias").size(), CV_32FC1); + for (int j = 0; j < bias.total(); j++) { + bias.at(0, j) = layerParams.get("bias").getRealValue(j); + } + layerParams.blobs.push_back(bias); + layerParams.erase("bias"); + } + else { + layerParams.set("scale", scale); + layerParams.type = "Power"; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseClip(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU6"; + replaceLayerParam(layerParams, "min", "min_value"); + replaceLayerParam(layerParams, "max", "max_value"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + replaceLayerParam(layerParams, "alpha", "negative_slope"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ReLU"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseElu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "ELU"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseTanh(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "TanH"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "PReLU"; + layerParams.blobs.push_back(getBlob(node_proto, 1)); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseLRN(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + replaceLayerParam(layerParams, "size", "local_size"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseInstanceNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + if (node_proto.input_size() != 3) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias"); + + layerParams.blobs.resize(4); + layerParams.blobs[2] = getBlob(node_proto, 1); // weightData + layerParams.blobs[3] = getBlob(node_proto, 2); // biasData + layerParams.set("has_bias", true); + layerParams.set("has_weight", true); + + // Get number of channels in input + int size = layerParams.blobs[2].total(); + layerParams.blobs[0] = Mat::zeros(size, 1, CV_32F); // mean + layerParams.blobs[1] = Mat::ones(size, 1, CV_32F); // std + + LayerParams mvnParams; + mvnParams.name = layerParams.name + "/MVN"; + mvnParams.type = "MVN"; + mvnParams.set("eps", layerParams.get("epsilon")); + layerParams.erase("epsilon"); + + //Create MVN layer + int id = dstNet.addLayer(mvnParams.name, mvnParams.type, mvnParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(0)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(mvnParams.name, LayerInfo(id, 0))); + outShapes[mvnParams.name] = outShapes[node_proto.input(0)]; + + //Replace Batch Norm's input to MVN + node_proto.set_input(0, mvnParams.name); + layerParams.type = "BatchNorm"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseBatchNormalization(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (node_proto.input_size() != 5) + CV_Error(Error::StsNotImplemented, + "Expected input, scale, bias, mean and var"); + + layerParams.type = "BatchNorm"; + replaceLayerParam(layerParams, "epsilon", "eps"); + replaceLayerParam(layerParams, "spatial", "use_global_stats"); + + Mat meanData = getBlob(node_proto, 3); + Mat stdData = getBlob(node_proto, 4); + + layerParams.blobs.push_back(meanData); + layerParams.blobs.push_back(stdData); + + if (!node_proto.input(1).empty()) { + layerParams.set("has_weight", true); + layerParams.blobs.push_back(getBlob(node_proto, 1)); // weightData + } else { + layerParams.set("has_weight", false); + } + + if (!node_proto.input(2).empty()) { + layerParams.set("has_bias", true); + layerParams.blobs.push_back(getBlob(node_proto, 2)); // biasData + } else { + layerParams.set("has_bias", false); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "InnerProduct"; + Mat weights = getBlob(node_proto, 1); + int ind_num_out = 0; + if (layerParams.has("transB") && !layerParams.get("transB")) { + transpose(weights, weights); + ind_num_out = 1; + } + layerParams.blobs.push_back(weights); + + if (node_proto.input_size() == 3) { + Mat bias = getBlob(node_proto, 2); + layerParams.blobs.push_back(bias); + } + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inputBuf = getBlob(node_proto, 0); + + LayerParams constParams; + constParams.name = node_proto.input(0); + constParams.type = "Const"; + constParams.blobs.push_back(inputBuf); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + + layerParams.set("num_output", layerParams.blobs[0].size[ind_num_out]); + layerParams.set("bias_term", node_proto.input_size() == 3); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2); + layerParams.type = "InnerProduct"; + layerParams.set("bias_term", false); + CV_Assert(constBlobs.find(node_proto.input(0)) == constBlobs.end()); + int firstInpDims = outShapes[node_proto.input(0)].size(); + int secondInpDims; + + if (constBlobs.find(node_proto.input(1)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 1); + secondInpDims = blob.dims; + layerParams.blobs.push_back(blob.t()); + layerParams.set("num_output", layerParams.blobs[0].size[0]); + } else { + secondInpDims = outShapes[node_proto.input(1)].size(); + } + layerParams.set("axis", firstInpDims - secondInpDims + 1); + addLayer(layerParams, node_proto); +} + +void findBroadAxis(const MatShape& broadShape, const MatShape& outShape, size_t& axis, int& broadAxis) +{ + const size_t diff = outShape.size() - broadShape.size(); + + // find the first non-one element of the broadcasting shape + axis = 0; + for (; axis < broadShape.size() && broadShape[axis] == 1; ++axis) {} + + // find the last non-one element of the broadcasting shape + size_t endAxis = broadShape.size(); + for (; endAxis > axis && broadShape[endAxis - 1] == 1; --endAxis) {} + + // find one between axis and endAxis - as it needs to be broadcasted, + // dimensions from the left of axis and from the right of endAxis will be handled by Scale layer + broadAxis = -1; + for (size_t i = axis; i < endAxis; ++i) + { + size_t outAxis = i + diff; + if (outShape[outAxis] == broadShape[i]) + { + continue; + } + + // ensure we need to broadcast only 1 dimension in the middle + CV_Assert(broadShape[i] == 1 && broadAxis == -1); + broadAxis = static_cast(outAxis); + } + + axis += diff; +} + +// "Mul" "Div" +void ONNXImporter::parseMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + const std::string& layer_type = node_proto.op_type(); + CV_Assert(node_proto.input_size() == 2); + + bool isDiv = layer_type == "Div"; + int constId = -1; + bool haveVariables = false; + for (int i = 0; i < 2; ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + constId = i; + else + haveVariables = true; + } + if (constId != -1 && haveVariables) + { + Mat blob = getBlob(node_proto, constId); + blob = blob.reshape(1, 1); + if (blob.total() == 1) { + float blob_value = blob.ptr()[0]; + float coeff = isDiv ? 1.0 / blob_value : blob_value; + layerParams.set("scale", coeff); + layerParams.type = "Power"; + } + else { + if (isDiv) + divide(1.0, blob, blob); + layerParams.blobs.push_back(blob); + layerParams.type = "Scale"; + } + } + else if (!haveVariables) + { + Mat inp0 = getBlob(node_proto, 0); + Mat inp1 = getBlob(node_proto, 1); + + if (inp0.size != inp1.size && (inp0.total() != 1 || inp1.total() != 1)) + CV_Error_(Error::StsNotImplemented, ("Different shapes case is not supported with constant inputs: %s", layer_type.c_str())); + + if (inp0.total() == 1 && inp1.total() == 1 && inp0.dims != inp1.dims) + { + if (inp0.dims < inp1.dims) + { + inp0 = inp0.reshape(1, inp1.dims, inp1.size); + inp0.dims = inp1.dims; + } + else + { + inp1 = inp1.reshape(1, inp0.dims, inp0.size); + inp1.dims = inp0.dims; + } + } + + Mat out; + if (inp0.total() != inp1.total()) + { + if (inp0.total() == 1) + { + float inp0_value = inp0.ptr()[0]; + float coeff = isDiv ? 1.0 / inp0_value : inp0_value; + multiply(inp1, coeff, out); + } + else + { + float inp1_value = inp1.ptr()[0]; + float coeff = isDiv ? 1.0 / inp1_value : inp1_value; + multiply(inp0, coeff, out); + } + + } + else + { + out = isDiv ? inp0 / inp1 : inp0.mul(inp1); + } + + if (inp0.dims == 1 && inp1.dims == 1) + out.dims = 1; // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)]) + { + layerParams.type = "Eltwise"; + layerParams.set("operation", isDiv ? "div" : "prod"); + } + else + { + // Scale layer allocate output with the first input shape + if (total(outShapes[node_proto.input(0)]) < total(outShapes[node_proto.input(1)])) + { + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(1)); + proto.add_input(node_proto.input(0)); + proto.add_output(layerParams.name); + node_proto = proto; + } + + if (isDiv) + { + LayerParams powerParams; + powerParams.name = layerParams.name + "/inv"; + powerParams.type = "Power"; + powerParams.set("power", -1); + + //Create Power layer + int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams); + //Connect to input + IterLayerId_t layerId = layer_id.find(node_proto.input(1)); + CV_Assert(layerId != layer_id.end()); + dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0); + //Add shape + layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0))); + outShapes[powerParams.name] = outShapes[node_proto.input(1)]; + + //Replace input to Power + node_proto.set_input(1, powerParams.name); + } + + const MatShape& broadShape = outShapes[node_proto.input(1)]; + const MatShape& outShape = outShapes[node_proto.input(0)]; + + size_t axis = 0; + int broadAxis = -1; + findBroadAxis(broadShape, outShape, axis, broadAxis); + + // if there is a one dimension in the middle that should be broadcasted, broadcast it + if (broadAxis != -1) + { + opencv_onnx::NodeProto concat_node_proto = node_proto; + const std::string& input1 = concat_node_proto.input(1); + + expandMid(layerParams.name, concat_node_proto, input1, outShape[broadAxis]); + + LayerParams concatLP; + concatLP.name = layerParams.name + "/concat"; + concatLP.set("axis", broadAxis); + concatLP.type = "Concat"; + concat_node_proto.set_output(0, concatLP.name); + + addLayer(concatLP, concat_node_proto); + node_proto.set_input(1, concatLP.name); + } + + CV_Assert(axis != outShape.size()); + layerParams.set("axis", static_cast(axis)); + layerParams.type = "Scale"; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Convolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + if (constBlobs.find(node_proto.input(j)) != constBlobs.end()) + { + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + } + int outCn = layerParams.blobs.empty() ? outShapes[node_proto.input(1)][0] : layerParams.blobs[0].size[0]; + layerParams.set("num_output", outCn); + + // Check for asymmetric padding in Conv2D + if (layerParams.has("pad")) + { + bool asymmetricPadding = false; + DictValue pads = layerParams.get("pad"); + const int dims = pads.size() / 2; + for (int i = 0; i < dims; ++i) + { + if (pads.get(i) != pads.get(i + dims)) + { + asymmetricPadding = true; + break; + } + } + if (asymmetricPadding && pads.size() == 4) // [pad_t, pad_l, pad_b, pad_r] + { + layerParams.erase("pad"); + // No paddings required for N, C axis + std::vector paddings(4, 0); + // Add paddings for H, W axis + for (int i = 0; i < dims; ++i) + { + paddings.push_back(pads.get(i)); + paddings.push_back(pads.get(dims + i)); + } + LayerParams padLp; + padLp.name = layerParams.name + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(padLp.name); + + addLayer(padLp, proto); + node_proto.set_input(0, padLp.name); + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConvTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() >= 2); + layerParams.type = "Deconvolution"; + for (int j = 1; j < node_proto.input_size(); j++) { + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + layerParams.set("num_output", layerParams.blobs[0].size[1] * layerParams.get("group", 1)); + layerParams.set("bias_term", node_proto.input_size() == 3); + + if (!layerParams.has("kernel_size")) + CV_Error(Error::StsNotImplemented, + "Required attribute 'kernel_size' is not present."); + + if (layerParams.has("output_shape")) + { + const DictValue& outShape = layerParams.get("output_shape"); + DictValue strides = layerParams.get("stride"); + DictValue kernel = layerParams.get("kernel_size"); + + String padMode; + std::vector adjust_pads; + if (layerParams.has("pad_mode")) + { + padMode = toUpperCase(layerParams.get("pad_mode")); + if (padMode != "SAME" && padMode != "VALID") + CV_Error(Error::StsError, "Unsupported padding mode " + padMode); + + for (int i = 0; i < strides.size(); i++) + { + int sz = outShape.get(2 + i); + int stride = strides.get(i); + adjust_pads.push_back(padMode == "SAME"? (sz - 1) % stride : + (sz - kernel.get(i)) % stride); + } + layerParams.set("adj", DictValue::arrayInt(&adjust_pads[0], adjust_pads.size())); + } + } + else if (layerParams.has("output_padding")) + { + replaceLayerParam(layerParams, "output_padding", "adj"); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseTranspose(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PermuteInt8" : "Permute"; + replaceLayerParam(layerParams, "perm", "order"); + + CV_Assert(node_proto.input_size() == 1); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + std::vector inputs(1, getBlob(node_proto, 0)), transposed; + runLayer(layerParams, inputs, transposed); + CV_Assert(transposed.size() == 1); + addConstant(layerParams.name, transposed[0]); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert_N(node_proto.input_size() == 1, layerParams.has("axes")); + DictValue axes_dict = layerParams.get("axes"); + MatShape inpShape = outShapes[node_proto.input(0)]; + + std::vector maskedAxes(inpShape.size(), false); + for (int i = 0; i < axes_dict.size(); ++i) + { + int axis = axes_dict.getIntValue(i); + CV_CheckLE(axis, static_cast(inpShape.size()), "Squeeze axis"); + maskedAxes[axis] = inpShape[axis] == 1; + } + MatShape outShape; + for (int i = 0; i < inpShape.size(); ++i) + { + if (!maskedAxes[i]) + outShape.push_back(inpShape[i]); + } + if (outShape.size() != inpShape.size()) + { + layerParams.type = "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + { + if (!maskedAxes[index]) + inputIndices.push_back(index); + } + for (int index = 0; index < outShape.size(); ++index) + dynamicAxes.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + } + else + layerParams.type = "Identity"; + + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat inp = getBlob(node_proto, 0); + Mat out = inp.reshape(1, outShape); + out.dims = outShape.size(); // to workaround dims == 1 + addConstant(layerParams.name, out); + return; + } + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseFlatten(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_CheckEQ(node_proto.input_size(), 1, ""); + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat input = getBlob(node_proto, 0); + int axis = normalize_axis(layerParams.get("axis", 1), input.dims); + + std::vector out_size(&input.size[0], &input.size[0] + axis); + out_size.push_back(input.total(axis)); + Mat output = input.reshape(1, out_size); + addConstant(layerParams.name, output); + return; + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUnsqueeze(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1 || node_proto.input_size() == 2); + DictValue axes; + if (node_proto.input_size() == 2) + { + Mat blob = getBlob(node_proto, 1); + axes = DictValue::arrayInt(blob.ptr(), blob.total()); + } + else + axes = layerParams.get("axes"); + + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + // Constant input. + Mat input = getBlob(node_proto, 0); + + std::vector dims; + for (int j = 0; j < input.dims; j++) { + dims.push_back(input.size[j]); + } + CV_Assert(axes.getIntValue(axes.size()-1) <= dims.size()); + for (int j = 0; j < axes.size(); j++) { + dims.insert(dims.begin() + axes.getIntValue(j), 1); + } + + Mat out = input.reshape(0, dims); + addConstant(layerParams.name, out); + return; + } + + // Variable input. + if (axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Multidimensional unsqueeze"); + + int depth = layerParams.get("depth", CV_32F); + + MatShape inpShape = outShapes[node_proto.input(0)]; + int axis = axes.getIntValue(0); + CV_Assert(0 <= axis && axis <= inpShape.size()); + std::vector outShape = inpShape; + outShape.insert(outShape.begin() + axis, 1); + layerParams.type = (depth == CV_8S) ? "ReshapeInt8" : "Reshape"; + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < outShape.size(); ++index) { + if (index != axis) + dynamicAxes.push_back(index); + } + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseExpand(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 2, ""); + const std::string& input0 = node_proto.input(0); + const std::string& input1 = node_proto.input(1); + Mat newShapeMat = getBlob(input1); + MatShape targetShape(newShapeMat.ptr(), newShapeMat.ptr() + newShapeMat.total()); + + MatShape inpShape; + bool haveVariables = constBlobs.find(input0) == constBlobs.end(); + if (haveVariables) + { + IterShape_t shapeIt = outShapes.find(input0); + CV_Assert(shapeIt != outShapes.end()); + inpShape = shapeIt->second; + } + else + { + inpShape = shape(getBlob(input0)); + } + + String srcName = input0; + // Unsqueeze and repeat along new axis + if (targetShape.size() == inpShape.size() + 1) + { + inpShape.insert(inpShape.begin(), targetShape.size() - inpShape.size(), 1); + for (int i = 0; i < targetShape.size(); i++) + { + if (abs(targetShape[i]) == 1) + targetShape[i] = inpShape[i]; + } + if (haveVariables) + { + LayerParams reshapeLp; + reshapeLp.name = layerParams.name + "/reshape"; + reshapeLp.type = "Reshape"; + CV_Assert(layer_id.find(reshapeLp.name) == layer_id.end()); + reshapeLp.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(reshapeLp.name); + addLayer(reshapeLp, proto); + srcName = reshapeLp.name; + } + } + CV_CheckEQ(inpShape.size(), targetShape.size(), "Unsupported Expand op with different dims"); + + std::vector broadcast_axes; + // shapes aren't right-aligned here because targetShape.size() == inpShape.size() + for (int i = 0; i < targetShape.size(); i++) + { + if (targetShape[i] != inpShape[i]) + { + if (inpShape[i] == 1) + { + broadcast_axes.push_back(i); + } + else if (targetShape[i] != 1) + { + CV_Error(Error::StsError, format("Could not be broadcast by axis: %d", i)); + } + } + } + + if (!haveVariables) + { + if (broadcast_axes.size() != 1) + CV_Error(Error::StsNotImplemented, "Expand op doesn't support multiple axes for constant input"); + + Mat input = getBlob(node_proto, 0); + input = input.reshape(0, total(inpShape, 0, broadcast_axes[0])); + Mat output = cv::repeat(input, 1, targetShape[broadcast_axes[0]]); + output = output.reshape(0, targetShape); + addConstant(layerParams.name, output); + return; + } + + if (broadcast_axes.size() == 2 && + broadcast_axes[0] == broadcast_axes[1] - 1 && broadcast_axes[1] == inpShape.size() - 1) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + CV_Assert(layer_id.find(constParams.name) == layer_id.end()); + constParams.type = "Const"; + + Mat inp = Mat::ones(newShapeMat.total(), newShapeMat.ptr(), CV_32F); + constParams.blobs.push_back(inp); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + + layerParams.type = "Scale"; + layerParams.set("bias_term", false); + node_proto.set_input(0, constParams.name); + node_proto.set_input(1, srcName); + } + else if (broadcast_axes.size() == 1 && broadcast_axes[0] <= 1) + { + expandMid(layerParams.name, node_proto, srcName, targetShape[broadcast_axes[0]]); + + layerParams.set("axis", broadcast_axes[0]); + layerParams.type = "Concat"; + node_proto.set_output(0, layerParams.name); + } + else if (broadcast_axes.empty()) + { + layerParams.type = "Identity"; + } + else + CV_Error(Error::StsNotImplemented, "Unsupported Expand op"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseReshape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 2 || layerParams.has("shape")); + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; + + if (node_proto.input_size() == 2) { + Mat blob = getBlob(node_proto, 1); + CV_Assert(blob.type() == CV_32SC1); + + layerParams.set("dim", DictValue::arrayInt(blob.ptr(), blob.total())); + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + std::vector inputs(1, getBlob(node_proto, 0)), outputs; + runLayer(layerParams, inputs, outputs); + addConstant(layerParams.name, outputs[0]); + return; + } + } + else { + DictValue shape = layerParams.get("shape"); + std::vector dim; + for (int j = 0; j < shape.size(); j++) { + dim.push_back(shape.getIntValue(j)); + } + + if (layer_id.find(node_proto.input(0)) == layer_id.end()) { + Mat input = getBlob(node_proto, 0); + Mat out = input.reshape(0, dim); + addConstant(layerParams.name, out); + return; + } + replaceLayerParam(layerParams, "shape", "dim"); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parsePad(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int depth = layerParams.get("depth", CV_32F); + layerParams.type = (depth == CV_8S) ? "PaddingInt8" : "Padding"; + replaceLayerParam(layerParams, "mode", "type"); + if (node_proto.input_size() == 3 || node_proto.input_size() == 2) + { + // Paddings are in order begin0, begin1, .. beginN, end0, end1, ..., endN. + // We need to shuffle it to begin0, end0, begin1, end1, ... + Mat paddings = getBlob(node_proto, 1).reshape(1, 2); + paddings = paddings.t(); + layerParams.set("paddings", DictValue::arrayInt(paddings.ptr(), paddings.total())); + + if (node_proto.input_size() == 3) + { + Mat value = getBlob(node_proto, 2); + float padValue = (depth == CV_8S) ? (float)value.ptr()[0] : value.ptr()[0]; + layerParams.set("value", padValue); + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseShape(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 1); + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + const MatShape& inpShape = shapeIt->second; + + Mat shapeMat(inpShape.size(), 1, CV_32S); + for (int j = 0; j < inpShape.size(); ++j) + shapeMat.at(j) = inpShape[j]; + shapeMat.dims = 1; + + addConstant(layerParams.name, shapeMat); +} + +void ONNXImporter::parseCast(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + if (constBlobs.find(node_proto.input(0)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, 0); + int type; + switch (layerParams.get("to")) + { + case opencv_onnx::TensorProto_DataType_FLOAT: type = CV_32F; break; + case opencv_onnx::TensorProto_DataType_UINT8: type = CV_8U; break; + case opencv_onnx::TensorProto_DataType_UINT16: type = CV_16U; break; + case opencv_onnx::TensorProto_DataType_FLOAT16: type = CV_16S; break; + case opencv_onnx::TensorProto_DataType_INT8: + case opencv_onnx::TensorProto_DataType_INT16: + case opencv_onnx::TensorProto_DataType_INT32: + case opencv_onnx::TensorProto_DataType_INT64: type = CV_32S; break; + default: type = blob.type(); + } + Mat dst; + blob.convertTo(dst, type); + dst.dims = blob.dims; + addConstant(layerParams.name, dst); + return; + } + else + layerParams.type = "Identity"; + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConstantFill(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int depth = CV_32F; + float fill_value; + if (!layerParams.blobs.empty()) + { + CV_Assert(!layerParams.has("value")); + depth = layerParams.blobs[0].depth(); + Mat floats; + layerParams.blobs[0].convertTo(floats, CV_32F); + fill_value = floats.at(0, 0); + } + else + fill_value = layerParams.get("value", 0); + + MatShape inpShape = getBlob(node_proto, 0); + for (int i = 0; i < inpShape.size(); i++) + CV_CheckGT(inpShape[i], 0, ""); + Mat tensor(inpShape.size(), &inpShape[0], depth, Scalar(fill_value)); + addConstant(layerParams.name, tensor); +} + +void ONNXImporter::parseGather(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() == 2); + Mat indexMat = getBlob(node_proto, 1); + CV_Assert_N(indexMat.type() == CV_32S, indexMat.total() == 1); + int index = indexMat.at(0); + int axis = layerParams.get("axis", 0); + + if ((constBlobs.find(node_proto.input(0)) != constBlobs.end())) + { + Mat input = getBlob(node_proto, 0); + Mat out; + std::vector ranges(input.dims, Range::all()); + ranges[axis] = Range(index, index + 1); + + out = input(ranges); + MatShape outShape = shape(out); + if (outShape.size() > 1) + { + outShape.erase(outShape.begin() + axis); + out.reshape(0, outShape); + } else { + out.dims = 1; + } + addConstant(layerParams.name, out); + return; + } + else + { + IterShape_t shapeIt = outShapes.find(node_proto.input(0)); + CV_Assert(shapeIt != outShapes.end()); + MatShape inpShape = shapeIt->second; + + LayerParams sliceLp; + sliceLp.type = "Slice"; + sliceLp.name = inpShape.size() > 1 ? layerParams.name + "/slice" : layerParams.name; + std::vector begin(inpShape.size(), 0); + std::vector end(inpShape.size(), -1); + begin[axis] = index; + end[axis] = index + 1; + + cv::dnn::DictValue paramBegin = cv::dnn::DictValue::arrayInt(begin.data(), begin.size()); + cv::dnn::DictValue paramEnd = cv::dnn::DictValue::arrayInt(end.data(), end.size()); + sliceLp.set("begin", paramBegin); + sliceLp.set("end", paramEnd); + sliceLp.set("has_dynamic_shapes", hasDynamicShapes); + + if (inpShape.size() > 1) + { + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(0)); + proto.add_output(sliceLp.name); + addLayer(sliceLp, proto); + + inpShape.erase(inpShape.begin() + axis); + layerParams.type = "Reshape"; + layerParams.set("axis", 0); + layerParams.set("dim", DictValue::arrayInt(&inpShape[0], inpShape.size())); + if (hasDynamicShapes) + { + std::vector dynamicAxes; + std::vector inputIndices; + for (int index = 0; index < inpShape.size(); ++index) + dynamicAxes.push_back(index); + for (int index = 0; index < inpShape.size(); ++index) + inputIndices.push_back(index); + layerParams.set("dynamic_axes", DictValue::arrayInt(dynamicAxes.data(), dynamicAxes.size())); + layerParams.set("input_indices", DictValue::arrayInt(inputIndices.data(), inputIndices.size())); + } + node_proto.set_input(0, sliceLp.name); + } + else + { + layerParams = sliceLp; + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + bool hasVariableInps = false; + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (layer_id.find(node_proto.input(i)) != layer_id.end()) + { + hasVariableInps = true; + break; + } + } + + if (!hasVariableInps) + { + std::vector inputs(node_proto.input_size()), concatenated; + // Due constant folding we can get inputs with different number of dimensions + // Insert the missing dimension to inputs + MatShape inputShape; + for (size_t i = 0; i < inputs.size(); ++i) + { + inputs[i] = getBlob(node_proto, i); + if (inputs[i].size.dims() > inputShape.size()) + { + inputShape = shape(inputs[i]); + } + } + + // Concat-1 has default value for axis is 1: https://github.com/onnx/onnx/blob/master/docs/Changelog.md#Concat-1 + int axis = layerParams.get("axis", 1); + for (size_t i = 0; i < inputs.size(); ++i) + { + MatShape targetShape = inputShape; + targetShape[axis] = shape(inputs[i])[axis]; + CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); + inputs[i] = inputs[i].reshape(0, targetShape); + } + runLayer(layerParams, inputs, concatenated); + + CV_Assert(concatenated.size() == 1); + addConstant(layerParams.name, concatenated[0]); + return; + } + else + { + for (int i = 0; i < node_proto.input_size(); ++i) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "Const"; + constParams.blobs.push_back(getBlob(node_proto, i)); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseResize(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int i = 1; i < node_proto.input_size(); i++) + CV_Assert(layer_id.find(node_proto.input(i)) == layer_id.end()); + + int depth = layerParams.get("depth", CV_32F); + layerParams.type += (depth == CV_8S) ? "Int8" : ""; + + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") + { + layerParams.set("mode", interp_mode == "pytorch_half_pixel" || interp_mode == "half_pixel" ? + "opencv_linear" : "bilinear"); + } + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + // input = [X, scales], [X, roi, scales] or [x, roi, scales, sizes] + int foundScaleId = hasDynamicShapes ? node_proto.input_size() - 1 + : node_proto.input_size() > 2 ? 2 : 1; + + Mat scales = getBlob(node_proto, foundScaleId); + if (scales.total() == 4) + { + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + else + { + const std::string& inputLast = node_proto.input(node_proto.input_size() - 1); + if (constBlobs.find(inputLast) != constBlobs.end()) + { + Mat shapes = getBlob(inputLast); + CV_CheckEQ(shapes.size[0], 4, ""); + CV_CheckEQ(shapes.size[1], 1, ""); + CV_CheckDepth(shapes.depth(), shapes.depth() == CV_32S || shapes.depth() == CV_32F, ""); + if (shapes.depth() == CV_32F) + shapes.convertTo(shapes, CV_32S); + layerParams.set("width", shapes.at(3)); + layerParams.set("height", shapes.at(2)); + } + } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + //fused from Resize Subgraph + if (layerParams.has("coordinate_transformation_mode")) + { + String interp_mode = layerParams.get("coordinate_transformation_mode"); + CV_Assert_N(interp_mode != "tf_crop_and_resize", interp_mode != "tf_half_pixel_for_nn"); + + layerParams.set("align_corners", interp_mode == "align_corners"); + if (layerParams.get("mode") == "linear") + { + layerParams.set("mode", interp_mode == "pytorch_half_pixel" ? + "opencv_linear" : "bilinear"); + } + } + if (layerParams.get("mode") == "linear" && framework_name == "pytorch") + layerParams.set("mode", "opencv_linear"); + + layerParams.type = "Resize"; + if (layerParams.has("scales")) + { + // Pytorch layer + DictValue scales = layerParams.get("scales"); + CV_Assert(scales.size() == 4); + layerParams.set("zoom_factor_y", scales.getIntValue(2)); + layerParams.set("zoom_factor_x", scales.getIntValue(3)); + } + else if (layerParams.has("height_scale") && layerParams.has("width_scale")) + { + // Caffe2 layer + replaceLayerParam(layerParams, "height_scale", "zoom_factor_y"); + replaceLayerParam(layerParams, "width_scale", "zoom_factor_x"); + } + else + { + // scales as input + const std::string& input1 = node_proto.input(1); + if (constBlobs.find(input1) != constBlobs.end()) + { + Mat scales = getBlob(input1); + CV_Assert(scales.total() == 4); + layerParams.set("zoom_factor_y", scales.at(2)); + layerParams.set("zoom_factor_x", scales.at(3)); + } + } + replaceLayerParam(layerParams, "mode", "interpolation"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseSoftMax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + const std::string& layer_type = node_proto.op_type(); + layerParams.type = "Softmax"; + layerParams.set("log_softmax", layer_type == "LogSoftmax"); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseDetectionOutput(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_CheckEQ(node_proto.input_size(), 3, ""); + if (constBlobs.find(node_proto.input(2)) != constBlobs.end()) + { + Mat priors = getBlob(node_proto, 2); + + LayerParams constParams; + constParams.name = layerParams.name + "/priors"; + constParams.type = "Const"; + constParams.blobs.push_back(priors); + + opencv_onnx::NodeProto priorsProto; + priorsProto.add_output(constParams.name); + addLayer(constParams, priorsProto); + + node_proto.set_input(2, constParams.name); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseCumSum(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + layerParams.type = "CumSum"; + + // Get axis. + const std::string& input1 = node_proto.input(1); + + if (constBlobs.find(input1) != constBlobs.end()) + { + Mat axis_blob = getBlob(input1); + CV_Assert(axis_blob.total() == 1u); + layerParams.set("axis", axis_blob.at(0)); + } + + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseCustomLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + for (int j = 0; j < node_proto.input_size(); j++) { + if (layer_id.find(node_proto.input(j)) == layer_id.end()) + layerParams.blobs.push_back(getBlob(node_proto, j)); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQuantDequant(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 3); + layerParams.type = (node_proto.op_type() == "QuantizeLinear") ? "Quantize" : "Dequantize"; + + if (node_proto.op_type() == "DequantizeLinear") + { + Mat scale = getBlob(node_proto, 1); + Mat zeropoint = getBlob(node_proto, 2); + + layerParams.set("scales", DictValue::arrayReal(scale.ptr(), 1)); + layerParams.set("zeropoints", DictValue::arrayInt(zeropoint.ptr(), 1)); + } + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 8 || ninputs == 9); + + Mat inp_sc = getBlob(node_proto, 1); + Mat inp_zp = getBlob(node_proto, 2); + + Mat weights = getBlob(node_proto, 3); + int outCn = weights.size[0]; + Mat w_scale = getBlob(node_proto, 4); + CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); + Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); + + Mat out_sc = getBlob(node_proto, 6); + Mat bias = (ninputs == 9) ? getBlob(node_proto, 8) : Mat::zeros(1, outCn, CV_32S); + + Mat weights_2d = weights.reshape(1, outCn); + Mat biasFused(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int i = 0; i < outCn; i++) + { + biasFused.at(i) = bias.at(i) - inp_zp.at(0)*(cv::sum(weights_2d.row(i))[0]); + outputMultiplier.at(i) = (inp_sc.at(0) * wt_sc.at(i)) / out_sc.at(0); + } + + layerParams.type = "ConvolutionInt8"; + layerParams.set("num_output", outCn); + layerParams.set("input_zeropoint", inp_zp.at(0)); + layerParams.blobs.push_back(weights); + layerParams.blobs.push_back(biasFused); + layerParams.blobs.push_back(outputMultiplier); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int ninputs = node_proto.input_size(); + CV_Assert(ninputs == 8); + + if (constBlobs.find(node_proto.input(3)) == constBlobs.end()) + CV_Error(Error::StsNotImplemented, "Variable weights is not supported"); + + int firstInpDims = outShapes[node_proto.input(0)].size(); + + Mat inp_sc = getBlob(node_proto, 1); + Mat inp_zp = getBlob(node_proto, 2); + + Mat weights = getBlob(node_proto, 3).t(); + int outCn = weights.size[0]; + int secondInpDims = weights.dims; + + Mat w_scale = getBlob(node_proto, 4); + CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn); + Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at(0))); + Mat out_sc = getBlob(node_proto, 6); + + Mat bias(1, outCn, CV_32S); + Mat outputMultiplier(1, outCn, CV_32F); + for (int i = 0; i < outCn; i++) + { + bias.at(i) = -inp_zp.at(0)*(cv::sum(weights.row(i))[0]); + outputMultiplier.at(i) = (inp_sc.at(0) * wt_sc.at(i)) / out_sc.at(0); + } + + layerParams.type = "InnerProductInt8"; + layerParams.set("num_output", outCn); + layerParams.set("axis", firstInpDims - secondInpDims + 1); + layerParams.blobs.push_back(weights); + layerParams.blobs.push_back(bias); + layerParams.blobs.push_back(outputMultiplier); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQEltwise(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + CV_Assert(node_proto.input_size() == 8); + std::string op = (node_proto.op_type() == "QLinearAdd") ? "sum" : "prod"; + int constId = -1; + for (int i = 0; i < 4; i += 3) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + constId = i; + } + + Mat inp_0_sc = getBlob(node_proto, 1); + Mat inp_0_zp = getBlob(node_proto, 2); + + Mat inp_1_sc = getBlob(node_proto, 4); + Mat inp_1_zp = getBlob(node_proto, 5); + + // Set 2nd input as the const input + if (constId == 0) + { + cv::swap(inp_0_sc, inp_1_sc); + cv::swap(inp_0_zp, inp_1_zp); + } + + float out_sc = getBlob(node_proto, 6).at(0); + int8_t out_zp = getBlob(node_proto, 7).at(0); + + std::vector inp_scales = {inp_0_sc.at(0), inp_1_sc.at(0)}; + std::vector inp_zps = {inp_0_zp.at(0), inp_1_zp.at(0)}; + + std::vector coeffs; + float offset; + if (op == "sum") + { + coeffs = {inp_scales[0]/out_sc, inp_scales[1]/out_sc}; + offset = out_zp - coeffs[0]*inp_zps[0] - coeffs[1]*inp_zps[1]; + } + else + { + coeffs = {inp_scales[0]/out_sc, inp_scales[1]}; + offset = out_zp; + } + + if (constId != -1) + { + Mat blob = getBlob(node_proto, constId); + if (blob.total() == 1) + { + float val = inp_scales[1] * (blob.at(0) - inp_zps[1]); + float scale = inp_scales[0] / out_sc; + if (op == "prod") + scale *= val; + + float shift = out_zp - scale*inp_zps[0]; + if (op == "sum") + shift += (val/out_sc); + + LayerParams rescaleParams; + rescaleParams.name = layerParams.name; + rescaleParams.type = "Requantize"; + rescaleParams.set("depth", CV_8S); + rescaleParams.set("scale", scale); + rescaleParams.set("shift", shift); + addLayer(rescaleParams, node_proto); + return; + } + else + { + MatShape inpShape = outShapes[node_proto.input(3 - constId)]; + if (blob.dims == 2) + blob = blob.t(); + + if (shape(blob) == inpShape) + { + LayerParams constParams; + constParams.name = layerParams.name + "/const"; + constParams.type = "ConstInt8"; + constParams.set("depth", CV_8S); + constParams.set("scales", DictValue::arrayReal(inp_1_sc.ptr(), 1)); + constParams.set("zeropoints", DictValue::arrayInt(inp_1_zp.ptr(), 1)); + constParams.blobs.push_back(blob); + + int id = dstNet.addLayer(constParams.name, constParams.type, CV_8S, constParams); + layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0))); + outShapes[constParams.name] = shape(blob); + node_proto.set_input(constId, constParams.name); + + layerParams.type = "EltwiseInt8"; + layerParams.set("operation", op); + layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size())); + layerParams.set("offset", offset); + } + else + { + layerParams.type = "ScaleInt8"; + layerParams.set("bias_term", op == "sum"); + int axis = 1; + for (int i = 0; i < graph_proto.initializer_size(); i++) + { + opencv_onnx::TensorProto tensor_proto = graph_proto.initializer(i); + if (tensor_proto.name() == node_proto.input(constId)) + { + axis = inpShape.size() - tensor_proto.dims_size(); + break; + } + } + layerParams.set("axis", axis); + blob = blob.reshape(1, 1); + Mat blob_dequantized; + blob.convertTo(blob_dequantized, CV_32F, inp_scales[1], -(inp_scales[1] * inp_zps[1])); + layerParams.blobs.push_back(blob_dequantized); + layerParams.set("input_scales", DictValue::arrayReal(inp_scales.data(), inp_scales.size())); + } + } + } + else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(3)]) + { + layerParams.type = "EltwiseInt8"; + layerParams.set("operation", op); + layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size())); + layerParams.set("offset", offset); + } + else + { + layerParams.type = "ScaleInt8"; + layerParams.set("bias_term", op == "sum"); + layerParams.set("input_scales", DictValue::arrayReal(inp_scales.data(), inp_scales.size())); + } + + layerParams.set("input_zeropoints", DictValue::arrayInt(inp_zps.data(), inp_zps.size())); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + + float slope = layerParams.get("alpha"); + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + int8_t out_zp = getBlob(node_proto, 4).at(0); + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inp_sc*(i - inp_zp); + float y = x >= 0.f ? x : slope*x; + int quantized = out_zp + cvRound(y/out_sc); + table[i+128] = saturate_cast(quantized); + } + + layerParams.type = "ReLUInt8"; + layerParams.blobs.push_back(lookUpTable); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQSigmoid(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + int8_t out_zp = getBlob(node_proto, 4).at(0); + + Mat lookUpTable(1, 256, CV_8S); + int8_t* table = lookUpTable.ptr(); + for (int i = -128; i < 128; i++) + { + float x = inp_sc*(i - inp_zp); + float y = 1.f/(1.f + std::exp(-x)); + int quantized = out_zp + cvRound(y/out_sc); + table[i+128] = saturate_cast(quantized); + } + + layerParams.type = "SigmoidInt8"; + layerParams.blobs.push_back(lookUpTable); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQAvgPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + CV_Assert(node_proto.input_size() == 5); + float inp_sc = getBlob(node_proto, 1).at(0); + int8_t inp_zp = getBlob(node_proto, 2).at(0); + float out_sc = getBlob(node_proto, 3).at(0); + + layerParams.type = "PoolingInt8"; + layerParams.set("pool", "ave"); + layerParams.set("global_pooling", node_proto.op_type() == "QLinearGlobalAveragePool"); + layerParams.set("multiplier", inp_sc/out_sc); + layerParams.set("input_zeropoint", inp_zp); + addLayer(layerParams, node_proto); +} + +void ONNXImporter::parseQConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_) +{ + opencv_onnx::NodeProto node_proto = node_proto_; + layerParams.type = "ConcatInt8"; + int num_inputs = node_proto.input_size(); + + float out_scale = getBlob(node_proto, 0).at(0); + int out_zp = getBlob(node_proto, 1).at(0); + + for (int i = 2; i < num_inputs; i += 3) + { + float inp_scale = getBlob(node_proto, i + 1).at(0); + int inp_zp = getBlob(node_proto, i + 2).at(0); + + if (inp_scale != out_scale || inp_zp != out_zp) + { + float scale = inp_scale/out_scale; + float shift = out_zp - scale*inp_zp; + + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + Mat blob = getBlob(node_proto, i); + Mat blob_rescaled; + blob.convertTo(blob_rescaled, CV_8S, scale, shift); + constBlobs[node_proto.input(i)] = blob_rescaled; + } + else + { + LayerParams rescaleParams; + rescaleParams.name = node_proto.input(i) + "/rescale"; + rescaleParams.type = "Requantize"; + rescaleParams.set("depth", CV_8S); + rescaleParams.set("scale", scale); + rescaleParams.set("shift", shift); + + opencv_onnx::NodeProto proto; + proto.add_input(node_proto.input(i)); + proto.add_output(rescaleParams.name); + addLayer(rescaleParams, proto); + node_proto.set_input(i, rescaleParams.name); + } + } + } + + bool hasVariableInps = false; + for (int i = 2; i < num_inputs; i += 3) + { + if (layer_id.find(node_proto.input(i)) != layer_id.end()) + { + hasVariableInps = true; + break; + } + } + + if (!hasVariableInps) + { + std::vector inputs, concatenated; + MatShape inputShape; + for (size_t i = 2; i < num_inputs; i += 3) + { + Mat blob = getBlob(node_proto, i); + if (blob.size.dims() > inputShape.size()) + { + inputShape = shape(blob); + } + inputs.push_back(blob); + } + + int axis = layerParams.get("axis", 1); + for (size_t i = 0; i < inputs.size(); ++i) + { + MatShape targetShape = inputShape; + targetShape[axis] = shape(inputs[i])[axis]; + CV_CheckEQ(total(targetShape), total(shape(inputs[i])), ""); + inputs[i] = inputs[i].reshape(0, targetShape); + } + runLayer(layerParams, inputs, concatenated); + CV_Assert(concatenated.size() == 1); + addConstant(layerParams.name, concatenated[0]); + return; + } + else + { + for (int i = 2; i < num_inputs; i += 3) + { + if (constBlobs.find(node_proto.input(i)) != constBlobs.end()) + { + LayerParams constParams; + constParams.name = node_proto.input(i); + constParams.type = "ConstInt8"; + constParams.blobs.push_back(getBlob(node_proto, i)); + constParams.set("depth", CV_8S); + + opencv_onnx::NodeProto proto; + proto.add_output(constParams.name); + addLayer(constParams, proto); + } + } + } + addLayer(layerParams, node_proto); +} + +const ONNXImporter::DispatchMap ONNXImporter::buildDispatchMap() +{ + DispatchMap dispatch; + + dispatch["MaxPool"] = &ONNXImporter::parseMaxPool; + dispatch["AveragePool"] = &ONNXImporter::parseAveragePool; + dispatch["GlobalAveragePool"] = dispatch["GlobalMaxPool"] = dispatch["ReduceMean"] = dispatch["ReduceSum"] = + dispatch["ReduceMax"] = &ONNXImporter::parseReduce; + dispatch["Slice"] = &ONNXImporter::parseSlice; + dispatch["Split"] = &ONNXImporter::parseSplit; + dispatch["Add"] = dispatch["Sum"] = dispatch["Sub"] = &ONNXImporter::parseBias; + dispatch["Pow"] = &ONNXImporter::parsePow; + dispatch["Min"] = dispatch["Max"] = &ONNXImporter::parseMinMax; + dispatch["Neg"] = &ONNXImporter::parseNeg; + dispatch["Constant"] = &ONNXImporter::parseConstant; + dispatch["LSTM"] = &ONNXImporter::parseLSTM; + dispatch["GRU"] = &ONNXImporter::parseGRU; + dispatch["ImageScaler"] = &ONNXImporter::parseImageScaler; + dispatch["Clip"] = &ONNXImporter::parseClip; + dispatch["LeakyRelu"] = &ONNXImporter::parseLeakyRelu; + dispatch["Relu"] = &ONNXImporter::parseRelu; + dispatch["Elu"] = &ONNXImporter::parseElu; + dispatch["Tanh"] = &ONNXImporter::parseTanh; + dispatch["PRelu"] = &ONNXImporter::parsePRelu; + dispatch["LRN"] = &ONNXImporter::parseLRN; + dispatch["InstanceNormalization"] = &ONNXImporter::parseInstanceNormalization; + dispatch["BatchNormalization"] = &ONNXImporter::parseBatchNormalization; + dispatch["Gemm"] = &ONNXImporter::parseGemm; + dispatch["MatMul"] = &ONNXImporter::parseMatMul; + dispatch["Mul"] = dispatch["Div"] = &ONNXImporter::parseMul; + dispatch["Conv"] = &ONNXImporter::parseConv; + dispatch["ConvTranspose"] = &ONNXImporter::parseConvTranspose; + dispatch["Transpose"] = &ONNXImporter::parseTranspose; + dispatch["Squeeze"] = &ONNXImporter::parseSqueeze; + dispatch["Flatten"] = &ONNXImporter::parseFlatten; + dispatch["Unsqueeze"] = &ONNXImporter::parseUnsqueeze; + dispatch["Expand"] = &ONNXImporter::parseExpand; + dispatch["Reshape"] = &ONNXImporter::parseReshape; + dispatch["Pad"] = &ONNXImporter::parsePad; + dispatch["Shape"] = &ONNXImporter::parseShape; + dispatch["Cast"] = &ONNXImporter::parseCast; + dispatch["ConstantFill"] = dispatch["ConstantOfShape"] = &ONNXImporter::parseConstantFill; + dispatch["Gather"] = &ONNXImporter::parseGather; + dispatch["Concat"] = &ONNXImporter::parseConcat; + dispatch["Resize"] = &ONNXImporter::parseResize; + dispatch["Upsample"] = &ONNXImporter::parseUpsample; + dispatch["SoftMax"] = dispatch["LogSoftmax"] = &ONNXImporter::parseSoftMax; + dispatch["DetectionOutput"] = &ONNXImporter::parseDetectionOutput; + dispatch["CumSum"] = &ONNXImporter::parseCumSum; + dispatch["QuantizeLinear"] = dispatch["DequantizeLinear"] = &ONNXImporter::parseQuantDequant; + dispatch["QLinearConv"] = &ONNXImporter::parseQConv; + dispatch["QLinearMatMul"] = &ONNXImporter::parseQMatMul; + dispatch["QLinearAdd"] = dispatch["QLinearMul"] = &ONNXImporter::parseQEltwise; + dispatch["QLinearLeakyRelu"] = &ONNXImporter::parseQLeakyRelu; + dispatch["QLinearSigmoid"] = &ONNXImporter::parseQSigmoid; + dispatch["QLinearAveragePool"] = dispatch["QLinearGlobalAveragePool"] = &ONNXImporter::parseQAvgPool; + dispatch["QLinearConcat"] = &ONNXImporter::parseQConcat; + + return dispatch; +} + Net readNetFromONNX(const String& onnxFile) { - Net net; - ONNXImporter onnxImporter(net, onnxFile.c_str()); - return net; + return detail::readNetDiagnostic(onnxFile.c_str()); } Net readNetFromONNX(const char* buffer, size_t sizeBuffer) { - Net net; - ONNXImporter onnxImporter(net, buffer, sizeBuffer); - return net; + return detail::readNetDiagnostic(buffer, sizeBuffer); } Net readNetFromONNX(const std::vector& buffer) diff --git a/modules/dnn/src/op_inf_engine.hpp b/modules/dnn/src/op_inf_engine.hpp index f52334bc45..ab2f161eaf 100644 --- a/modules/dnn/src/op_inf_engine.hpp +++ b/modules/dnn/src/op_inf_engine.hpp @@ -30,10 +30,11 @@ #define INF_ENGINE_RELEASE_2021_1 2021010000 #define INF_ENGINE_RELEASE_2021_2 2021020000 #define INF_ENGINE_RELEASE_2021_3 2021030000 +#define INF_ENGINE_RELEASE_2021_4 2021040000 #ifndef INF_ENGINE_RELEASE -#warning("IE version have not been provided via command-line. Using 2021.3 by default") -#define INF_ENGINE_RELEASE INF_ENGINE_RELEASE_2021_3 +#warning("IE version have not been provided via command-line. Using 2021.4 by default") +#define INF_ENGINE_RELEASE INF_ENGINE_RELEASE_2021_4 #endif #define INF_ENGINE_VER_MAJOR_GT(ver) (((INF_ENGINE_RELEASE) / 10000) > ((ver) / 10000)) diff --git a/modules/dnn/src/opencl/conv_layer_spatial.cl b/modules/dnn/src/opencl/conv_layer_spatial.cl index 236e8d029a..c9ddacfb8e 100644 --- a/modules/dnn/src/opencl/conv_layer_spatial.cl +++ b/modules/dnn/src/opencl/conv_layer_spatial.cl @@ -74,18 +74,22 @@ (_dst_)[(_offset_)] = ACTIVATION_RELU_FUNCTION(_x_, _channel_); \ } while(0) #define ELTWISE_DATA_ARG __global Dtype* eltwise_data, +#define ELTWISE_DATA_ARG_WITH_OFFSET __global Dtype* eltwise_ptr, int eltwise_offset, #else #define ACTIVATION_FUNCTION(_dst_, _offset_, _data_, _channel_) do { \ const Dtype _x_ = (_data_); \ (_dst_)[(_offset_)] = ACTIVATION_RELU_FUNCTION(_x_, _channel_); \ } while(0) #define ELTWISE_DATA_ARG +#define ELTWISE_DATA_ARG_WITH_OFFSET #endif #if APPLY_BIAS #define BIAS_KERNEL_ARG __global Dtype * biases_base, +#define BIAS_KERNEL_ARG_WITH_OFFSET __global Dtype * biases_base_ptr, int biases_base_offset, #else #define BIAS_KERNEL_ARG +#define BIAS_KERNEL_ARG_WITH_OFFSET #endif #define __CAT(x, y) x##y @@ -154,22 +158,18 @@ __kernel void ConvolveBasic( ) { __global Dtype* convolved_image = convolved_image_base + convolved_image_base_offset; - const int outputX = get_global_id(0); - const int outputY = get_global_id(1); - const int kernelNum = get_global_id(2) * ZPAR; - if (outputX < output_width && outputY < output_height) + const int out_idx = get_global_id(0); // 1D task layout: [output_width * output_height * OUTPUT_Z] + const int plane_size = output_width * output_height; + const int out_plane_idx = out_idx % plane_size; + const int outputZ = out_idx / plane_size; // kernelNum + const int outputY = out_plane_idx / output_width; + const int outputX = out_plane_idx % output_width; + if (outputZ < OUTPUT_Z) { - Dtype sum[ZPAR]; - for (int kern = 0; kern < ZPAR; kern++) - { - sum[kern] = 0.0f; - } + Dtype sum = 0.0f; const int org_y = outputY * STRIDE_Y - pad_h; const int org_x = outputX * STRIDE_X - pad_w; - const int currentKernelOffset = kernel_offset + kernelNum*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS; -#if APPLY_BIAS - const int biasIndex = bias_offset + kernelNum; -#endif + const int currentKernelOffset = kernel_offset + outputZ*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS; const int local_image_offset = org_y * input_width + org_x; const int imageSize = input_width * input_height; __global Dtype* image_dataPtr = (image_data + (image_offset + local_image_offset)); @@ -178,17 +178,13 @@ __kernel void ConvolveBasic( { for (int y = 0; y < KERNEL_HEIGHT; y++) { + int y_ = org_y + y * DILATION_Y; for (int x = 0; x < KERNEL_WIDTH; x++) { - int y_ = org_y + y * DILATION_Y; int x_ = org_x + x * DILATION_X; - if (!(y_ >= 0 && y_ < input_height && x_ >= 0 && x_ < input_width)) + if (y_ >= 0 && y_ < input_height && x_ >= 0 && x_ < input_width) { - continue; - } - for (int kern = 0; kern < ZPAR; kern++) - { - sum[kern] += image_dataPtr[x * DILATION_X] * kernel_dataPtr[kern*KERNEL_HEIGHT*KERNEL_WIDTH*CHANNELS + x]; + sum = mad(image_dataPtr[x * DILATION_X], kernel_dataPtr[x], sum); } } image_dataPtr += input_width * DILATION_Y; @@ -197,18 +193,13 @@ __kernel void ConvolveBasic( image_dataPtr += imageSize - input_width*KERNEL_HEIGHT*DILATION_Y; } - for (int kern = 0; kern < ZPAR; kern++) - { - if (kernelNum + kern < OUTPUT_Z) - { - int offset = convolved_image_offset + (kernelNum+kern)*output_height*output_width + outputY*output_width + outputX; + int offset = convolved_image_offset + out_idx; #if APPLY_BIAS - ACTIVATION_FUNCTION(convolved_image, offset, sum[kern] + bias[biasIndex + kern], biasIndex + kern); + int biasIndex = bias_offset + outputZ; + ACTIVATION_FUNCTION(convolved_image, offset, sum + bias[biasIndex], biasIndex); #else - ACTIVATION_FUNCTION(convolved_image, offset, sum[kern], kernelNum + kern); + ACTIVATION_FUNCTION(convolved_image, offset, sum, outputZ); #endif - } - } } } @@ -223,19 +214,28 @@ __attribute__((reqd_work_group_size(1, 1, SIMD_SIZE))) __attribute__((intel_reqd_sub_group_size(SIMD_SIZE))) __kernel void convolve_simd( - ELTWISE_DATA_ARG + ELTWISE_DATA_ARG_WITH_OFFSET FUSED_ARG - __global Dtype* inputs, - __global Dtype* weights, - BIAS_KERNEL_ARG - __global Dtype* outputs_base, - const int outputs_offset, + __global Dtype* inputs_ptr, const int inputs_offset, + __global Dtype* weights_ptr, const int weights_offset, + BIAS_KERNEL_ARG_WITH_OFFSET + __global Dtype* outputs_base, const int outputs_offset, const ushort input_width, const ushort input_height, const ushort output_width, const ushort output_height) { + __global Dtype* inputs = inputs_ptr + inputs_offset; + __global Dtype* weights = weights_ptr + weights_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype* outputs = outputs_base + outputs_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + unsigned int oc = get_global_id(0) * OUT_BLOCK_WIDTH; // oc = Output Column unsigned int or = get_global_id(1) * OUT_BLOCK_HEIGHT; // or = Output Row unsigned int fm = get_global_id(2); // fm = Feature Map = od = Output Depth @@ -388,13 +388,12 @@ typedef struct half0 { half s0; } half0; //never used but makes compiler happy. #define ROW_PITCH input_width #define GEMM_LIKE_KERNEL_ARGS \ - ELTWISE_DATA_ARG \ + ELTWISE_DATA_ARG_WITH_OFFSET \ FUSED_ARG \ - const __global Dtype *src0, \ - const __global Dtype *src1, \ - BIAS_KERNEL_ARG \ - __global Dtype *dst_base, \ - const int dst_offset, \ + const __global Dtype *src0_ptr, const unsigned int src0_offset, const unsigned int src0_limit, \ + const __global Dtype *src1_ptr, const unsigned int src1_offset, const unsigned int src1_limit, \ + BIAS_KERNEL_ARG_WITH_OFFSET \ + __global Dtype *dst_base, const unsigned int dst_offset, const unsigned int dst_limit, \ const ushort input_width, \ const ushort input_height, \ const ushort output_width, \ @@ -424,7 +423,17 @@ typedef struct half0 { half s0; } half0; //never used but makes compiler happy. __attribute__((intel_reqd_sub_group_size(8))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -447,6 +456,14 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) } typedef CAT( Dtype, KERNEL_WIDTH ) Dtype_t; +// U_GEMM_LIKE_CONV_k11x11_cn3_g1_s4x4_d1x1_b1_in240x240_p0x0_num1_M96_activ1_eltwise0_FP32_5_1_8_32_SIMD8 doesn't run properly (src0_read out of bounds) +// Test: DNNTestNetwork.AlexNet/0 (to run all kernels use OPENCV_OCL4DNN_FORCE_AUTO_TUNING=1) +#if 0 // INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 + #define OPTIMIZE_READ 1 +#else + #define OPTIMIZE_READ 0 +#endif + // True for all threads if filter_width is multiple of TILE_N // else, true for all but right-most column of threads. if( TILE_N_LAST == 0 || global_x < WIDTH1 / TILE_N ) @@ -463,7 +480,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // atile is M rows x K columns. int curr_x = ( global_y % output_width ) * STRIDE_X; int curr_y = ( global_y / output_width ) * STRIDE_Y; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ int saved_y = curr_y; #endif const __global Dtype *src0_read = src0 @@ -483,7 +500,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) do { int patch_row = 0; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ curr_y = saved_y; #endif @@ -501,11 +518,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // ... const bool kernel_width_is_odd = KERNEL_WIDTH % 2 == 1; -#if INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 +#if OPTIMIZE_READ #if KERNEL_WIDTH == 3 Dtype_t blockA00 = vload3(0, src0_read); Dtype* pblockA00 = (Dtype*)(&blockA00); #else + #if 0 // debug + if ((int)(src0_read - src0) >= src0_limit - KERNEL_WIDTH) + { + printf("CATCH: src0_read-src0: %d limit=%d curr_y,curr_x=%d,%d\n", (int)(src0_read - src0), src0_limit, curr_y, curr_x); + } + #endif Dtype_t blockA00 = ( (const __global Dtype_t*)src0_read )[ 0 ]; Dtype* pblockA00 = (Dtype*)(&blockA00); #endif @@ -626,7 +649,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) // atile is M rows x K columns. int curr_x = ( global_y % output_width ) * STRIDE_X; int curr_y = ( global_y / output_width ) * STRIDE_Y; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ int saved_y = curr_y; #endif const __global Dtype *src0_read = src0 @@ -646,14 +669,14 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) do { int patch_row = 0; -#if INPUT_PAD_H != 0 || INPUT_PAD_W != 0 || DILATION_X != 1 || DILATION_Y != 1 || INPUT_PAD_BOTTOM != 0 || INPUT_PAD_RIGHT != 0 +#if !OPTIMIZE_READ curr_y = saved_y; #endif do { // Load atile and interleaved btile. const bool kernel_width_is_odd = KERNEL_WIDTH % 2 == 1; -#if INPUT_PAD_W == 0 && INPUT_PAD_H == 0 && DILATION_X == 1 && DILATION_Y == 1 && INPUT_PAD_BOTTOM == 0 && INPUT_PAD_RIGHT == 0 +#if OPTIMIZE_READ Dtype_t blockA00 = ( (const __global Dtype_t*)src0_read )[ 0 ]; Dtype* pblockA00 = (Dtype*)(&blockA00); #else @@ -790,7 +813,7 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) } } } -#endif +#endif // TILE_N_LAST > 0 } #endif #ifdef GEMM_LIKE_CONV_32_2 @@ -813,7 +836,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(8))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -1375,7 +1408,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(16))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -1561,7 +1604,17 @@ __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) __attribute__((intel_reqd_sub_group_size(16))) __kernel void Conv_Interleaved(GEMM_LIKE_KERNEL_ARGS) { + const __global Dtype *src0 = src0_ptr + src0_offset; + const __global Dtype *src1 = src1_ptr + src1_offset; +#if APPLY_BIAS + __global Dtype* biases_base = biases_base_ptr + biases_base_offset; +#endif + __global Dtype *dst = dst_base + dst_offset; +#ifdef FUSED_CONV_ELTWISE + __global Dtype* eltwise_data = eltwise_ptr + eltwise_offset; +#endif + const int group_x = get_group_id(0); const int group_y = get_group_id(1); const int global_x = get_global_id(0); @@ -1780,10 +1833,13 @@ __kernel void DWCONV( const ushort output_width, const ushort output_height) { __global Dtype* convolved_image = convolved_image_base + convolved_image_offset; - const int outputX = get_global_id(0); - const int outputY = get_global_id(1); - const int outputZ = get_global_id(2); - if(outputX < output_width && outputY < output_height) + const int out_idx = get_global_id(0); // 1D task layout: [output_width * output_height * OUTPUT_Z] + const int plane_size = output_width * output_height; + const int out_plane_idx = out_idx % plane_size; + const int outputZ = out_idx / plane_size; + const int outputY = out_plane_idx / output_width; + const int outputX = out_plane_idx % output_width; + if (outputZ < OUTPUT_Z) { Dtype sum = 0.; diff --git a/modules/dnn/src/opencl/conv_spatial_helper.cl b/modules/dnn/src/opencl/conv_spatial_helper.cl index 33d9db57c8..660d085956 100644 --- a/modules/dnn/src/opencl/conv_spatial_helper.cl +++ b/modules/dnn/src/opencl/conv_spatial_helper.cl @@ -62,8 +62,8 @@ __kernel void TEMPLATE(copyWeightsSwizzled, Dtype) //Original location //Output location - int outputSublayer = channels / swizzleFactor; - int outputSublayerIndex = channels % swizzleFactor; + //int outputSublayer = channels / swizzleFactor; + //int outputSublayerIndex = channels % swizzleFactor; int filter = sX / (kernel_w*kernel_h*channels); int kernel_X = sX % kernel_w; @@ -73,6 +73,10 @@ __kernel void TEMPLATE(copyWeightsSwizzled, Dtype) int FP = filter / swizzleFactor; int F1 = filter % swizzleFactor; - weightOut[FP*(kernel_w*kernel_h*channels*swizzleFactor) + kernel_C*(kernel_w*kernel_h*swizzleFactor) + kernel_Y*(kernel_w*swizzleFactor) + kernel_X*swizzleFactor + F1] - = weightIn[filter*(kernel_w*kernel_h*channels) + kernel_C*(kernel_w*kernel_h) + kernel_Y*kernel_w + kernel_X]; + int idxOut = FP*(kernel_w*kernel_h*channels*swizzleFactor) + kernel_C*(kernel_w*kernel_h*swizzleFactor) + kernel_Y*(kernel_w*swizzleFactor) + kernel_X*swizzleFactor + F1; + int idxIn = filter*(kernel_w*kernel_h*channels) + kernel_C*(kernel_w*kernel_h) + kernel_Y*kernel_w + kernel_X; + + // idxIn is not valid if (filter >= outputs) - no data for these elements. Output alignment gaps are filled by zeros + Dtype v = (filter < outputs) ? weightIn[idxIn] : (Dtype)0; + weightOut[idxOut] = v; } diff --git a/modules/dnn/src/opencl/gemm_image.cl b/modules/dnn/src/opencl/gemm_image.cl index 710637a093..f6e0020d82 100644 --- a/modules/dnn/src/opencl/gemm_image.cl +++ b/modules/dnn/src/opencl/gemm_image.cl @@ -954,6 +954,10 @@ __kernel void TEMPLATE(gemm_buffer_copy_image_transpose, Dtype)( { const int gidx = get_global_id(0); const int gidy = get_global_id(1); + + if (gidx >= width || gidy >= height) + return; + int2 coord_dst = (int2)(gidx, gidy); __global Dtype* A_off = A + offA; Dtype srcA = A_off[gidy * ldA + gidx]; @@ -968,12 +972,18 @@ __kernel void TEMPLATE(gemm_buffer_copy_image_no_transpose, Dtype)( __global Dtype* A, __write_only image2d_t ImA, int offA, + int padded_width, + int padded_height, int width, int height, int ldA) { const int gidx = get_global_id(0); const int gidy = get_global_id(1); + + if (gidx >= padded_width || gidy >= padded_height) + return; + int2 coord_dst = (int2)(gidx, gidy); #if TYPE == TYPE_HALF if (gidx >= width || gidy >= height) { diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp index c35d7dc5ec..4c856592ab 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.cpp @@ -19,6 +19,16 @@ CV__DNN_INLINE_NS_BEGIN using ::google::protobuf::RepeatedField; using ::google::protobuf::MapPair; +static Mat getTensorContentRef_(const tensorflow::TensorProto& tensor); +static inline +bool isAlignedMat(const Mat& m) +{ + int depth = m.depth(); + int alignment = CV_ELEM_SIZE1(depth); + return (((size_t)m.data) & (alignment - 1)) == 0; +} + + class TFNodeWrapper : public ImportNodeWrapper { public: @@ -719,8 +729,19 @@ public: { if (!negativeScales) { - Mat scales = getTensorContent(inputNodes[1]->attr().at("value").tensor(), /*copy*/false); - scales *= -1; + Mat scalesRef = getTensorContentRef_(inputNodes[1]->attr().at("value").tensor()); + // FIXME: This breaks the const guarantees of tensor() by writing to scalesRef + if (isAlignedMat(scalesRef)) + { + scalesRef *= -1; + } + else + { + Mat scales = scalesRef.clone() * -1; + CV_Assert(scalesRef.isContinuous()); + CV_Assert(scales.isContinuous()); + memcpy(scalesRef.data, scales.data, scales.total() * scales.elemSize()); + } } } @@ -832,7 +853,8 @@ void RemoveIdentityOps(tensorflow::GraphDef& net) } } -Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy) +// NB: returned Mat::data pointer may be unaligned +Mat getTensorContentRef_(const tensorflow::TensorProto& tensor) { const std::string& content = tensor.tensor_content(); Mat m; @@ -904,7 +926,18 @@ Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy) CV_Error(Error::StsError, "Tensor's data type is not supported"); break; } - return copy ? m.clone() : m; + + return m; +} + +Mat getTensorContent(const tensorflow::TensorProto& tensor, bool forceCopy) +{ + // If necessary clone m to have aligned data pointer + Mat m = getTensorContentRef_(tensor); + if (forceCopy || !isAlignedMat(m)) + return m.clone(); + else + return m; } void releaseTensor(tensorflow::TensorProto* tensor) diff --git a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp index 8a77dda6d4..2b7ba86251 100644 --- a/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp +++ b/modules/dnn/src/tensorflow/tf_graph_simplifier.hpp @@ -21,7 +21,7 @@ void RemoveIdentityOps(tensorflow::GraphDef& net); void simplifySubgraphs(tensorflow::GraphDef& net); -Mat getTensorContent(const tensorflow::TensorProto &tensor, bool copy = true); +Mat getTensorContent(const tensorflow::TensorProto& tensor, bool forceCopy = true); void releaseTensor(tensorflow::TensorProto* tensor); diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 65695b8504..4010c5ef28 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -32,6 +32,8 @@ namespace cv { namespace dnn { CV__DNN_INLINE_NS_BEGIN +extern bool DNN_DIAGNOSTICS_RUN; + #if HAVE_PROTOBUF using ::google::protobuf::RepeatedField; @@ -122,8 +124,10 @@ void parseTensor(const tensorflow::TensorProto &tensor, Mat &dstBlob) } dstBlob.create(shape, CV_32F); + CV_Assert(dstBlob.isContinuous()); Mat tensorContent = getTensorContent(tensor, /*no copy*/false); + CV_Assert(tensorContent.isContinuous()); int size = tensorContent.total(); CV_Assert(size == (int)dstBlob.total()); @@ -404,12 +408,53 @@ void setKSize(LayerParams &layerParams, const tensorflow::NodeDef &layer) } } -void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer) +void setPadMode(LayerParams &layerParams, const tensorflow::NodeDef &layer) { if (hasLayerAttr(layer, "padding")) layerParams.set("pad_mode", getLayerAttr(layer, "padding").s()); } +bool getExplicitPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, int64_t (&pads)[8]) +{ + if (!layerParams.has("pad_mode") || + layerParams.get("pad_mode").getStringValue() != "EXPLICIT") + { + return false; + } + + CV_Assert(hasLayerAttr(layer, "explicit_paddings")); + + const tensorflow::AttrValue& protoPads = getLayerAttr(layer, "explicit_paddings"); + if (protoPads.list().i_size() != 8) + { + CV_Error(Error::StsNotImplemented, "Unsupported asymmetric padding configuration."); + } + + int n = sizeof(pads) / sizeof(pads[0]); + for (int i = 0; i < n; ++i) + { + pads[i] = protoPads.list().i(i); + } + + if (getDataLayout(layer) != DATA_LAYOUT_NCHW) + { + CV_LOG_DEBUG(NULL, "DNN/TF: Data format " << getLayerAttr(layer, "data_format").s() << ", assuming NHWC."); + // Perhaps, we have NHWC padding dimensions order. + // N H W C + // 0 1 2 3 4 5 6 7 + std::swap(pads[2], pads[6]); + std::swap(pads[3], pads[7]); + // N C W H + // 0 1 2 3 4 5 6 7 + std::swap(pads[4], pads[6]); + std::swap(pads[5], pads[7]); + // N C H W + // 0 1 2 3 4 5 6 7 + } + + return true; +} + Pin parsePin(const std::string &name) { Pin pin(name); @@ -464,6 +509,8 @@ void ExcludeLayer(tensorflow::GraphDef& net, const int layer_index, const int in net.mutable_node()->DeleteSubrange(layer_index, 1); } +class TFLayerHandler; + class TFImporter { public: @@ -471,6 +518,7 @@ public: TFImporter(Net& net, const char *dataModel, size_t lenModel, const char *dataConfig = NULL, size_t lenConfig = 0); protected: + std::unique_ptr layerHandler; Net& dstNet; void populateNet(); @@ -509,11 +557,2055 @@ protected: std::map layer_id; private: - void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId); + void addPermuteLayer(const int* order, const std::string& permName, Pin& inpId, int orderSize = 4); + void setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value = 0.); + + friend class TFLayerHandler; + typedef void (TFImporter::*TFImporterNodeParser)(tensorflow::GraphDef&, const tensorflow::NodeDef&, LayerParams&); + typedef std::map DispatchMap; + + const DispatchMap dispatch; + static const DispatchMap buildDispatchMap(); + + void parseConvolution (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseBias (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMatMul (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseReshape (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseFlatten (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseTranspose (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConstant (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseLrn (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConcat (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMaxPool (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseAvgPool (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMaxPoolGrad (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePlaceholder (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSplit (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSlice (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseStridedSlice (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMul (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseFusedBatchNorm (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseConv2DBackpropInput(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseBlockLSTM (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseResize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseL2Normalize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePriorBox (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseSoftmax (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseCropAndResize (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseMean (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePack (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseClipByValue (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseLeakyRelu (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseActivation (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parseExpandDims (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + + void parseCustomLayer (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); }; +void TFImporter::setPadding(LayerParams &layerParams, const tensorflow::NodeDef &layer, std::string& inputName, float value) +{ + setPadMode(layerParams, layer); + int64_t pads[8]; + + if (!getExplicitPadding(layerParams, layer, pads)) + { + return; + } + + LayerParams padLp; + padLp.name = layer.name() + "/pad"; + padLp.type = "Padding"; + padLp.set("paddings", DictValue::arrayInt(pads, sizeof(pads) / sizeof(pads[0]))); + padLp.set("value", value); + + int id = dstNet.addLayer(padLp.name, padLp.type, padLp); + layer_id[padLp.name] = id; + + connect(layer_id, dstNet, parsePin(inputName), id, 0); + inputName = padLp.name; + + layerParams.set("pad_mode", "VALID"); +} + +class TFLayerHandler : public detail::LayerHandler +{ +public: + explicit TFLayerHandler(TFImporter* importer_); + + void fillRegistry(const tensorflow::GraphDef& net); + bool handleMissing(const tensorflow::NodeDef& layer); + void handleFailed(const tensorflow::NodeDef& layer); + +protected: + TFImporter* importer; +}; + +const TFImporter::DispatchMap TFImporter::buildDispatchMap() +{ + static DispatchMap dispatch; + dispatch["Conv2D"] = dispatch["SpaceToBatchND"] = dispatch["DepthwiseConv2dNative"] = + dispatch["Pad"] = dispatch["MirrorPad"] = dispatch["Conv3D"] = &TFImporter::parseConvolution; + dispatch["BiasAdd"] = dispatch["Add"] = dispatch["AddV2"] = dispatch["Sub"] = dispatch["AddN"] = &TFImporter::parseBias; + dispatch["MatMul"] = &TFImporter::parseMatMul; + dispatch["Reshape"] = &TFImporter::parseReshape; + dispatch["Flatten"] = dispatch["Squeeze"] = &TFImporter::parseFlatten; + dispatch["Transpose"] = &TFImporter::parseTranspose; + dispatch["Const"] = &TFImporter::parseConstant; + dispatch["LRN"] = &TFImporter::parseLrn; + dispatch["Concat"] = dispatch["ConcatV2"] = &TFImporter::parseConcat; + dispatch["MaxPool"] = dispatch["MaxPool3D"] = &TFImporter::parseMaxPool; + dispatch["AvgPool"] = dispatch["AvgPool3D"] = &TFImporter::parseAvgPool; + dispatch["MaxPoolGrad"] = &TFImporter::parseMaxPoolGrad; + dispatch["Placeholder"] = &TFImporter::parsePlaceholder; + dispatch["Split"] = &TFImporter::parseSplit; + dispatch["Slice"] = &TFImporter::parseSlice; + dispatch["StridedSlice"] = &TFImporter::parseStridedSlice; + dispatch["Mul"] = dispatch["RealDiv"] = &TFImporter::parseMul; + dispatch["FusedBatchNorm"] = dispatch["FusedBatchNormV3"] = &TFImporter::parseFusedBatchNorm; + dispatch["Conv2DBackpropInput"] = &TFImporter::parseConv2DBackpropInput; + dispatch["BlockLSTM"] = &TFImporter::parseBlockLSTM; + dispatch["ResizeNearestNeighbor"] = dispatch["ResizeBilinear"] = dispatch["FusedResizeAndPadConv2D"] = &TFImporter::parseResize; + dispatch["L2Normalize"] = &TFImporter::parseL2Normalize; + dispatch["PriorBox"] = &TFImporter::parsePriorBox; + dispatch["Softmax"] = &TFImporter::parseSoftmax; + dispatch["CropAndResize"] = &TFImporter::parseCropAndResize; + dispatch["Mean"] = dispatch["Sum"] = dispatch["Max"] = &TFImporter::parseMean; + dispatch["Pack"] = &TFImporter::parsePack; + dispatch["ClipByValue"] = &TFImporter::parseClipByValue; + dispatch["LeakyRelu"] = &TFImporter::parseLeakyRelu; + dispatch["Abs"] = dispatch["Tanh"] = dispatch["Sigmoid"] = dispatch["Relu"] = + dispatch["Elu"] = dispatch["Exp"] = dispatch["Identity"] = dispatch["Relu6"] = &TFImporter::parseActivation; + dispatch["ExpandDims"] = &TFImporter::parseExpandDims; + + return dispatch; +} + +// "Conv2D" "SpaceToBatchND" "DepthwiseConv2dNative" "Pad" "MirrorPad" "Conv3D" +void TFImporter::parseConvolution(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) +{ + tensorflow::NodeDef layer = layer_; + std::string name = layer.name(); + std::string type = layer.op(); + int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + // The first node of dilated convolution subgraph. + // Extract input node, dilation rate and paddings. + std::string input = layer.input(0); + StrIntVector next_layers; + if (type == "SpaceToBatchND" || type == "Pad") + { + next_layers = getNextLayers(net, name, "Conv2D"); + if (next_layers.empty()) + next_layers = getNextLayers(net, name, "DepthwiseConv2dNative"); + } + + if (type == "SpaceToBatchND") + { + // op: "SpaceToBatchND" + // input: "input" + // input: "SpaceToBatchND/block_shape" + // input: "SpaceToBatchND/paddings" + CV_CheckEQ(num_inputs, 3, ""); + + DictValue dilation = parseDims(getConstBlob(layer, value_id, 1)); + CV_Assert(dilation.size() == 2); + layerParams.set("dilation_h", dilation.get(0)); + layerParams.set("dilation_w", dilation.get(1)); + + Mat paddings; + parseTensor(getConstBlob(layer, value_id, 2), paddings); + + // paddings is a 2x2 matrix: [[top, bot], [left, right]] + layerParams.set("pad_h", paddings.at(0)); + layerParams.set("pad_w", paddings.at(2)); + + CV_Assert(next_layers.size() == 1); + layers_to_ignore.insert(next_layers[0].first); + + // FIXIT don't override, rewrite this code + layer = net.node(next_layers[0].second); + name = layer.name(); + type = layer.op(); + num_inputs = layer.input_size(); + CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); + } + else if (type == "Pad" || type == "MirrorPad") + { + Mat paddings = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(paddings.type() == CV_32SC1); + if (paddings.total() == 8) + { + // Perhaps, we have NHWC padding dimensions order. + // N H W C + // 0 1 2 3 4 5 6 7 + std::swap(paddings.at(2), paddings.at(6)); + std::swap(paddings.at(3), paddings.at(7)); + // N C W H + // 0 1 2 3 4 5 6 7 + std::swap(paddings.at(4), paddings.at(6)); + std::swap(paddings.at(5), paddings.at(7)); + // N C H W + // 0 1 2 3 4 5 6 7 + } + + if (next_layers.empty() || paddings.total() != 8 || + paddings.at(4) != paddings.at(5) || + paddings.at(6) != paddings.at(7) || type == "MirrorPad") + { + // Just a single padding layer. + layerParams.set("paddings", DictValue::arrayInt((int*)paddings.data, paddings.total())); + if (type == "MirrorPad") + layerParams.set("type", "reflect"); + + int id = dstNet.addLayer(name, "Padding", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(input), id, 0); + return; + } + else + { + // Merge with subsequent convolutional layer. + CV_Assert(next_layers.size() == 1); + + layerParams.set("pad_h", paddings.at(4)); + layerParams.set("pad_w", paddings.at(6)); + + layers_to_ignore.insert(next_layers[0].first); + + // FIXIT don't override, rewrite this code + layer = net.node(next_layers[0].second); + name = layer.name(); + type = layer.op(); + num_inputs = layer.input_size(); + CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); + } + } + + // For the object detection networks, TensorFlow Object Detection API + // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) + // order. We can manage it at DetectionOutput layer parsing predictions + // or shuffle last convolution's weights. + bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && + getLayerAttr(layer, "loc_pred_transposed").b(); + + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); + + next_layers = getNextLayers(net, name, "BiasAdd"); + if (next_layers.size() == 1) { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + + // Shuffle bias from yxYX to xyXY. + if (locPredTransposed) + { + const int numWeights = layerParams.blobs[1].total(); + float* biasData = reinterpret_cast(layerParams.blobs[1].data); + CV_Assert(numWeights % 4 == 0); + for (int i = 0; i < numWeights; i += 2) + { + std::swap(biasData[i], biasData[i + 1]); + } + } + } + + int kernelTensorInpId = -1; + const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernelTensorInpId); + const String kernelTensorName = layer.input(kernelTensorInpId); + std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); + if (sharedWeightsIt == sharedWeights.end()) + { + kernelFromTensor(kernelTensor, layerParams.blobs[0]); + releaseTensor(const_cast(&kernelTensor)); + + int* kshape = layerParams.blobs[0].size.p; + const int outCh = kshape[0]; + const int inCh = kshape[1]; + const int height = kshape[2]; + const int width = kshape[3]; + if (type == "DepthwiseConv2dNative") + { + CV_Assert(!locPredTransposed); + const int chMultiplier = kshape[0]; + + Mat copy = layerParams.blobs[0].clone(); + float* src = (float*)copy.data; + float* dst = (float*)layerParams.blobs[0].data; + for (int i = 0; i < chMultiplier; ++i) + for (int j = 0; j < inCh; ++j) + for (int s = 0; s < height * width; ++s) + { + int src_i = (i * inCh + j) * height * width + s; + int dst_i = (j * chMultiplier + i) * height* width + s; + dst[dst_i] = src[src_i]; + } + // TODO Use reshape instead + kshape[0] = inCh * chMultiplier; + kshape[1] = 1; + size_t* kstep = layerParams.blobs[0].step.p; + kstep[0] = kstep[1]; // fix steps too + } + + // Shuffle output channels from yxYX to xyXY. + if (locPredTransposed) + { + const int slice = height * width * inCh; + for (int i = 0; i < outCh; i += 2) + { + cv::Mat src(1, slice, CV_32F, layerParams.blobs[0].ptr(i)); + cv::Mat dst(1, slice, CV_32F, layerParams.blobs[0].ptr(i + 1)); + std::swap_ranges(src.begin(), src.end(), dst.begin()); + } + } + sharedWeights[kernelTensorName] = layerParams.blobs[0]; + } + else + { + layerParams.blobs[0] = sharedWeightsIt->second; + } + Mat weights = layerParams.blobs[0]; + layerParams.set("kernel_size", DictValue::arrayInt(&weights.size[2], weights.dims - 2)); + + layerParams.set("num_output", layerParams.blobs[0].size[0]); + + setStrides(layerParams, layer); + if (!layerParams.has("pad_w") && !layerParams.has("pad_h")) + setPadding(layerParams, layer, input); + + // The final node of dilated convolution subgraph. + next_layers = getNextLayers(net, name, "BatchToSpaceND"); + if (!next_layers.empty()) + { + CV_Assert(next_layers.size() == 1); + ExcludeLayer(net, next_layers[0].second, 0, false); + layers_to_ignore.insert(next_layers[0].first); + } + + int id = dstNet.addLayer(name, "Convolution", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(input), id, 0); + + + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_UNKNOWN) + data_layouts[name] = DATA_LAYOUT_NHWC; +} + +// "BiasAdd" "Add" "AddV2" "Sub" "AddN" +void TFImporter::parseBias(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + bool haveConst = false; + for(int ii = 0; !haveConst && ii < num_inputs; ++ii) + { + Pin input = parsePin(layer.input(ii)); + haveConst = value_id.find(input.name) != value_id.end(); + } + CV_Assert(!haveConst || num_inputs == 2); + + if (haveConst) + { + Mat values = getTensorContent(getConstBlob(layer, value_id)); + CV_Assert(values.type() == CV_32FC1); + if (type == "Sub") + values *= -1.0f; + + int id; + if (values.total() == 1) // is a scalar. + { + layerParams.set("shift", values.at(0)); + id = dstNet.addLayer(name, "Power", layerParams); + } + else // is a vector + { + layerParams.blobs.resize(1, values); + id = dstNet.addLayer(name, "Shift", layerParams); + } + layer_id[name] = id; + + // one input only + Pin inp0 = parsePin(layer.input(0)); + if (layer_id.find(inp0.name) != layer_id.end()) + // First operand is a constant. + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + else + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + } + else + { + layerParams.set("operation", "sum"); + if (type == "Sub") + { + static float subCoeffs[] = {1.f, -1.f}; + layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); + } + + int id = dstNet.addLayer(name, "Eltwise", layerParams); + layer_id[name] = id; + + for (int ii = 0; ii < num_inputs; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii); + } + } +} + +void TFImporter::parseMatMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + + // For the object detection networks, TensorFlow Object Detection API + // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) + // order. We can manage it at DetectionOutput layer parsing predictions + // or shuffle last Faster-RCNN's matmul weights. + bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && + getLayerAttr(layer, "loc_pred_transposed").b(); + + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); + + StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); // FIXIT Use layers fusion instead + if (next_layers.empty()) + { + next_layers = getNextLayers(net, name, "Add"); + } + if (next_layers.size() == 1) { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + + if (locPredTransposed) + { + const int numWeights = layerParams.blobs[1].total(); + float* biasData = reinterpret_cast(layerParams.blobs[1].data); + CV_Assert(numWeights % 4 == 0); + for (int i = 0; i < numWeights; i += 2) + { + std::swap(biasData[i], biasData[i + 1]); + } + } + } + + int kernel_blob_index = -1; + const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernel_blob_index); + const String kernelTensorName = layer.input(kernel_blob_index); + std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); + if (sharedWeightsIt == sharedWeights.end()) + { + blobFromTensor(kernelTensor, layerParams.blobs[0]); + releaseTensor(const_cast(&kernelTensor)); + sharedWeights[kernelTensorName] = layerParams.blobs[0]; + } + else + { + layerParams.blobs[0] = sharedWeightsIt->second; + } + + if (kernel_blob_index == 1) { // In this case output is computed by x*W formula - W should be transposed + Mat data = layerParams.blobs[0].t(); + layerParams.blobs[0] = data.clone(); + } + + layerParams.set("num_output", layerParams.blobs[0].size[0]); + if (locPredTransposed) + { + CV_Assert(layerParams.blobs[0].dims == 2); + for (int i = 0; i < layerParams.blobs[0].size[0]; i += 2) + { + cv::Mat src = layerParams.blobs[0].row(i); + cv::Mat dst = layerParams.blobs[0].row(i + 1); + std::swap_ranges(src.begin(), src.end(), dst.begin()); + } + } + + int id = dstNet.addLayer(name, "InnerProduct", layerParams); + layer_id[name] = id; + + // one input only + int input_blob_index = kernel_blob_index == 0 ? 1 : 0; + connect(layer_id, dstNet, parsePin(layer.input(input_blob_index)), id, 0); + data_layouts[name] = DATA_LAYOUT_PLANAR; +} + +void TFImporter::parseReshape(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); + // There are two possible implementations: reshape an input using + // predefined sizes or use a second input blob as a source of new shape. + if (value_id.find(layer.input(1)) != value_id.end()) + { + Mat newShape = getTensorContent(getConstBlob(layer, value_id, 1)); + int newShapeSize = newShape.total(); + bool hasSwap = false; + if (newShapeSize == 4 && hasAllOnes(newShape, 0, 2)) + { + // NHWC->NCHW + std::swap(*newShape.ptr(0, 2), *newShape.ptr(0, 3)); + std::swap(*newShape.ptr(0, 1), *newShape.ptr(0, 2)); + hasSwap = true; + } + if (inpLayout == DATA_LAYOUT_NHWC) + { + if (newShapeSize >= 2 || newShape.at(1) == 1) + { + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + addPermuteLayer(order, name + "/nhwc", inpId); + if (newShapeSize < 4) + { + inpLayout = DATA_LAYOUT_NCHW; + } + else + { + inpLayout = DATA_LAYOUT_NHWC; + } + } + } + layerParams.set("dim", DictValue::arrayInt(newShape.ptr(), newShapeSize)); + + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, inpId, id, 0); + inpId = Pin(name); + + if ((inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_UNKNOWN || inpLayout == DATA_LAYOUT_PLANAR) && + newShapeSize == 4 && !hasSwap) + { + int order[] = {0, 3, 1, 2}; // Transform back to OpenCV's NCHW. + addPermuteLayer(order, name + "/nchw", inpId); + inpLayout = DATA_LAYOUT_NCHW; + } + + data_layouts[name] = newShapeSize == 2 ? DATA_LAYOUT_PLANAR : inpLayout; + } + else + { + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + data_layouts[name] = inpLayout; + } +} + +void TFImporter::parseExpandDims(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_Assert(!netInputShapes.empty()); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); + + // Get input shape + std::vector inShape_, outShape_; + int inpIdindex = layer_id.find(inpId.name)->second; + + dstNet.getLayerShapes(netInputShapes, inpIdindex, inShape_, outShape_); + MatShape inpShape = outShape_[0]; + std::vector outShape = inpShape; + + int outShapeSize = outShape.size(); + + CV_Assert(inpShape.size() >= 1); + // 2nd blob is dims tensor + int axis = getConstBlob(layer, value_id, 1).int_val().Get(0); + + // Convert negative numbers to positive numbers, axis can be in range [-(D+1), D]. + if(axis < 0) + { + axis = inpShape.size() + axis + 1; + } + + CV_Assert(0 <= axis && axis <= inpShape.size()); + + // After ExpendDims, 3-dim data will become 4-dim data, and OpenCV retains 4-dim data as NCHW data layout. + // Convert OpenCV's NHC to NCH first. + if(outShapeSize == 3) + { + // If axis equal to outShapeSize, that mean we expand in Channel dimmension, and do not add permuteLayer. + if(axis != outShapeSize) + { + int order[] = {0, 2, 1}; // From OpenCV's NHC to NCH. + addPermuteLayer(order, name + "/nch", inpId, 3); + + std::swap(outShape[1], outShape[2]); + } + axis = (axis != 0)?(axis % outShapeSize + 1):2; + } + + if(inpShape.size() == 4) + { + if(axis == inpShape.size()) + { + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + addPermuteLayer(order, name + "/nhwc", inpId); + + // Convert shape From OpenCV's NCHW to NHWC. + if(inpLayout == DATA_LAYOUT_NHWC) + { + std::swap(outShape[1], outShape[2]); + std::swap(outShape[2], outShape[3]); + } + } + if(inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_NCHW) + { + // toNCHW + axis = (axis != 0)?(axis % outShapeSize + 1):0; + } + } + + // After ExpendDims, 5-dim data will become 6-dim data, and OpenCV retains 6-dim data as original data layout. + // Convert OpenCV's NCDHW to NDHWC first. + if (inpShape.size() == 5 && (inpLayout == DATA_LAYOUT_NDHWC || inpLayout == DATA_LAYOUT_UNKNOWN)) + { + int order[] = {0, 2, 3, 4, 1}; // From OpenCV's NCDHW to NDHWC. + addPermuteLayer(order, name + "/ndhwc", inpId, 5); + + // Convert shape From OpenCV's NCDHW to NDHWC. + if(inpLayout == DATA_LAYOUT_NDHWC) + { + std::swap(outShape[1], outShape[2]); + std::swap(outShape[2], outShape[3]); + std::swap(outShape[3], outShape[4]); + } + } + + outShape.insert(outShape.begin() + axis, 1); + outShapeSize += 1; + + // From OpenCV's NCDHW to NDHWC. + if((inpLayout != DATA_LAYOUT_NHWC && inpLayout != DATA_LAYOUT_NCHW) && outShapeSize == 5) + { + for(int i = 1; i < outShapeSize - 1; i++) + { + std::swap(outShape[outShapeSize - i - 1], outShape[outShapeSize - i]); + } + } + + layerParams.set("dim", DictValue::arrayInt(&outShape[0], outShape.size())); + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, inpId, id, 0); + + if(outShapeSize == 5) + { + data_layouts[name] = DATA_LAYOUT_NDHWC; + } + else if(outShapeSize == 4) + { + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else + { + data_layouts[name] = inpLayout; + } +} + +// "Flatten" "Squeeze" +void TFImporter::parseFlatten(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Pin inpId = parsePin(layer.input(0)); + int inpLayout = getDataLayout(layer.input(0), data_layouts); + if (type == "Squeeze") + { + CV_Assert(hasLayerAttr(layer, "squeeze_dims")); + const tensorflow::AttrValue& dims = getLayerAttr(layer, "squeeze_dims"); + std::vector dimsVector(dims.list().i_size()); + for (int i = 0; i < dimsVector.size(); ++i) + dimsVector[i] = dims.list().i(i); + + // Flatten layer can squeeze dimensions range into one. + std::sort(dimsVector.begin(), dimsVector.end()); + for (int i = 1; i < dimsVector.size(); ++i) + { + if (dimsVector[i] != dimsVector[i - 1] + 1) + CV_Error(Error::StsNotImplemented, "Unsupported squeeze configuration"); + } + int start = dimsVector.front() - 1, end = dimsVector.back(); + if (start == -1 && end == 0) // squeeze 0th dimension + { + start = 0; + end = 1; + } + layerParams.set("axis", start); + layerParams.set("end_axis", end); + } + if (inpLayout == DATA_LAYOUT_NHWC) + { + LayerParams permLP; + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + permLP.set("order", DictValue::arrayInt(order, 4)); + + std::string permName = name + "/nchw"; + CV_Assert(layer_id.find(permName) == layer_id.end()); + int permId = dstNet.addLayer(permName, "Permute", permLP); + layer_id[permName] = permId; + connect(layer_id, dstNet, inpId, permId, 0); + inpId = Pin(permName); + } + int id = dstNet.addLayer(name, "Flatten", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); + data_layouts[name] = DATA_LAYOUT_PLANAR; +} + +void TFImporter::parseTranspose(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + Mat perm = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(perm.type() == CV_32SC1); + int* permData = (int*)perm.data; + if (perm.total() == 4) + { + // Only NHWC <-> NCHW permutations are allowed. OpenCV is always + // keep NCHW layout this way. + int inpLayout = getDataLayout(layer.input(0), data_layouts); + std::string type = "Identity"; + if (inpLayout == DATA_LAYOUT_NHWC) + { + if (permData[0] == 0 && permData[1] == 3 && permData[2] == 1 && permData[3] == 2) + { + // in TensorFlow: NHWC->NCHW + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) + { + // in TensorFlow: NHWC->NHWC + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NHWC; + } + else if (permData[0] == 0 && permData[1] == 3 && permData[2] == 2 && permData[3] == 1) + { + // in TensorFlow: NHWC->NCWH + // in OpenCV: NCHW->NCWH + int permData[] = {0, 1, 3, 2}; + layerParams.set("order", DictValue::arrayInt(permData, perm.total())); + data_layouts[name] = DATA_LAYOUT_NCHW; // we keep track NCHW because channels position only matters + type = "Permute"; + } + else + CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); + } + else if (inpLayout == DATA_LAYOUT_NCHW) + { + if (permData[0] == 0 && permData[1] == 2 && permData[2] == 3 && permData[3] == 1) + { + // in TensorFlow: NCHW->NHWC + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NHWC; + } + else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) + { + // in TensorFlow: NCHW->NCHW + // in OpenCV: NCHW->NCHW + data_layouts[name] = DATA_LAYOUT_NCHW; + } + else + CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); + } + int id = dstNet.addLayer(name, type, layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + } + else + { + layerParams.set("order", DictValue::arrayInt(permData, perm.total())); + + int id = dstNet.addLayer(name, "Permute", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; + } +} + +void TFImporter::parseConstant(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ +} + +void TFImporter::parseLrn(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + if(hasLayerAttr(layer, "alpha")) { + layerParams.set("alpha", getLayerAttr(layer, "alpha").f()); + } + if(hasLayerAttr(layer, "beta")) { + layerParams.set("beta", getLayerAttr(layer, "beta").f()); + } + if(hasLayerAttr(layer, "depth_radius")) { + int radius = (int)getLayerAttr(layer, "depth_radius").i(); + layerParams.set("local_size", 2*radius + 1); + } + if(hasLayerAttr(layer, "bias")) { + layerParams.set("bias", getLayerAttr(layer, "bias").f()); + } + layerParams.set("norm_by_size", false); + + int id = dstNet.addLayer(name, "LRN", layerParams); + layer_id[name] = id; + + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +// "Concat" "ConcatV2" +void TFImporter::parseConcat(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + int axisId = (type == "Concat" ? 0 : num_inputs - 1); + int axis = getConstBlob(layer, value_id, axisId).int_val().Get(0); + + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + axis = toNCHW(axis); + else if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NDHWC) + axis = toNCDHW(axis); + layerParams.set("axis", axis); + + // input(0) or input(n-1) is concat_dim + int from = (type == "Concat" ? 1 : 0); + int to = (type == "Concat" ? num_inputs : num_inputs - 1); + + for (int ii = from; ii < to; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + { + // There are constant inputs. + LayerParams lp; + lp.name = inp.name; + lp.type = "Const"; + lp.blobs.resize(1); + blobFromTensor(getConstBlob(layer, value_id, ii), lp.blobs.back()); + CV_Assert_N(!lp.blobs[0].empty(), lp.blobs[0].type() == CV_32F); + + int constInpId = dstNet.addLayer(lp.name, lp.type, lp); + layer_id[lp.name] = constInpId; + } + } + + int id = dstNet.addLayer(name, "Concat", layerParams); + layer_id[name] = id; + + for (int ii = from; ii < to; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii - from); + } +} + +// "MaxPool" "MaxPool3D" +void TFImporter::parseMaxPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + std::string inputName = layer.input(0); + + CV_CheckGT(num_inputs, 0, ""); + layerParams.set("pool", "max"); + + setKSize(layerParams, layer); + setStrides(layerParams, layer); + setPadding(layerParams, layer, inputName, -std::numeric_limits::infinity()); + // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU + layerParams.set("ceil_mode", false); + + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + + connectToAllBlobs(layer_id, dstNet, parsePin(inputName), id, num_inputs); +} + +// "AvgPool" "AvgPool3D" +void TFImporter::parseAvgPool(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + layerParams.set("pool", "ave"); + layerParams.set("ave_pool_padded_area", false); + setKSize(layerParams, layer); + setStrides(layerParams, layer); + setPadMode(layerParams, layer); + + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseMaxPoolGrad(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, ""); + + layerParams.set("pool_k_h", 0); + layerParams.set("pool_k_w", 0); + layerParams.set("pool_stride_h", 0); + layerParams.set("pool_stride_w", 0); + layerParams.set("pool_pad_h", 0); + layerParams.set("pool_pad_w", 0); + + int id = dstNet.addLayer(name, "MaxUnpool", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1) + ":1"), id, 1); + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 2); +} + +void TFImporter::parsePlaceholder(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + + DataLayout predictedLayout = data_layouts[name]; + + if (!hasLayerAttr(layer, "dtype") || + getLayerAttr(layer, "dtype").type() != tensorflow::DT_BOOL) // If input is not a train/test flag. + { + netInputsNames.push_back(name); + layer_id[name] = 0; + } + tensorflow::TensorShapeProto shape; + if (hasLayerAttr(layer, "shape")) + shape = getLayerAttr(layer, "shape").shape(); + else if (hasLayerAttr(layer, "_output_shapes")) + { + tensorflow::AttrValue_ListValue list = getLayerAttr(layer, "_output_shapes").list(); + if (list.shape_size()) + shape = list.shape()[0]; + } + if (shape.dim_size()) + { + MatShape dims(shape.dim_size()); + for (int i = 0; i < dims.size(); ++i) + dims[i] = shape.dim(i).size(); + if (dims.size() == 4 && predictedLayout == DATA_LAYOUT_NHWC) + { + std::swap(dims[1], dims[3]); // NHWC->NCWH + std::swap(dims[2], dims[3]); // NCWH->NCHW + if (dims[0] == -1) // It's OK to have undetermined batch size + dims[0] = 1; + } + + if (dims.size() == 5 && predictedLayout == DATA_LAYOUT_NDHWC) + { + std::swap(dims[3], dims[4]); // NDHWC->NDHCW + std::swap(dims[2], dims[3]); // NDHCW->NDCHW + std::swap(dims[1], dims[2]); // NDCHW->NCDHW + if (dims[0] == -1) // It's OK to have undetermined batch size + dims[0] = 1; + } + bool hasNeg = false; + for (int i = 0; i < dims.size() && !hasNeg; ++i) + { + hasNeg = dims[i] < 0; + } + if (!hasNeg) + netInputShapes.push_back(dims); + } +} + +void TFImporter::parseSplit(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // TODO: determining axis index remapping by input dimensions order of input blob + // TODO: slicing input may be Const op + // TODO: slicing kernels for convolutions - in current implementation it is impossible + // TODO: add parsing num of slices parameter + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + // num_split + // 1st blob is dims tensor + int axis = getConstBlob(layer, value_id, 0).int_val().Get(0); + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + axis = toNCHW(axis); + layerParams.set("axis", axis); + + if (hasLayerAttr(layer, "num_split")) + layerParams.set("num_split", getLayerAttr(layer, "num_split").i()); + + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); +} + +void TFImporter::parseSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "Slice" + // input: "input_node" + // input: "Slice/begin" + // input: "Slice/size" + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, ""); + Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat sizes = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_Assert_N(!begins.empty(), !sizes.empty()); + CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); + CV_CheckTypeEQ(sizes.type(), CV_32SC1, ""); + + if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + { + // Swap NHWC parameters' order to NCHW. + std::swap(*begins.ptr(0, 2), *begins.ptr(0, 3)); + std::swap(*begins.ptr(0, 1), *begins.ptr(0, 2)); + std::swap(*sizes.ptr(0, 2), *sizes.ptr(0, 3)); + std::swap(*sizes.ptr(0, 1), *sizes.ptr(0, 2)); + } + layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); + layerParams.set("size", DictValue::arrayInt((int*)sizes.data, sizes.total())); + + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + +void TFImporter::parseStridedSlice(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 4, ""); + Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat ends = getTensorContent(getConstBlob(layer, value_id, 2)); + Mat strides = getTensorContent(getConstBlob(layer, value_id, 3)); + CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); + CV_CheckTypeEQ(ends.type(), CV_32SC1, ""); + CV_CheckTypeEQ(strides.type(), CV_32SC1, ""); + const int num = begins.total(); + CV_Assert_N(num == ends.total(), num == strides.total()); + + int end_mask = getLayerAttr(layer, "end_mask").i(); + for (int i = 0; i < num; ++i) + { + if (ends.at(i) < 0) + ends.at(i) -= 1; + if (end_mask & (1 << i)) + ends.at(i) = -1; + if (strides.at(i) != 1) + CV_Error(Error::StsNotImplemented, + format("StridedSlice with stride %d", strides.at(i))); + } + if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + { + // Swap NHWC parameters' order to NCHW. + std::swap(begins.at(2), begins.at(3)); + std::swap(begins.at(1), begins.at(2)); + std::swap(ends.at(2), ends.at(3)); + std::swap(ends.at(1), ends.at(2)); + } + layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); + layerParams.set("end", DictValue::arrayInt((int*)ends.data, ends.total())); + + int id = dstNet.addLayer(name, "Slice", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + +// "Mul" "RealDiv" +void TFImporter::parseMul(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + int constId = -1; + for(int ii = 0; ii < num_inputs; ++ii) + { + Pin input = parsePin(layer.input(ii)); + if (value_id.find(input.name) != value_id.end()) + { + constId = ii; + break; + } + } + CV_Assert((constId != -1) || (num_inputs == 2)); + + if (constId != -1) + { + // Multiplication by constant. + CV_CheckEQ(num_inputs, 2, ""); + Mat scaleMat = getTensorContent(getConstBlob(layer, value_id)); + CV_Assert(scaleMat.type() == CV_32FC1); + if (type == "RealDiv") + { + if (constId == 0) + CV_Error(Error::StsNotImplemented, "Division of constant over variable"); + scaleMat = 1.0f / scaleMat; + } + + int id; + if (scaleMat.total() == 1) // is a scalar. + { + // Try to match with a LeakyRelu: + // node { + // name: "LeakyRelu/mul" + // op: "Mul" + // input: "LeakyRelu/alpha" + // input: "input" + // } + // node { + // name: "LeakyRelu/Maximum" + // op: "Maximum" + // input: "LeakyRelu/mul" + // input: "input" + // } + StrIntVector next_layers = getNextLayers(net, name, "Maximum"); + if (!next_layers.empty()) + { + int maximumLayerIdx = next_layers[0].second; + + CV_Assert(net.node(maximumLayerIdx).input_size() == 2); + + // The input from the Mul layer can also be at index 1. + int mulInputIdx = (net.node(maximumLayerIdx).input(0) == name) ? 0 : 1; + + ExcludeLayer(net, maximumLayerIdx, mulInputIdx, false); + layers_to_ignore.insert(next_layers[0].first); + + layerParams.set("negative_slope", scaleMat.at(0)); + id = dstNet.addLayer(name, "ReLU", layerParams); + } + else + { + // Just a multiplication. + layerParams.set("scale", scaleMat.at(0)); + id = dstNet.addLayer(name, "Power", layerParams); + } + } + else // is a vector + { + layerParams.blobs.resize(1, scaleMat); + + StrIntVector next_layers = getNextLayers(net, name, "Add"); + if (!next_layers.empty()) + { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs.back()); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + } + + if (hasLayerAttr(layer, "axis")) + layerParams.set("axis", getLayerAttr(layer, "axis").i()); + + id = dstNet.addLayer(name, "Scale", layerParams); + } + layer_id[name] = id; + + Pin inp0 = parsePin(layer.input(0)); + if (layer_id.find(inp0.name) != layer_id.end()) + // First operand is a constant. + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + else + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + } + else + { + // Check if all the inputs have the same shape. + bool equalInpShapes = true; + bool isShapeOnes = false; + MatShape outShape0; + for (int ii = 0; ii < num_inputs && !netInputShapes.empty(); ii++) + { + Pin pin = parsePin(layer.input(ii)); + int inpId = layer_id.find(pin.name)->second; + + // Get input shape + MatShape outShape; + std::vector inpShapes, outShapes; + dstNet.getLayerShapes(netInputShapes, inpId, inpShapes, outShapes); + CV_CheckGT(static_cast(outShapes.size()), pin.blobIndex, ""); + outShape = outShapes[pin.blobIndex]; + + if (ii == 0) + { + outShape0 = outShape; + } + else if (outShape != outShape0) + { + equalInpShapes = false; + isShapeOnes = isAllOnes(outShape, 2, outShape.size()) || + isAllOnes(outShape0, 2, outShape0.size()); + break; + } + } + + int id; + if (equalInpShapes || netInputShapes.empty() || (!equalInpShapes && isShapeOnes)) + { + layerParams.set("operation", type == "RealDiv" ? "div" : "prod"); + id = dstNet.addLayer(name, "Eltwise", layerParams); + } + else + { + if (type == "RealDiv") + CV_Error(Error::StsNotImplemented, "Division of non equal tensors"); + id = dstNet.addLayer(name, "Scale", layerParams); + } + + layer_id[name] = id; + + for (int ii = 0; ii < num_inputs; ii++) + { + Pin inp = parsePin(layer.input(ii)); + if (layer_id.find(inp.name) == layer_id.end()) + CV_Error(Error::StsError, "Input layer not found: " + inp.name); + connect(layer_id, dstNet, inp, id, ii); + } + } +} + +// "FusedBatchNorm" "FusedBatchNormV3" +void TFImporter::parseFusedBatchNorm(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "FusedBatchNorm" + // input: "input" + // input: "BatchNorm/gamma" + // input: "BatchNorm/beta" + // input: "BatchNorm/moving_mean" + // input: "BatchNorm/moving_variance" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 5, "Expected gamma, beta, mean and std"); + Pin inpId = parsePin(layer.input(0)); + + bool isTraining = hasLayerAttr(layer, "is_training") && getLayerAttr(layer, "is_training").b(); + + layerParams.blobs.resize(2); + + const tensorflow::TensorProto& gammaTensor = getConstBlob(layer, value_id, 1); + if (!gammaTensor.tensor_content().empty()) + { + layerParams.blobs.resize(layerParams.blobs.size() + 1); + layerParams.set("has_weight", true); + blobFromTensor(gammaTensor, layerParams.blobs.back()); + } + else + layerParams.set("has_weight", false); + + const tensorflow::TensorProto& betaTensor = getConstBlob(layer, value_id, 2); + if (!betaTensor.tensor_content().empty()) + { + layerParams.blobs.resize(layerParams.blobs.size() + 1); + layerParams.set("has_bias", true); + blobFromTensor(betaTensor, layerParams.blobs.back()); + } + else + layerParams.set("has_bias", false); + + Mat mean, std; + if (isTraining) + { + if (layerParams.blobs.size() == 2) + CV_Error(Error::StsNotImplemented, "Cannot determine number " + "of parameters for batch normalization layer."); + mean = Mat::zeros(1, layerParams.blobs[2].total(), CV_32F); + std = Mat::ones(1, layerParams.blobs[2].total(), CV_32F); + + // Add an extra layer: Mean-Variance normalization + LayerParams mvnParams; + std::string mvnName = name + "/MVN"; + CV_Assert(layer_id.find(mvnName) == layer_id.end()); + int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); + layer_id[mvnName] = mvnId; + connect(layer_id, dstNet, inpId, mvnId, 0); + inpId = Pin(mvnName); + } + else + { + blobFromTensor(getConstBlob(layer, value_id, 3), mean); + blobFromTensor(getConstBlob(layer, value_id, 4), std); + } + layerParams.blobs[0] = mean; + layerParams.blobs[1] = std; + + if (hasLayerAttr(layer, "epsilon")) + layerParams.set("eps", getLayerAttr(layer, "epsilon").f()); + + int id = dstNet.addLayer(name, "BatchNorm", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, inpId, id, 0); +} + +void TFImporter::parseConv2DBackpropInput(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "Conv2DBackpropInput" + // input: "conv2d_transpose/output_shape" + // input: "weights" + // input: "input" + + std::string name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, "Expected output shape, weights and input nodes"); + + layerParams.set("bias_term", false); + layerParams.blobs.resize(1); + + StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); + if (next_layers.size() == 1) + { + layerParams.set("bias_term", true); + layerParams.blobs.resize(2); + + int weights_layer_index = next_layers[0].second; + + blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); + ExcludeLayer(net, weights_layer_index, 0, false); + layers_to_ignore.insert(next_layers[0].first); + } + + kernelFromTensor(getConstBlob(layer, value_id, 1), layerParams.blobs[0]); + + const int* kshape = layerParams.blobs[0].size.p; + const int kernelH = kshape[2]; + const int kernelW = kshape[3]; + layerParams.set("kernel_h", kernelH); + layerParams.set("kernel_w", kernelW); + layerParams.set("num_output", kshape[1]); + + setStrides(layerParams, layer); + setPadMode(layerParams, layer); + int64_t pads[8]; + bool explicit_pads = getExplicitPadding(layerParams, layer, pads); + int64_t begs[4] = {}; + int64_t ends[4] = {-1, -1, -1, -1}; + if (explicit_pads) + { + name += "/deconv"; + layerParams.set("pad_mode", "VALID"); + for (int i = 2; i < 4; ++i) // begins=[0, 0, a, b], ends=[-1, -1, c, d] + { + begs[i] = pads[2*i]; + ends[i] = -1 - pads[2*i + 1]; + } + } + + // For convolution layer, output shape computes as + // o = 1 + (i - k + 2*p) / s + // i - input size, o - output size, k - kernel size, p - pad, s - stride + // In TensorFlow, p == 0 is padMode == 'VALID' or p == (k - 1) / 2 + // considering that k is odd. + // SAME: o = 1 + (i - 1) / s + // VALID: o = 1 + i / s + // Deconvolution's layer output shape computes as + // SAME: o = 1 + (i - 1)*s + // VALID: o = (i - 1)*s + // If output_shape differs from formulas above then adjust padding is applied. + + const int strideY = layerParams.get("stride_h"); + const int strideX = layerParams.get("stride_w"); + Mat outShape = getTensorContent(getConstBlob(layer, value_id, 0)); + int shift = (getDataLayout(layer) == DATA_LAYOUT_NCHW); + const int outH = outShape.at(1 + shift) + begs[2] - 1 - ends[2]; + const int outW = outShape.at(2 + shift) + begs[3] - 1 - ends[3]; + if (layerParams.get("pad_mode") == "SAME") + { + layerParams.set("adj_w", (outW - 1) % strideX); + layerParams.set("adj_h", (outH - 1) % strideY); + } + else if (layerParams.get("pad_mode") == "VALID") + { + layerParams.set("adj_w", (outW - kernelW) % strideX); + layerParams.set("adj_h", (outH - kernelH) % strideY); + } + int id = dstNet.addLayer(name, "Deconvolution", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); + if (explicit_pads) // If we have explicit paddings, remove extra data + { + layerParams.set("begin", DictValue::arrayInt(begs, sizeof(begs) / sizeof(begs[0]))); + layerParams.set("end", DictValue::arrayInt(ends, sizeof(ends) / sizeof(ends[0]))); + + int id = dstNet.addLayer(layer.name(), "Slice", layerParams); + layer_id[layer.name()] = id; + + connect(layer_id, dstNet, parsePin(name), id, 0); + } +} + +void TFImporter::parseBlockLSTM(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "BlockLSTM" + // input: "lstm_block_wrapper/ToInt64/x" (ignore, number of time stamps) + // input: "input" + // input: "lstm_block_wrapper/zeros" + // input: "lstm_block_wrapper/zeros" + // input: "lstm_block_wrapper/kernel" + // input: "lstm_block_wrapper/w_i_diag" + // input: "lstm_block_wrapper/w_f_diag" + // input: "lstm_block_wrapper/w_o_diag" + // input: "lstm_block_wrapper/bias" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 9, "Unexpected number of input nodes"); + + if (hasLayerAttr(layer, "forget_bias")) + layerParams.set("forget_bias", getLayerAttr(layer, "forget_bias").f()); + + if (hasLayerAttr(layer, "forget_bias")) + { + float cellClip = getLayerAttr(layer, "cell_clip").f(); + // Cell clip disabled if it's negative. + if (cellClip >= 0) + { + layerParams.set("use_cell_clip", true); + layerParams.set("cell_clip", cellClip); + } + } + + Mat W, Wh, Wx, b, cs_prev, h_prev; + blobFromTensor(getConstBlob(layer, value_id, 4), W); + blobFromTensor(getConstBlob(layer, value_id, 8), b); + blobFromTensor(getConstBlob(layer, value_id, 2), cs_prev); + blobFromTensor(getConstBlob(layer, value_id, 3), h_prev); + const int outSize = W.cols / 4; + + // IGFO->IFOG + float* weightData = (float*)W.data; + for (int i = 0; i < W.rows; ++i) + for (int j = 0; j < outSize; ++j) + { + std::swap(weightData[i * W.cols + 1 * outSize + j], + weightData[i * W.cols + 2 * outSize + j]); + std::swap(weightData[i * W.cols + 2 * outSize + j], + weightData[i * W.cols + 3 * outSize + j]); + } + Wx = W.rowRange(0, W.rows - outSize).t(); + Wh = W.rowRange(W.rows - outSize, W.rows).t(); + + layerParams.blobs.resize(5); + layerParams.blobs[0] = Wh; + layerParams.blobs[1] = Wx; + layerParams.blobs[2] = b; + layerParams.blobs[3] = h_prev; + layerParams.blobs[4] = cs_prev; + + if (hasLayerAttr(layer, "use_peephole")) + { + bool usePeephole = getLayerAttr(layer, "use_peephole").b(); + if (usePeephole) + { + layerParams.set("use_peephole", true); + layerParams.blobs.resize(8); + for (int i = 0; i < 3; ++i) + { + Mat w; + blobFromTensor(getConstBlob(layer, value_id, 5 + i), w); + w = w.reshape(1, w.total()); // Single column. + w = Mat::diag(w); // Make a diagonal matrix. + layerParams.blobs[5 + i] = w; + } + } + } + + int id = dstNet.addLayer(name, "LSTM", layerParams); + layer_id[name] = id; + + // one input only + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; +} + +// "ResizeNearestNeighbor" "ResizeBilinear" "FusedResizeAndPadConv2D" +void TFImporter::parseResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer_, LayerParams& layerParams) +{ + tensorflow::NodeDef layer = layer_; + std::string name = layer.name(); + const std::string& type = layer.op(); + int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + std::string convWeights = ""; + if (type == "FusedResizeAndPadConv2D") + { + // input: "mul_1" + // input: "decoder/ResizeBilinear/size" + // input: "decoder/decoder_conv0/Conv2D_dummy_paddings" + // input: "decoder/decoder_conv0/weights" + CV_CheckEQ(num_inputs, 4, "Number of input for FusedResizeAndPadConv2D"); + + Mat paddings = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckEQ(countNonZero(paddings), 0, "Unsupported mode"); + + convWeights = layer.input(3); + layer.mutable_input()->DeleteSubrange(2, 2); // FIXIT do NOT modify input model + num_inputs = layer.input_size(); + name = name + "/resize"; + + if (hasLayerAttr(layer, "resize_align_corners")) + { + // FIXIT do NOT modify input model + layer.mutable_attr()->insert( + ::google::protobuf::MapPair("align_corners", + getLayerAttr(layer, "resize_align_corners"))); + } + } + if (num_inputs == 2) + { + Mat outSize = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_CheckTypeEQ(outSize.type(), CV_32SC1, ""); CV_CheckEQ(outSize.total(), (size_t)2, ""); + layerParams.set("height", outSize.at(0, 0)); + layerParams.set("width", outSize.at(0, 1)); + } + else if (num_inputs == 3) + { + Mat factorHeight = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat factorWidth = getTensorContent(getConstBlob(layer, value_id, 2)); + factorHeight.convertTo(factorHeight, CV_32F); + factorWidth.convertTo(factorWidth, CV_32F); + layerParams.set("zoom_factor_x", factorWidth.at(0)); + layerParams.set("zoom_factor_y", factorHeight.at(0)); + } + else + CV_Check(num_inputs, num_inputs == 2 || num_inputs == 3, ""); + + if (type == "ResizeNearestNeighbor") + layerParams.set("interpolation", "nearest"); + else + layerParams.set("interpolation", "bilinear"); + + if (hasLayerAttr(layer, "align_corners")) + layerParams.set("align_corners", getLayerAttr(layer, "align_corners").b()); + + if (hasLayerAttr(layer, "half_pixel_centers")) + layerParams.set("half_pixel_centers", getLayerAttr(layer, "half_pixel_centers").b()); + + int id = dstNet.addLayer(name, "Resize", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + + // Step back to add convolution + if (type == "FusedResizeAndPadConv2D") + { + tensorflow::NodeDef conv = layer_; + conv.clear_input(); + conv.add_input(name); + conv.add_input(convWeights); + conv.set_op("Conv2D"); + parseNode(conv); + } +} + +void TFImporter::parseL2Normalize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "L2Normalize" + // input: "input" + // input: "reduction_indices" (axis) + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + Mat reductionIndices = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(reductionIndices.type() == CV_32SC1); + + const int numAxes = reductionIndices.total(); + if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) + for (int i = 0; i < numAxes; ++i) + reductionIndices.at(i) = toNCHW(reductionIndices.at(i)); + + cv::sort(reductionIndices, reductionIndices, SORT_ASCENDING); + for (int i = 1; i < numAxes; ++i) + { + CV_Assert(reductionIndices.at(i) == reductionIndices.at(i - 1) + 1); + // Axes have the same sign. + CV_Assert(reductionIndices.at(i) * reductionIndices.at(i - 1) >= 0); + } + layerParams.set("start_axis", reductionIndices.at(0)); + layerParams.set("end_axis", reductionIndices.at(numAxes - 1)); + + int id = dstNet.addLayer(name, "Normalize", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + +void TFImporter::parsePriorBox(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 2, ""); + if (hasLayerAttr(layer, "min_size")) + layerParams.set("min_size", getLayerAttr(layer, "min_size").i()); + if (hasLayerAttr(layer, "max_size")) + layerParams.set("max_size", getLayerAttr(layer, "max_size").i()); + if (hasLayerAttr(layer, "flip")) + layerParams.set("flip", getLayerAttr(layer, "flip").b()); + if (hasLayerAttr(layer, "clip")) + layerParams.set("clip", getLayerAttr(layer, "clip").b()); + if (hasLayerAttr(layer, "offset")) + layerParams.set("offset", getLayerAttr(layer, "offset").f()); + if (hasLayerAttr(layer, "step")) + layerParams.set("step", getLayerAttr(layer, "step").f()); + + const std::string paramNames[] = {"variance", "aspect_ratio", "scales", + "width", "height"}; + for (int i = 0; i < 5; ++i) + { + if (hasLayerAttr(layer, paramNames[i])) + { + Mat values = getTensorContent(getLayerAttr(layer, paramNames[i]).tensor()); + layerParams.set(paramNames[i], + DictValue::arrayReal((float*)values.data, values.total())); + } + } + int id = dstNet.addLayer(name, "PriorBox", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); + data_layouts[name] = DATA_LAYOUT_UNKNOWN; +} + +void TFImporter::parseSoftmax(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + if (hasLayerAttr(layer, "axis")) + layerParams.set("axis", getLayerAttr(layer, "axis").i()); + + int id = dstNet.addLayer(name, "Softmax", layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseCropAndResize(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "CropAndResize" + // input: "input" + // input: "boxes" + // input: "sizes" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + CV_CheckEQ(num_inputs, 3, ""); + + Mat cropSize = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckTypeEQ(cropSize.type(), CV_32SC1, ""); CV_CheckEQ(cropSize.total(), (size_t)2, ""); + + layerParams.set("height", cropSize.at(0)); + layerParams.set("width", cropSize.at(1)); + + int id = dstNet.addLayer(name, "CropAndResize", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); +} + +// "Mean" "Sum" "Max" +void TFImporter::parseMean(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // Computes the mean of elements across dimensions of a tensor. + // If keepdims is false (default) reduces input_tensor along the dimensions given in axis, + // else the reduced dimensions are retained with length 1. + // if indices = [1, 2] in NHWC layout we use global pooling: NxCxHxW --Pooling--> NxCx1x1 + // if keepdims is false we use Flatten after Pooling: out_shape = NxC + // if indices = [0] we use a global pooling by indices. + // To return correct shape, we use Reshape after Pooling. To determine input shape use Slice for input, + // if keepdims is false we use Flatten after Slice. + // Example: input_shape = NxCxHxW + // determine out shape: NxCxHxW --Slice--> 1xCxHxW + // out_shape = 1xCxHxW if keepDims else (1xCxHxW --Flatten--> CxHxW) + // global pool: NxCxHxW --Flatten--> Nx(C*H*W) --Reshape--> 1x1xNx(C*H*W) --Pooling--> 1x1x1x(C*H*W) --Reshape--> out_shape + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + std::string pool_type = cv::toLowerCase(type); + + if (pool_type == "mean") + { + pool_type = "ave"; + } + CV_CheckGT(num_inputs, 0, ""); + + Mat indices = getTensorContent(getConstBlob(layer, value_id, 1)); + CV_Assert(indices.type() == CV_32SC1); + + // There are two attributes, "keepdims" and a deprecated "keep_dims". + bool keepDims = false; + if (hasLayerAttr(layer, "keepdims")) + keepDims = getLayerAttr(layer, "keepdims").b(); + else if (hasLayerAttr(layer, "keep_dims")) + keepDims = getLayerAttr(layer, "keep_dims").b(); + + if (indices.total() == 1 && indices.at(0) == 0) + { + LayerParams flattenLp; + std::string flattenName = name + "/flatten"; + CV_Assert(layer_id.find(flattenName) == layer_id.end()); + int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); + layer_id[flattenName] = flattenId; + connect(layer_id, dstNet, parsePin(layer.input(0)), flattenId, 0); + + LayerParams reshapeLp; + std::string reshapeName = name + "/reshape"; + CV_Assert(layer_id.find(reshapeName) == layer_id.end()); + reshapeLp.set("axis", 0); + reshapeLp.set("num_axes", 1); + int newShape[] = {1, 1, -1}; + reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 3)); + + int reshapeId = dstNet.addLayer(reshapeName, "Reshape", reshapeLp); + layer_id[reshapeName] = reshapeId; + connect(layer_id, dstNet, Pin(flattenName), reshapeId, 0); + + LayerParams avgLp; + std::string avgName = name + "/avg"; + CV_Assert(layer_id.find(avgName) == layer_id.end()); + avgLp.set("pool", pool_type); + // pooling kernel H x 1 + avgLp.set("global_pooling_h", true); + avgLp.set("kernel_w", 1); + int avgId = dstNet.addLayer(avgName, "Pooling", avgLp); + layer_id[avgName] = avgId; + connect(layer_id, dstNet, Pin(reshapeName), avgId, 0); + + LayerParams sliceLp; + std::string layerShapeName = name + "/slice"; + CV_Assert(layer_id.find(layerShapeName) == layer_id.end()); + sliceLp.set("axis", 0); + int begin[] = {0}; + int size[] = {1}; + sliceLp.set("begin", DictValue::arrayInt(&begin[0], 1)); + sliceLp.set("size", DictValue::arrayInt(&size[0], 1)); + int sliceId = dstNet.addLayer(layerShapeName, "Slice", sliceLp); + layer_id[layerShapeName] = sliceId; + connect(layer_id, dstNet, Pin(layer.input(0)), sliceId, 0); + + if (!keepDims) + { + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + squeezeLp.set("axis", 0); + squeezeLp.set("end_axis", 1); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(layerShapeName), squeezeId, 0); + layerShapeName = squeezeName; + } + + int id = dstNet.addLayer(name, "Reshape", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, Pin(avgName), id, 0); + connect(layer_id, dstNet, Pin(layerShapeName), id, 1); + } else if (indices.total() == 1) { + int axis = toNCHW(indices.at(0)); + if (axis == 2 || axis == 3) + { + layerParams.set("pool", pool_type); + layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); + layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + + if (!keepDims) + { + // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC + LayerParams permLP; + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + std::string permName = name + "/nchw"; + Pin inpId = Pin(name); + addPermuteLayer(order, permName, inpId); + + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + squeezeLp.set("axis", indices.at(0)); + squeezeLp.set("end_axis", indices.at(0) + 1); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(permName), squeezeId, 0); + } + } + else if (axis == 1) + { + int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. + Pin inpId = parsePin(layer.input(0)); + addPermuteLayer(order, name + "/nhwc", inpId); + + layerParams.set("pool", pool_type); + layerParams.set("kernel_h", 1); + layerParams.set("global_pooling_w", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, inpId, id, 0); + + if (!keepDims) + { + LayerParams squeezeLp; + std::string squeezeName = name + "/squeeze"; + CV_Assert(layer_id.find(squeezeName) == layer_id.end()); + int channel_id = 3; // TF NHWC layout + squeezeLp.set("axis", channel_id - 1); + squeezeLp.set("end_axis", channel_id); + int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); + layer_id[squeezeName] = squeezeId; + connect(layer_id, dstNet, Pin(name), squeezeId, 0); + } + else + { + int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. + Pin inpId = parsePin(name); + addPermuteLayer(order, name + "/nchw", inpId); + } + } + } else { + if (indices.total() != 2 || indices.at(0) != 1 || indices.at(1) != 2) + CV_Error(Error::StsNotImplemented, "Unsupported mode of reduce_mean or reduce_sum operation."); + + layerParams.set("pool", pool_type); + layerParams.set("global_pooling", true); + int id = dstNet.addLayer(name, "Pooling", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); + + if (!keepDims) + { + LayerParams flattenLp; + std::string flattenName = name + "/flatten"; + CV_Assert(layer_id.find(flattenName) == layer_id.end()); + int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); + layer_id[flattenName] = flattenId; + connect(layer_id, dstNet, Pin(name), flattenId, 0); + } + } +} + +void TFImporter::parsePack(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: tf.stack(list of tensors, axis=0) + // Join a list of inputs along a new axis. + // The "axis" specifies the index of the new axis in the dimensions of the output. + // Example: given a list with "N" tensors of shape (C, H, W): + // if axis == 0 then the output tensor will have the shape (N, C, H, W), + // if axis == 1 then the output tensor will have the shape (C, N, H, W). + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + CV_Assert(hasLayerAttr(layer, "axis")); + int dim = (int)getLayerAttr(layer, "axis").i(); + if (dim != 0) + CV_Error(Error::StsNotImplemented, "Unsupported mode of pack operation."); + + CV_Assert(hasLayerAttr(layer, "N")); + int num = (int)getLayerAttr(layer, "N").i(); + CV_CheckEQ(num_inputs, num, ""); + std::string base_name = name + "/reshape_"; + std::vector reshape_ids; + for (int i = 0; i < num; i++) { + std::ostringstream ss; + ss << i; + std::string reshape_name = base_name + ss.str(); + LayerParams reshapeLP; + reshapeLP.set("axis", dim); + reshapeLP.set("num_axes", 1); + int outShape[] = {1, -1}; + reshapeLP.set("dim", DictValue::arrayInt(&outShape[0], 2)); + int id = dstNet.addLayer(reshape_name, "Reshape", reshapeLP); + layer_id[reshape_name] = id; + reshape_ids.push_back(id); + connect(layer_id, dstNet, parsePin(layer.input(i)), id, 0); + } + + layerParams.set("axis", dim); + int id = dstNet.addLayer(name, "Concat", layerParams); + layer_id[name] = id; + + for (int li = 0; li < num; li++) + dstNet.connect(reshape_ids[li], 0, id, li); +} + +void TFImporter::parseClipByValue(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // op: "ClipByValue" + // input: "input" + // input: "mix" + // input: "max" + + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckEQ(num_inputs, 3, ""); + + Mat minValue = getTensorContent(getConstBlob(layer, value_id, 1)); + Mat maxValue = getTensorContent(getConstBlob(layer, value_id, 2)); + CV_CheckEQ(minValue.total(), (size_t)1, ""); CV_CheckTypeEQ(minValue.type(), CV_32FC1, ""); + CV_CheckEQ(maxValue.total(), (size_t)1, ""); CV_CheckTypeEQ(maxValue.type(), CV_32FC1, ""); + + layerParams.set("min_value", minValue.at(0)); + layerParams.set("max_value", maxValue.at(0)); + + int id = dstNet.addLayer(name, "ReLU6", layerParams); + layer_id[name] = id; + + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + +void TFImporter::parseLeakyRelu(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + CV_Assert(hasLayerAttr(layer, "alpha")); + layerParams.set("negative_slope", getLayerAttr(layer, "alpha").f()); + + int id = dstNet.addLayer(name, "ReLU", layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +// "Abs" "Tanh" "Sigmoid" "Relu" "Elu" "Exp" "Identity" "Relu6" +void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + CV_CheckGT(num_inputs, 0, ""); + std::string dnnType = type; + if (type == "Abs") dnnType = "AbsVal"; + else if (type == "Tanh") dnnType = "TanH"; + else if (type == "Relu") dnnType = "ReLU"; + else if (type == "Relu6") dnnType = "ReLU6"; + else if (type == "Elu") dnnType = "ELU"; + + int id = dstNet.addLayer(name, dnnType, layerParams); + layer_id[name] = id; + connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); +} + +void TFImporter::parseCustomLayer(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + // Importer does not know how to map this TensorFlow's operation onto OpenCV's layer. + // However we create a layer with the same type and rely that user defined a custom layer. + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + const int num_inputs = layer.input_size(); + + // All the attributes are added to LayerParams. + google::protobuf::Map attr = layer.attr(); + for (google::protobuf::Map::const_iterator ai = attr.begin(); + ai != attr.end(); ++ai) + { + if (ai->second.value_case() == tensorflow::AttrValue::kS) // string + layerParams.set(ai->first, ai->second.s()); + if (ai->second.value_case() == tensorflow::AttrValue::kI) // int64 + layerParams.set(ai->first, ai->second.i()); + if (ai->second.value_case() == tensorflow::AttrValue::kF) // float + layerParams.set(ai->first, ai->second.f()); + if (ai->second.value_case() == tensorflow::AttrValue::kB) // bool + layerParams.set(ai->first, ai->second.b()); + } + + // All the Const input nodes are added to layer's blobs. + std::vector inputsNames; + for (int i = 0; i < num_inputs; ++i) + { + // Check if input is a Const node. + if (value_id.find(layer.input(i)) != value_id.end()) + { + Mat blob = getTensorContent(getConstBlob(layer, value_id, i)); + layerParams.blobs.push_back(blob); + } + else + inputsNames.push_back(layer.input(i)); + } + int id = dstNet.addLayer(name, type, layerParams); + layer_id[name] = id; + + for (int i = 0; i < inputsNames.size(); ++i) + { + connect(layer_id, dstNet, parsePin(inputsNames[i]), id, i); + } +} + TFImporter::TFImporter(Net& net, const char *model, const char *config) - : dstNet(net) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new TFLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) { if (model && model[0]) { @@ -534,7 +2626,8 @@ TFImporter::TFImporter( const char *dataModel, size_t lenModel, const char *dataConfig, size_t lenConfig ) - : dstNet(net) + : layerHandler(DNN_DIAGNOSTICS_RUN ? new TFLayerHandler(this) : nullptr), + dstNet(net), dispatch(buildDispatchMap()) { if (dataModel != NULL && lenModel > 0) { @@ -580,8 +2673,10 @@ void TFImporter::kernelFromTensor(const tensorflow::TensorProto &tensor, Mat &ds out_c = shape[0]; input_c = shape[1]; dstBlob.create(shape, CV_32F); + CV_Assert(dstBlob.isContinuous()); Mat tensorContent = getTensorContent(tensor, /*no copy*/false); + CV_Assert(tensorContent.isContinuous()); int size = tensorContent.total(); CV_Assert(size == (int)dstBlob.total()); @@ -893,6 +2988,10 @@ void TFImporter::populateNet() addConstNodes(netBin, value_id, layers_to_ignore); addConstNodes(netTxt, value_id, layers_to_ignore); + if (DNN_DIAGNOSTICS_RUN) { + CV_LOG_INFO(NULL, "DNN/TF: start diagnostic run!"); + layerHandler->fillRegistry(net); + } for (int li = 0; li < layersSize; li++) { @@ -912,13 +3011,13 @@ void TFImporter::populateNet() CV_Assert(!netInputsNames[i].empty()); } dstNet.setInputsNames(netInputsNames); - CV_LOG_DEBUG(NULL, "DNN/TF: ===================== Import completed ====================="); + CV_LOG_DEBUG(NULL, (DNN_DIAGNOSTICS_RUN? "DNN/TF: diagnostic run completed!" : "DNN/TF: import completed!")); } -void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId) +void TFImporter::addPermuteLayer(const int* order, const std::string& permName, Pin& inpId, int orderSize) { LayerParams permLP; - permLP.set("order", DictValue::arrayInt(order, 4)); + permLP.set("order", DictValue::arrayInt(order, orderSize)); CV_Assert(layer_id.find(permName) == layer_id.end()); int permId = dstNet.addLayer(permName, "Permute", permLP); layer_id[permName] = permId; @@ -926,19 +3025,16 @@ void TFImporter::addPermuteLayer(const int* order, const std::string& permName, inpId = Pin(permName); } -void TFImporter::parseNode(const tensorflow::NodeDef& layer_) +void TFImporter::parseNode(const tensorflow::NodeDef& layer) { - tensorflow::NodeDef layer = layer_; - tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin; - /*const*/ std::string name = layer.name(); - /*const*/ std::string type = layer.op(); - /*const*/ int num_inputs = layer.input_size(); + const std::string& name = layer.name(); + const std::string& type = layer.op(); + LayerParams layerParams; try { - LayerParams layerParams; if (layers_to_ignore.find(name) != layers_to_ignore.end()) { @@ -949,1618 +3045,70 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer_) DataLayout predictedLayout = predictOutputDataLayout(layer); data_layouts[name] = predictedLayout; - if (type == "Conv2D" || type == "SpaceToBatchND" || type == "DepthwiseConv2dNative" || type == "Pad" || type == "MirrorPad" || type == "Conv3D") + DispatchMap::const_iterator iter = dispatch.find(type); + if (iter != dispatch.end()) { - CV_CheckGT(num_inputs, 0, ""); - // The first node of dilated convolution subgraph. - // Extract input node, dilation rate and paddings. - std::string input = layer.input(0); - StrIntVector next_layers; - if (type == "SpaceToBatchND" || type == "Pad") - { - next_layers = getNextLayers(net, name, "Conv2D"); - if (next_layers.empty()) - next_layers = getNextLayers(net, name, "DepthwiseConv2dNative"); - } - - if (type == "SpaceToBatchND") - { - // op: "SpaceToBatchND" - // input: "input" - // input: "SpaceToBatchND/block_shape" - // input: "SpaceToBatchND/paddings" - CV_CheckEQ(num_inputs, 3, ""); - - DictValue dilation = parseDims(getConstBlob(layer, value_id, 1)); - CV_Assert(dilation.size() == 2); - layerParams.set("dilation_h", dilation.get(0)); - layerParams.set("dilation_w", dilation.get(1)); - - Mat paddings; - parseTensor(getConstBlob(layer, value_id, 2), paddings); - - // paddings is a 2x2 matrix: [[top, bot], [left, right]] - layerParams.set("pad_h", paddings.at(0)); - layerParams.set("pad_w", paddings.at(2)); - - CV_Assert(next_layers.size() == 1); - layers_to_ignore.insert(next_layers[0].first); - - // FIXIT don't override, rewrite this code - layer = net.node(next_layers[0].second); - name = layer.name(); - type = layer.op(); - num_inputs = layer.input_size(); - CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); - } - else if (type == "Pad" || type == "MirrorPad") - { - Mat paddings = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(paddings.type() == CV_32SC1); - if (paddings.total() == 8) - { - // Perhaps, we have NHWC padding dimensions order. - // N H W C - // 0 1 2 3 4 5 6 7 - std::swap(paddings.at(2), paddings.at(6)); - std::swap(paddings.at(3), paddings.at(7)); - // N C W H - // 0 1 2 3 4 5 6 7 - std::swap(paddings.at(4), paddings.at(6)); - std::swap(paddings.at(5), paddings.at(7)); - // N C H W - // 0 1 2 3 4 5 6 7 - } - - if (next_layers.empty() || paddings.total() != 8 || - paddings.at(4) != paddings.at(5) || - paddings.at(6) != paddings.at(7) || type == "MirrorPad") - { - // Just a single padding layer. - layerParams.set("paddings", DictValue::arrayInt((int*)paddings.data, paddings.total())); - if (type == "MirrorPad") - layerParams.set("type", "reflect"); - - int id = dstNet.addLayer(name, "Padding", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(input), id, 0); - return; - } - else - { - // Merge with subsequent convolutional layer. - CV_Assert(next_layers.size() == 1); - - layerParams.set("pad_h", paddings.at(4)); - layerParams.set("pad_w", paddings.at(6)); - - layers_to_ignore.insert(next_layers[0].first); - - // FIXIT don't override, rewrite this code - layer = net.node(next_layers[0].second); - name = layer.name(); - type = layer.op(); - num_inputs = layer.input_size(); - CV_LOG_DEBUG(NULL, "DNN/TF: switched to layer " << name << " @ " << type << ") with " << num_inputs << " inputs"); - } - } - - // For the object detection networks, TensorFlow Object Detection API - // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) - // order. We can manage it at DetectionOutput layer parsing predictions - // or shuffle last convolution's weights. - bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && - getLayerAttr(layer, "loc_pred_transposed").b(); - - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); - - next_layers = getNextLayers(net, name, "BiasAdd"); - if (next_layers.size() == 1) { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - - // Shuffle bias from yxYX to xyXY. - if (locPredTransposed) - { - const int numWeights = layerParams.blobs[1].total(); - float* biasData = reinterpret_cast(layerParams.blobs[1].data); - CV_Assert(numWeights % 4 == 0); - for (int i = 0; i < numWeights; i += 2) - { - std::swap(biasData[i], biasData[i + 1]); - } - } - } - - int kernelTensorInpId = -1; - const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernelTensorInpId); - const String kernelTensorName = layer.input(kernelTensorInpId); - std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); - if (sharedWeightsIt == sharedWeights.end()) - { - kernelFromTensor(kernelTensor, layerParams.blobs[0]); - releaseTensor(const_cast(&kernelTensor)); - - int* kshape = layerParams.blobs[0].size.p; - const int outCh = kshape[0]; - const int inCh = kshape[1]; - const int height = kshape[2]; - const int width = kshape[3]; - if (type == "DepthwiseConv2dNative") - { - CV_Assert(!locPredTransposed); - const int chMultiplier = kshape[0]; - - Mat copy = layerParams.blobs[0].clone(); - float* src = (float*)copy.data; - float* dst = (float*)layerParams.blobs[0].data; - for (int i = 0; i < chMultiplier; ++i) - for (int j = 0; j < inCh; ++j) - for (int s = 0; s < height * width; ++s) - { - int src_i = (i * inCh + j) * height * width + s; - int dst_i = (j * chMultiplier + i) * height* width + s; - dst[dst_i] = src[src_i]; - } - // TODO Use reshape instead - kshape[0] = inCh * chMultiplier; - kshape[1] = 1; - size_t* kstep = layerParams.blobs[0].step.p; - kstep[0] = kstep[1]; // fix steps too - } - - // Shuffle output channels from yxYX to xyXY. - if (locPredTransposed) - { - const int slice = height * width * inCh; - for (int i = 0; i < outCh; i += 2) - { - cv::Mat src(1, slice, CV_32F, layerParams.blobs[0].ptr(i)); - cv::Mat dst(1, slice, CV_32F, layerParams.blobs[0].ptr(i + 1)); - std::swap_ranges(src.begin(), src.end(), dst.begin()); - } - } - sharedWeights[kernelTensorName] = layerParams.blobs[0]; - } - else - { - layerParams.blobs[0] = sharedWeightsIt->second; - } - Mat weights = layerParams.blobs[0]; - layerParams.set("kernel_size", DictValue::arrayInt(&weights.size[2], weights.dims - 2)); - - layerParams.set("num_output", layerParams.blobs[0].size[0]); - - setStrides(layerParams, layer); - if (!layerParams.has("pad_w") && !layerParams.has("pad_h")) - setPadding(layerParams, layer); - - // The final node of dilated convolution subgraph. - next_layers = getNextLayers(net, name, "BatchToSpaceND"); - if (!next_layers.empty()) - { - CV_Assert(next_layers.size() == 1); - ExcludeLayer(net, next_layers[0].second, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } - - int id = dstNet.addLayer(name, "Convolution", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(input), id, 0); - - - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_UNKNOWN) - data_layouts[name] = DATA_LAYOUT_NHWC; + CALL_MEMBER_FN(*this, iter->second)(net, layer, layerParams); } - else if (type == "BiasAdd" || type == "Add" || type == "AddV2" || type == "Sub" || type=="AddN") + else if (!DNN_DIAGNOSTICS_RUN || !layerHandler->handleMissing(layer)) { - CV_CheckGT(num_inputs, 0, ""); - bool haveConst = false; - for(int ii = 0; !haveConst && ii < num_inputs; ++ii) - { - Pin input = parsePin(layer.input(ii)); - haveConst = value_id.find(input.name) != value_id.end(); - } - CV_Assert(!haveConst || num_inputs == 2); - - if (haveConst) - { - Mat values = getTensorContent(getConstBlob(layer, value_id)); - CV_Assert(values.type() == CV_32FC1); - if (type == "Sub") - values *= -1.0f; - - int id; - if (values.total() == 1) // is a scalar. - { - layerParams.set("shift", values.at(0)); - id = dstNet.addLayer(name, "Power", layerParams); - } - else // is a vector - { - layerParams.blobs.resize(1, values); - id = dstNet.addLayer(name, "Shift", layerParams); - } - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else - { - layerParams.set("operation", "sum"); - if (type == "Sub") - { - static float subCoeffs[] = {1.f, -1.f}; - layerParams.set("coeff", DictValue::arrayReal(subCoeffs, 2)); - } - - int id = dstNet.addLayer(name, "Eltwise", layerParams); - layer_id[name] = id; - - for (int ii = 0; ii < num_inputs; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii); - } - } - } - else if (type == "MatMul") - { - CV_CheckEQ(num_inputs, 2, ""); - - // For the object detection networks, TensorFlow Object Detection API - // predicts deltas for bounding boxes in yxYX (ymin, xmin, ymax, xmax) - // order. We can manage it at DetectionOutput layer parsing predictions - // or shuffle last Faster-RCNN's matmul weights. - bool locPredTransposed = hasLayerAttr(layer, "loc_pred_transposed") && - getLayerAttr(layer, "loc_pred_transposed").b(); - - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); - - StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); // FIXIT Use layers fusion instead - if (next_layers.empty()) - { - next_layers = getNextLayers(net, name, "Add"); - } - if (next_layers.size() == 1) { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - - if (locPredTransposed) - { - const int numWeights = layerParams.blobs[1].total(); - float* biasData = reinterpret_cast(layerParams.blobs[1].data); - CV_Assert(numWeights % 4 == 0); - for (int i = 0; i < numWeights; i += 2) - { - std::swap(biasData[i], biasData[i + 1]); - } - } - } - - int kernel_blob_index = -1; - const tensorflow::TensorProto& kernelTensor = getConstBlob(layer, value_id, -1, &kernel_blob_index); - const String kernelTensorName = layer.input(kernel_blob_index); - std::map::iterator sharedWeightsIt = sharedWeights.find(kernelTensorName); - if (sharedWeightsIt == sharedWeights.end()) - { - blobFromTensor(kernelTensor, layerParams.blobs[0]); - releaseTensor(const_cast(&kernelTensor)); - sharedWeights[kernelTensorName] = layerParams.blobs[0]; - } - else - { - layerParams.blobs[0] = sharedWeightsIt->second; - } - - if (kernel_blob_index == 1) { // In this case output is computed by x*W formula - W should be transposed - Mat data = layerParams.blobs[0].t(); - layerParams.blobs[0] = data.clone(); - } - - layerParams.set("num_output", layerParams.blobs[0].size[0]); - if (locPredTransposed) - { - CV_Assert(layerParams.blobs[0].dims == 2); - for (int i = 0; i < layerParams.blobs[0].size[0]; i += 2) - { - cv::Mat src = layerParams.blobs[0].row(i); - cv::Mat dst = layerParams.blobs[0].row(i + 1); - std::swap_ranges(src.begin(), src.end(), dst.begin()); - } - } - - int id = dstNet.addLayer(name, "InnerProduct", layerParams); - layer_id[name] = id; - - // one input only - int input_blob_index = kernel_blob_index == 0 ? 1 : 0; - connect(layer_id, dstNet, parsePin(layer.input(input_blob_index)), id, 0); - data_layouts[name] = DATA_LAYOUT_PLANAR; - } - else if (type == "Reshape") - { - CV_CheckGT(num_inputs, 0, ""); - Pin inpId = parsePin(layer.input(0)); - DataLayout inpLayout = getDataLayout(layer.input(0), data_layouts); - // There are two possible implementations: reshape an input using - // predefined sizes or use a second input blob as a source of new shape. - if (value_id.find(layer.input(1)) != value_id.end()) - { - Mat newShape = getTensorContent(getConstBlob(layer, value_id, 1)); - int newShapeSize = newShape.total(); - bool hasSwap = false; - if (newShapeSize == 4 && hasAllOnes(newShape, 0, 2)) - { - // NHWC->NCHW - std::swap(*newShape.ptr(0, 2), *newShape.ptr(0, 3)); - std::swap(*newShape.ptr(0, 1), *newShape.ptr(0, 2)); - hasSwap = true; - } - if (inpLayout == DATA_LAYOUT_NHWC) - { - if (newShapeSize >= 2 || newShape.at(1) == 1) - { - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - addPermuteLayer(order, name + "/nhwc", inpId); - if (newShapeSize < 4) - { - inpLayout = DATA_LAYOUT_NCHW; - } - else - { - inpLayout = DATA_LAYOUT_NHWC; - } - } - } - layerParams.set("dim", DictValue::arrayInt(newShape.ptr(), newShapeSize)); - - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, inpId, id, 0); - inpId = Pin(name); - - if ((inpLayout == DATA_LAYOUT_NHWC || inpLayout == DATA_LAYOUT_UNKNOWN || inpLayout == DATA_LAYOUT_PLANAR) && - newShapeSize == 4 && !hasSwap) - { - int order[] = {0, 3, 1, 2}; // Transform back to OpenCV's NCHW. - addPermuteLayer(order, name + "/nchw", inpId); - inpLayout = DATA_LAYOUT_NCHW; - } - - data_layouts[name] = newShapeSize == 2 ? DATA_LAYOUT_PLANAR : inpLayout; - } - else - { - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - data_layouts[name] = inpLayout; - } - } - else if (type == "Flatten" || type == "Squeeze") - { - CV_CheckGT(num_inputs, 0, ""); - Pin inpId = parsePin(layer.input(0)); - int inpLayout = getDataLayout(layer.input(0), data_layouts); - if (type == "Squeeze") - { - CV_Assert(hasLayerAttr(layer, "squeeze_dims")); - const tensorflow::AttrValue& dims = getLayerAttr(layer, "squeeze_dims"); - std::vector dimsVector(dims.list().i_size()); - for (int i = 0; i < dimsVector.size(); ++i) - dimsVector[i] = dims.list().i(i); - - // Flatten layer can squeeze dimensions range into one. - std::sort(dimsVector.begin(), dimsVector.end()); - for (int i = 1; i < dimsVector.size(); ++i) - { - if (dimsVector[i] != dimsVector[i - 1] + 1) - CV_Error(Error::StsNotImplemented, "Unsupported squeeze configuration"); - } - int start = dimsVector.front() - 1, end = dimsVector.back(); - if (start == -1 && end == 0) // squeeze 0th dimension - { - start = 0; - end = 1; - } - layerParams.set("axis", start); - layerParams.set("end_axis", end); - } - if (inpLayout == DATA_LAYOUT_NHWC) - { - LayerParams permLP; - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - permLP.set("order", DictValue::arrayInt(order, 4)); - - std::string permName = name + "/nchw"; - CV_Assert(layer_id.find(permName) == layer_id.end()); - int permId = dstNet.addLayer(permName, "Permute", permLP); - layer_id[permName] = permId; - connect(layer_id, dstNet, inpId, permId, 0); - inpId = Pin(permName); - } - int id = dstNet.addLayer(name, "Flatten", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - data_layouts[name] = DATA_LAYOUT_PLANAR; - } - else if (type == "Transpose") - { - CV_CheckGT(num_inputs, 0, ""); - Mat perm = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(perm.type() == CV_32SC1); - int* permData = (int*)perm.data; - if (perm.total() == 4) - { - // Only NHWC <-> NCHW permutations are allowed. OpenCV is always - // keep NCHW layout this way. - int inpLayout = getDataLayout(layer.input(0), data_layouts); - std::string type = "Identity"; - if (inpLayout == DATA_LAYOUT_NHWC) - { - if (permData[0] == 0 && permData[1] == 3 && permData[2] == 1 && permData[3] == 2) - { - // in TensorFlow: NHWC->NCHW - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NCHW; - } - else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) - { - // in TensorFlow: NHWC->NHWC - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NHWC; - } - else if (permData[0] == 0 && permData[1] == 3 && permData[2] == 2 && permData[3] == 1) - { - // in TensorFlow: NHWC->NCWH - // in OpenCV: NCHW->NCWH - int permData[] = {0, 1, 3, 2}; - layerParams.set("order", DictValue::arrayInt(permData, perm.total())); - data_layouts[name] = DATA_LAYOUT_NCHW; // we keep track NCHW because channels position only matters - type = "Permute"; - } - else - CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); - } - else if (inpLayout == DATA_LAYOUT_NCHW) - { - if (permData[0] == 0 && permData[1] == 2 && permData[2] == 3 && permData[3] == 1) - { - // in TensorFlow: NCHW->NHWC - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NHWC; - } - else if (permData[0] == 0 && permData[1] == 1 && permData[2] == 2 && permData[3] == 3) - { - // in TensorFlow: NCHW->NCHW - // in OpenCV: NCHW->NCHW - data_layouts[name] = DATA_LAYOUT_NCHW; - } - else - CV_Error(Error::StsParseError, "Only NHWC <-> NCHW permutations are allowed."); - } - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else - { - layerParams.set("order", DictValue::arrayInt(permData, perm.total())); - - int id = dstNet.addLayer(name, "Permute", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; - } - } - else if (type == "Const") - { - } - else if (type == "LRN") - { - CV_CheckGT(num_inputs, 0, ""); - if(hasLayerAttr(layer, "alpha")) { - layerParams.set("alpha", getLayerAttr(layer, "alpha").f()); - } - if(hasLayerAttr(layer, "beta")) { - layerParams.set("beta", getLayerAttr(layer, "beta").f()); - } - if(hasLayerAttr(layer, "depth_radius")) { - int radius = (int)getLayerAttr(layer, "depth_radius").i(); - layerParams.set("local_size", 2*radius + 1); - } - if(hasLayerAttr(layer, "bias")) { - layerParams.set("bias", getLayerAttr(layer, "bias").f()); - } - layerParams.set("norm_by_size", false); - - int id = dstNet.addLayer(name, "LRN", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "Concat" || type == "ConcatV2") - { - CV_CheckGT(num_inputs, 0, ""); - int axisId = (type == "Concat" ? 0 : num_inputs - 1); - int axis = getConstBlob(layer, value_id, axisId).int_val().Get(0); - - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - axis = toNCHW(axis); - else if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NDHWC) - axis = toNCDHW(axis); - layerParams.set("axis", axis); - - // input(0) or input(n-1) is concat_dim - int from = (type == "Concat" ? 1 : 0); - int to = (type == "Concat" ? num_inputs : num_inputs - 1); - - for (int ii = from; ii < to; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - { - // There are constant inputs. - LayerParams lp; - lp.name = inp.name; - lp.type = "Const"; - lp.blobs.resize(1); - blobFromTensor(getConstBlob(layer, value_id, ii), lp.blobs.back()); - CV_Assert_N(!lp.blobs[0].empty(), lp.blobs[0].type() == CV_32F); - - int constInpId = dstNet.addLayer(lp.name, lp.type, lp); - layer_id[lp.name] = constInpId; - } - } - - int id = dstNet.addLayer(name, "Concat", layerParams); - layer_id[name] = id; - - for (int ii = from; ii < to; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii - from); - } - } - else if (type == "MaxPool" || type == "MaxPool3D") - { - CV_CheckGT(num_inputs, 0, ""); - layerParams.set("pool", "max"); - - setKSize(layerParams, layer); - setStrides(layerParams, layer); - setPadding(layerParams, layer); - // Test_TensorFlow_nets.EAST_text_detection/1, NGRAPH/CPU - layerParams.set("ceil_mode", false); - - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "AvgPool" || type == "AvgPool3D") - { - CV_CheckGT(num_inputs, 0, ""); - layerParams.set("pool", "ave"); - layerParams.set("ave_pool_padded_area", false); - setKSize(layerParams, layer); - setStrides(layerParams, layer); - setPadding(layerParams, layer); - - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "MaxPoolGrad") - { - CV_CheckEQ(num_inputs, 3, ""); - - layerParams.set("pool_k_h", 0); - layerParams.set("pool_k_w", 0); - layerParams.set("pool_stride_h", 0); - layerParams.set("pool_stride_w", 0); - layerParams.set("pool_pad_h", 0); - layerParams.set("pool_pad_w", 0); - - int id = dstNet.addLayer(name, "MaxUnpool", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1) + ":1"), id, 1); - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 2); - } - else if (type == "Placeholder") - { - if (!hasLayerAttr(layer, "dtype") || - getLayerAttr(layer, "dtype").type() != tensorflow::DT_BOOL) // If input is not a train/test flag. - { - netInputsNames.push_back(name); - layer_id[name] = 0; - } - tensorflow::TensorShapeProto shape; - if (hasLayerAttr(layer, "shape")) - shape = getLayerAttr(layer, "shape").shape(); - else if (hasLayerAttr(layer, "_output_shapes")) - { - tensorflow::AttrValue_ListValue list = getLayerAttr(layer, "_output_shapes").list(); - if (list.shape_size()) - shape = list.shape()[0]; - } - if (shape.dim_size()) - { - MatShape dims(shape.dim_size()); - for (int i = 0; i < dims.size(); ++i) - dims[i] = shape.dim(i).size(); - if (dims.size() == 4 && predictedLayout == DATA_LAYOUT_NHWC) - { - std::swap(dims[1], dims[3]); // NHWC->NCWH - std::swap(dims[2], dims[3]); // NCWH->NCHW - if (dims[0] == -1) // It's OK to have undetermined batch size - dims[0] = 1; - } - bool hasNeg = false; - for (int i = 0; i < dims.size() && !hasNeg; ++i) - { - hasNeg = dims[i] < 0; - } - if (!hasNeg) - netInputShapes.push_back(dims); - } - } - else if (type == "Split") { - // TODO: determining axis index remapping by input dimensions order of input blob - // TODO: slicing input may be Const op - // TODO: slicing kernels for convolutions - in current implementation it is impossible - // TODO: add parsing num of slices parameter - CV_CheckEQ(num_inputs, 2, ""); - // num_split - // 1st blob is dims tensor - int axis = getConstBlob(layer, value_id, 0).int_val().Get(0); - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - axis = toNCHW(axis); - layerParams.set("axis", axis); - - if (hasLayerAttr(layer, "num_split")) - layerParams.set("num_split", getLayerAttr(layer, "num_split").i()); - - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - } - else if (type == "Slice") - { - // op: "Slice" - // input: "input_node" - // input: "Slice/begin" - // input: "Slice/size" - CV_CheckEQ(num_inputs, 3, ""); - Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat sizes = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_Assert_N(!begins.empty(), !sizes.empty()); - CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); - CV_CheckTypeEQ(sizes.type(), CV_32SC1, ""); - - if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - { - // Swap NHWC parameters' order to NCHW. - std::swap(*begins.ptr(0, 2), *begins.ptr(0, 3)); - std::swap(*begins.ptr(0, 1), *begins.ptr(0, 2)); - std::swap(*sizes.ptr(0, 2), *sizes.ptr(0, 3)); - std::swap(*sizes.ptr(0, 1), *sizes.ptr(0, 2)); - } - layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); - layerParams.set("size", DictValue::arrayInt((int*)sizes.data, sizes.total())); - - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "StridedSlice") - { - CV_CheckEQ(num_inputs, 4, ""); - Mat begins = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat ends = getTensorContent(getConstBlob(layer, value_id, 2)); - Mat strides = getTensorContent(getConstBlob(layer, value_id, 3)); - CV_CheckTypeEQ(begins.type(), CV_32SC1, ""); - CV_CheckTypeEQ(ends.type(), CV_32SC1, ""); - CV_CheckTypeEQ(strides.type(), CV_32SC1, ""); - const int num = begins.total(); - CV_Assert_N(num == ends.total(), num == strides.total()); - - int end_mask = getLayerAttr(layer, "end_mask").i(); - for (int i = 0; i < num; ++i) - { - if (ends.at(i) < 0) - ends.at(i) -= 1; - if (end_mask & (1 << i)) - ends.at(i) = -1; - if (strides.at(i) != 1) - CV_Error(Error::StsNotImplemented, - format("StridedSlice with stride %d", strides.at(i))); - } - if (begins.total() == 4 && getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - { - // Swap NHWC parameters' order to NCHW. - std::swap(begins.at(2), begins.at(3)); - std::swap(begins.at(1), begins.at(2)); - std::swap(ends.at(2), ends.at(3)); - std::swap(ends.at(1), ends.at(2)); - } - layerParams.set("begin", DictValue::arrayInt((int*)begins.data, begins.total())); - layerParams.set("end", DictValue::arrayInt((int*)ends.data, ends.total())); - - int id = dstNet.addLayer(name, "Slice", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "Mul" || type == "RealDiv") - { - CV_CheckGT(num_inputs, 0, ""); - int constId = -1; - for(int ii = 0; ii < num_inputs; ++ii) - { - Pin input = parsePin(layer.input(ii)); - if (value_id.find(input.name) != value_id.end()) - { - constId = ii; - break; - } - } - CV_Assert((constId != -1) || (num_inputs == 2)); - - if (constId != -1) - { - // Multiplication by constant. - CV_CheckEQ(num_inputs, 2, ""); - Mat scaleMat = getTensorContent(getConstBlob(layer, value_id)); - CV_Assert(scaleMat.type() == CV_32FC1); - if (type == "RealDiv") - { - if (constId == 0) - CV_Error(Error::StsNotImplemented, "Division of constant over variable"); - scaleMat = 1.0f / scaleMat; - } - - int id; - if (scaleMat.total() == 1) // is a scalar. - { - // Try to match with a LeakyRelu: - // node { - // name: "LeakyRelu/mul" - // op: "Mul" - // input: "LeakyRelu/alpha" - // input: "input" - // } - // node { - // name: "LeakyRelu/Maximum" - // op: "Maximum" - // input: "LeakyRelu/mul" - // input: "input" - // } - StrIntVector next_layers = getNextLayers(net, name, "Maximum"); - if (!next_layers.empty()) - { - int maximumLayerIdx = next_layers[0].second; - - CV_Assert(net.node(maximumLayerIdx).input_size() == 2); - - // The input from the Mul layer can also be at index 1. - int mulInputIdx = (net.node(maximumLayerIdx).input(0) == name) ? 0 : 1; - - ExcludeLayer(net, maximumLayerIdx, mulInputIdx, false); - layers_to_ignore.insert(next_layers[0].first); - - layerParams.set("negative_slope", scaleMat.at(0)); - id = dstNet.addLayer(name, "ReLU", layerParams); - } - else - { - // Just a multiplication. - layerParams.set("scale", scaleMat.at(0)); - id = dstNet.addLayer(name, "Power", layerParams); - } - } - else // is a vector - { - layerParams.blobs.resize(1, scaleMat); - - StrIntVector next_layers = getNextLayers(net, name, "Add"); - if (!next_layers.empty()) - { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs.back()); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } - - if (hasLayerAttr(layer, "axis")) - layerParams.set("axis", getLayerAttr(layer, "axis").i()); - - id = dstNet.addLayer(name, "Scale", layerParams); - } - layer_id[name] = id; - - Pin inp0 = parsePin(layer.input(0)); - if (layer_id.find(inp0.name) != layer_id.end()) - // First operand is a constant. - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - else - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - } - else - { - // Check if all the inputs have the same shape. - bool equalInpShapes = true; - bool isShapeOnes = false; - MatShape outShape0; - for (int ii = 0; ii < num_inputs && !netInputShapes.empty(); ii++) - { - Pin pin = parsePin(layer.input(ii)); - int inpId = layer_id.find(pin.name)->second; - - // Get input shape - MatShape outShape; - std::vector inpShapes, outShapes; - dstNet.getLayerShapes(netInputShapes, inpId, inpShapes, outShapes); - CV_CheckGT(static_cast(outShapes.size()), pin.blobIndex, ""); - outShape = outShapes[pin.blobIndex]; - - if (ii == 0) - { - outShape0 = outShape; - } - else if (outShape != outShape0) - { - equalInpShapes = false; - isShapeOnes = isAllOnes(outShape, 2, outShape.size()) || - isAllOnes(outShape0, 2, outShape0.size()); - break; - } - } - - int id; - if (equalInpShapes || netInputShapes.empty() || (!equalInpShapes && isShapeOnes)) - { - layerParams.set("operation", type == "RealDiv" ? "div" : "prod"); - id = dstNet.addLayer(name, "Eltwise", layerParams); - } - else - { - if (type == "RealDiv") - CV_Error(Error::StsNotImplemented, "Division of non equal tensors"); - id = dstNet.addLayer(name, "Scale", layerParams); - } - - layer_id[name] = id; - - for (int ii = 0; ii < num_inputs; ii++) - { - Pin inp = parsePin(layer.input(ii)); - if (layer_id.find(inp.name) == layer_id.end()) - CV_Error(Error::StsError, "Input layer not found: " + inp.name); - connect(layer_id, dstNet, inp, id, ii); - } - } - } - else if (type == "FusedBatchNorm" || type == "FusedBatchNormV3") - { - // op: "FusedBatchNorm" - // input: "input" - // input: "BatchNorm/gamma" - // input: "BatchNorm/beta" - // input: "BatchNorm/moving_mean" - // input: "BatchNorm/moving_variance" - CV_CheckEQ(num_inputs, 5, "Expected gamma, beta, mean and std"); - Pin inpId = parsePin(layer.input(0)); - - bool isTraining = hasLayerAttr(layer, "is_training") && getLayerAttr(layer, "is_training").b(); - - layerParams.blobs.resize(2); - - const tensorflow::TensorProto& gammaTensor = getConstBlob(layer, value_id, 1); - if (!gammaTensor.tensor_content().empty()) - { - layerParams.blobs.resize(layerParams.blobs.size() + 1); - layerParams.set("has_weight", true); - blobFromTensor(gammaTensor, layerParams.blobs.back()); - } - else - layerParams.set("has_weight", false); - - const tensorflow::TensorProto& betaTensor = getConstBlob(layer, value_id, 2); - if (!betaTensor.tensor_content().empty()) - { - layerParams.blobs.resize(layerParams.blobs.size() + 1); - layerParams.set("has_bias", true); - blobFromTensor(betaTensor, layerParams.blobs.back()); - } - else - layerParams.set("has_bias", false); - - Mat mean, std; - if (isTraining) - { - if (layerParams.blobs.size() == 2) - CV_Error(Error::StsNotImplemented, "Cannot determine number " - "of parameters for batch normalization layer."); - mean = Mat::zeros(1, layerParams.blobs[2].total(), CV_32F); - std = Mat::ones(1, layerParams.blobs[2].total(), CV_32F); - - // Add an extra layer: Mean-Variance normalization - LayerParams mvnParams; - std::string mvnName = name + "/MVN"; - CV_Assert(layer_id.find(mvnName) == layer_id.end()); - int mvnId = dstNet.addLayer(mvnName, "MVN", mvnParams); - layer_id[mvnName] = mvnId; - connect(layer_id, dstNet, inpId, mvnId, 0); - inpId = Pin(mvnName); - } - else - { - blobFromTensor(getConstBlob(layer, value_id, 3), mean); - blobFromTensor(getConstBlob(layer, value_id, 4), std); - } - layerParams.blobs[0] = mean; - layerParams.blobs[1] = std; - - if (hasLayerAttr(layer, "epsilon")) - layerParams.set("eps", getLayerAttr(layer, "epsilon").f()); - - int id = dstNet.addLayer(name, "BatchNorm", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, inpId, id, 0); - } - else if (type == "Conv2DBackpropInput") - { - // op: "Conv2DBackpropInput" - // input: "conv2d_transpose/output_shape" - // input: "weights" - // input: "input" - CV_CheckEQ(num_inputs, 3, "Expected output shape, weights and input nodes"); - - layerParams.set("bias_term", false); - layerParams.blobs.resize(1); - - StrIntVector next_layers = getNextLayers(net, name, "BiasAdd"); - if (next_layers.size() == 1) - { - layerParams.set("bias_term", true); - layerParams.blobs.resize(2); - - int weights_layer_index = next_layers[0].second; - - blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]); - ExcludeLayer(net, weights_layer_index, 0, false); - layers_to_ignore.insert(next_layers[0].first); - } - - kernelFromTensor(getConstBlob(layer, value_id, 1), layerParams.blobs[0]); - - const int* kshape = layerParams.blobs[0].size.p; - const int kernelH = kshape[2]; - const int kernelW = kshape[3]; - layerParams.set("kernel_h", kernelH); - layerParams.set("kernel_w", kernelW); - layerParams.set("num_output", kshape[1]); - - setStrides(layerParams, layer); - setPadding(layerParams, layer); - - // For convolution layer, output shape computes as - // o = 1 + (i - k + 2*p) / s - // i - input size, o - output size, k - kernel size, p - pad, s - stride - // In TensorFlow, p == 0 is padMode == 'VALID' or p == (k - 1) / 2 - // considering that k is odd. - // SAME: o = 1 + (i - 1) / s - // VALID: o = 1 + i / s - // Deconvolution's layer output shape computes as - // SAME: o = 1 + (i - 1)*s - // VALID: o = (i - 1)*s - // If output_shape differs from formulas above then adjust padding is applied. - - const int strideY = layerParams.get("stride_h"); - const int strideX = layerParams.get("stride_w"); - Mat outShape = getTensorContent(getConstBlob(layer, value_id, 0)); - const int outH = outShape.at(1); - const int outW = outShape.at(2); - if (layerParams.get("pad_mode") == "SAME") - { - layerParams.set("adj_w", (outW - 1) % strideX); - layerParams.set("adj_h", (outH - 1) % strideY); - } - else if (layerParams.get("pad_mode") == "VALID") - { - layerParams.set("adj_w", (outW - kernelW) % strideX); - layerParams.set("adj_h", (outH - kernelH) % strideY); - } - int id = dstNet.addLayer(name, "Deconvolution", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(2)), id, 0); - } - else if (type == "BlockLSTM") - { - // op: "BlockLSTM" - // input: "lstm_block_wrapper/ToInt64/x" (ignore, number of time stamps) - // input: "input" - // input: "lstm_block_wrapper/zeros" (ignore) - // input: "lstm_block_wrapper/zeros" (ignore) - // input: "lstm_block_wrapper/kernel" - // input: "lstm_block_wrapper/w_i_diag" - // input: "lstm_block_wrapper/w_f_diag" - // input: "lstm_block_wrapper/w_o_diag" - // input: "lstm_block_wrapper/bias" - CV_CheckEQ(num_inputs, 9, "Unexpected number of input nodes"); - - if (hasLayerAttr(layer, "forget_bias")) - layerParams.set("forget_bias", getLayerAttr(layer, "forget_bias").f()); - - if (hasLayerAttr(layer, "forget_bias")) - { - float cellClip = getLayerAttr(layer, "cell_clip").f(); - // Cell clip disabled if it's negative. - if (cellClip >= 0) - { - layerParams.set("use_cell_clip", true); - layerParams.set("cell_clip", cellClip); - } - } - - Mat W, Wh, Wx, b; - blobFromTensor(getConstBlob(layer, value_id, 4), W); - blobFromTensor(getConstBlob(layer, value_id, 8), b); - const int outSize = W.cols / 4; - - // IGFO->IFOG - float* weightData = (float*)W.data; - for (int i = 0; i < W.rows; ++i) - for (int j = 0; j < outSize; ++j) - { - std::swap(weightData[i * W.cols + 1 * outSize + j], - weightData[i * W.cols + 2 * outSize + j]); - std::swap(weightData[i * W.cols + 2 * outSize + j], - weightData[i * W.cols + 3 * outSize + j]); - } - Wx = W.rowRange(0, W.rows - outSize).t(); - Wh = W.rowRange(W.rows - outSize, W.rows).t(); - - layerParams.blobs.resize(3); - layerParams.blobs[0] = Wh; - layerParams.blobs[1] = Wx; - layerParams.blobs[2] = b; - - if (hasLayerAttr(layer, "use_peephole")) - { - bool usePeephole = getLayerAttr(layer, "use_peephole").b(); - if (usePeephole) - { - layerParams.set("use_peephole", true); - layerParams.blobs.resize(6); - for (int i = 0; i < 3; ++i) - { - Mat w; - blobFromTensor(getConstBlob(layer, value_id, 5 + i), w); - w = w.reshape(1, w.total()); // Single column. - w = Mat::diag(w); // Make a diagonal matrix. - layerParams.blobs[3 + i] = w; - } - } - } - - int id = dstNet.addLayer(name, "LSTM", layerParams); - layer_id[name] = id; - - // one input only - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 0); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; - } - else if (type == "ResizeNearestNeighbor" || type == "ResizeBilinear" || type == "FusedResizeAndPadConv2D") - { - CV_CheckGT(num_inputs, 0, ""); - std::string convWeights = ""; - if (type == "FusedResizeAndPadConv2D") - { - // input: "mul_1" - // input: "decoder/ResizeBilinear/size" - // input: "decoder/decoder_conv0/Conv2D_dummy_paddings" - // input: "decoder/decoder_conv0/weights" - CV_CheckEQ(num_inputs, 4, "Number of input for FusedResizeAndPadConv2D"); - - Mat paddings = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckEQ(countNonZero(paddings), 0, "Unsupported mode"); - - convWeights = layer.input(3); - layer.mutable_input()->DeleteSubrange(2, 2); // FIXIT do NOT modify input model - num_inputs = layer.input_size(); - name = name + "/resize"; - - if (hasLayerAttr(layer, "resize_align_corners")) - { - // FIXIT do NOT modify input model - layer.mutable_attr()->insert( - ::google::protobuf::MapPair("align_corners", - getLayerAttr(layer, "resize_align_corners"))); - } - } - if (num_inputs == 2) - { - Mat outSize = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_CheckTypeEQ(outSize.type(), CV_32SC1, ""); CV_CheckEQ(outSize.total(), (size_t)2, ""); - layerParams.set("height", outSize.at(0, 0)); - layerParams.set("width", outSize.at(0, 1)); - } - else if (num_inputs == 3) - { - Mat factorHeight = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat factorWidth = getTensorContent(getConstBlob(layer, value_id, 2)); - factorHeight.convertTo(factorHeight, CV_32F); - factorWidth.convertTo(factorWidth, CV_32F); - layerParams.set("zoom_factor_x", factorWidth.at(0)); - layerParams.set("zoom_factor_y", factorHeight.at(0)); - } - else - CV_Check(num_inputs, num_inputs == 2 || num_inputs == 3, ""); - - if (type == "ResizeNearestNeighbor") - layerParams.set("interpolation", "nearest"); - else - layerParams.set("interpolation", "bilinear"); - - if (hasLayerAttr(layer, "align_corners")) - layerParams.set("align_corners", getLayerAttr(layer, "align_corners").b()); - - if (hasLayerAttr(layer, "half_pixel_centers")) - layerParams.set("half_pixel_centers", getLayerAttr(layer, "half_pixel_centers").b()); - - int id = dstNet.addLayer(name, "Resize", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - - // Step back to add convolution - if (type == "FusedResizeAndPadConv2D") - { - tensorflow::NodeDef conv = layer_; - conv.clear_input(); - conv.add_input(name); - conv.add_input(convWeights); - conv.set_op("Conv2D"); - parseNode(conv); - } - } - else if (type == "L2Normalize") - { - // op: "L2Normalize" - // input: "input" - // input: "reduction_indices" (axis) - CV_CheckEQ(num_inputs, 2, ""); - Mat reductionIndices = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(reductionIndices.type() == CV_32SC1); - - const int numAxes = reductionIndices.total(); - if (getDataLayout(name, data_layouts) == DATA_LAYOUT_NHWC) - for (int i = 0; i < numAxes; ++i) - reductionIndices.at(i) = toNCHW(reductionIndices.at(i)); - - cv::sort(reductionIndices, reductionIndices, SORT_ASCENDING); - for (int i = 1; i < numAxes; ++i) - { - CV_Assert(reductionIndices.at(i) == reductionIndices.at(i - 1) + 1); - // Axes have the same sign. - CV_Assert(reductionIndices.at(i) * reductionIndices.at(i - 1) >= 0); - } - layerParams.set("start_axis", reductionIndices.at(0)); - layerParams.set("end_axis", reductionIndices.at(numAxes - 1)); - - int id = dstNet.addLayer(name, "Normalize", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "PriorBox") - { - CV_CheckEQ(num_inputs, 2, ""); - if (hasLayerAttr(layer, "min_size")) - layerParams.set("min_size", getLayerAttr(layer, "min_size").i()); - if (hasLayerAttr(layer, "max_size")) - layerParams.set("max_size", getLayerAttr(layer, "max_size").i()); - if (hasLayerAttr(layer, "flip")) - layerParams.set("flip", getLayerAttr(layer, "flip").b()); - if (hasLayerAttr(layer, "clip")) - layerParams.set("clip", getLayerAttr(layer, "clip").b()); - if (hasLayerAttr(layer, "offset")) - layerParams.set("offset", getLayerAttr(layer, "offset").f()); - if (hasLayerAttr(layer, "step")) - layerParams.set("step", getLayerAttr(layer, "step").f()); - - const std::string paramNames[] = {"variance", "aspect_ratio", "scales", - "width", "height"}; - for (int i = 0; i < 5; ++i) - { - if (hasLayerAttr(layer, paramNames[i])) - { - Mat values = getTensorContent(getLayerAttr(layer, paramNames[i]).tensor()); - layerParams.set(paramNames[i], - DictValue::arrayReal((float*)values.data, values.total())); - } - } - int id = dstNet.addLayer(name, "PriorBox", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - data_layouts[name] = DATA_LAYOUT_UNKNOWN; - } - else if (type == "Softmax") - { - CV_CheckGT(num_inputs, 0, ""); - if (hasLayerAttr(layer, "axis")) - layerParams.set("axis", getLayerAttr(layer, "axis").i()); - - int id = dstNet.addLayer(name, "Softmax", layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "CropAndResize") - { - // op: "CropAndResize" - // input: "input" - // input: "boxes" - // input: "sizes" - CV_CheckEQ(num_inputs, 3, ""); - - Mat cropSize = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckTypeEQ(cropSize.type(), CV_32SC1, ""); CV_CheckEQ(cropSize.total(), (size_t)2, ""); - - layerParams.set("height", cropSize.at(0)); - layerParams.set("width", cropSize.at(1)); - - int id = dstNet.addLayer(name, "CropAndResize", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1); - } - else if (type == "Mean" || type == "Sum") - { - // Computes the mean of elements across dimensions of a tensor. - // If keepdims is false (default) reduces input_tensor along the dimensions given in axis, - // else the reduced dimensions are retained with length 1. - // if indices = [1, 2] in NHWC layout we use global pooling: NxCxHxW --Pooling--> NxCx1x1 - // if keepdims is false we use Flatten after Pooling: out_shape = NxC - // if indices = [0] we use a global pooling by indices. - // To return correct shape, we use Reshape after Pooling. To determine input shape use Slice for input, - // if keepdims is false we use Flatten after Slice. - // Example: input_shape = NxCxHxW - // determine out shape: NxCxHxW --Slice--> 1xCxHxW - // out_shape = 1xCxHxW if keepDims else (1xCxHxW --Flatten--> CxHxW) - // global pool: NxCxHxW --Flatten--> Nx(C*H*W) --Reshape--> 1x1xNx(C*H*W) --Pooling--> 1x1x1x(C*H*W) --Reshape--> out_shape - CV_CheckGT(num_inputs, 0, ""); - - Mat indices = getTensorContent(getConstBlob(layer, value_id, 1)); - CV_Assert(indices.type() == CV_32SC1); - - // There are two attributes, "keepdims" and a deprecated "keep_dims". - bool keepDims = false; - if (hasLayerAttr(layer, "keepdims")) - keepDims = getLayerAttr(layer, "keepdims").b(); - else if (hasLayerAttr(layer, "keep_dims")) - keepDims = getLayerAttr(layer, "keep_dims").b(); - - if (indices.total() == 1 && indices.at(0) == 0) - { - LayerParams flattenLp; - std::string flattenName = name + "/flatten"; - CV_Assert(layer_id.find(flattenName) == layer_id.end()); - int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); - layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, parsePin(layer.input(0)), flattenId, 0); - - LayerParams reshapeLp; - std::string reshapeName = name + "/reshape"; - CV_Assert(layer_id.find(reshapeName) == layer_id.end()); - reshapeLp.set("axis", 0); - reshapeLp.set("num_axes", 1); - int newShape[] = {1, 1, -1}; - reshapeLp.set("dim", DictValue::arrayInt(&newShape[0], 3)); - - int reshapeId = dstNet.addLayer(reshapeName, "Reshape", reshapeLp); - layer_id[reshapeName] = reshapeId; - connect(layer_id, dstNet, Pin(flattenName), reshapeId, 0); - - LayerParams avgLp; - std::string avgName = name + "/avg"; - CV_Assert(layer_id.find(avgName) == layer_id.end()); - avgLp.set("pool", type == "Mean" ? "ave" : "sum"); - // pooling kernel H x 1 - avgLp.set("global_pooling_h", true); - avgLp.set("kernel_w", 1); - int avgId = dstNet.addLayer(avgName, "Pooling", avgLp); - layer_id[avgName] = avgId; - connect(layer_id, dstNet, Pin(reshapeName), avgId, 0); - - LayerParams sliceLp; - std::string layerShapeName = name + "/slice"; - CV_Assert(layer_id.find(layerShapeName) == layer_id.end()); - sliceLp.set("axis", 0); - int begin[] = {0}; - int size[] = {1}; - sliceLp.set("begin", DictValue::arrayInt(&begin[0], 1)); - sliceLp.set("size", DictValue::arrayInt(&size[0], 1)); - int sliceId = dstNet.addLayer(layerShapeName, "Slice", sliceLp); - layer_id[layerShapeName] = sliceId; - connect(layer_id, dstNet, Pin(layer.input(0)), sliceId, 0); - - if (!keepDims) - { - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - squeezeLp.set("axis", 0); - squeezeLp.set("end_axis", 1); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(layerShapeName), squeezeId, 0); - layerShapeName = squeezeName; - } - - int id = dstNet.addLayer(name, "Reshape", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, Pin(avgName), id, 0); - connect(layer_id, dstNet, Pin(layerShapeName), id, 1); - } else if (indices.total() == 1) { - int axis = toNCHW(indices.at(0)); - if (axis == 2 || axis == 3) - { - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set(axis == 2 ? "kernel_w" : "kernel_h", 1); - layerParams.set(axis == 2 ? "global_pooling_h" : "global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - - if (!keepDims) - { - // To keep correct order after squeeze dims we first need to change layout from NCHW to NHWC - LayerParams permLP; - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - std::string permName = name + "/nchw"; - Pin inpId = Pin(name); - addPermuteLayer(order, permName, inpId); - - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - squeezeLp.set("axis", indices.at(0)); - squeezeLp.set("end_axis", indices.at(0) + 1); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(permName), squeezeId, 0); - } - } - else if (axis == 1) - { - int order[] = {0, 2, 3, 1}; // From OpenCV's NCHW to NHWC. - Pin inpId = parsePin(layer.input(0)); - addPermuteLayer(order, name + "/nhwc", inpId); - - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set("kernel_h", 1); - layerParams.set("global_pooling_w", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, inpId, id, 0); - - if (!keepDims) - { - LayerParams squeezeLp; - std::string squeezeName = name + "/squeeze"; - CV_Assert(layer_id.find(squeezeName) == layer_id.end()); - int channel_id = 3; // TF NHWC layout - squeezeLp.set("axis", channel_id - 1); - squeezeLp.set("end_axis", channel_id); - int squeezeId = dstNet.addLayer(squeezeName, "Flatten", squeezeLp); - layer_id[squeezeName] = squeezeId; - connect(layer_id, dstNet, Pin(name), squeezeId, 0); - } - else - { - int order[] = {0, 3, 1, 2}; // From NHWC to OpenCV's NCHW. - Pin inpId = parsePin(name); - addPermuteLayer(order, name + "/nchw", inpId); - } - } - } else { - if (indices.total() != 2 || indices.at(0) != 1 || indices.at(1) != 2) - CV_Error(Error::StsNotImplemented, "Unsupported mode of reduce_mean or reduce_sum operation."); - - layerParams.set("pool", type == "Mean" ? "ave" : "sum"); - layerParams.set("global_pooling", true); - int id = dstNet.addLayer(name, "Pooling", layerParams); - layer_id[name] = id; - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - - if (!keepDims) - { - LayerParams flattenLp; - std::string flattenName = name + "/flatten"; - CV_Assert(layer_id.find(flattenName) == layer_id.end()); - int flattenId = dstNet.addLayer(flattenName, "Flatten", flattenLp); - layer_id[flattenName] = flattenId; - connect(layer_id, dstNet, Pin(name), flattenId, 0); - } - } - } - else if (type == "Pack") - { - // op: tf.stack(list of tensors, axis=0) - // Join a list of inputs along a new axis. - // The "axis" specifies the index of the new axis in the dimensions of the output. - // Example: given a list with "N" tensors of shape (C, H, W): - // if axis == 0 then the output tensor will have the shape (N, C, H, W), - // if axis == 1 then the output tensor will have the shape (C, N, H, W). - CV_CheckGT(num_inputs, 0, ""); - CV_Assert(hasLayerAttr(layer, "axis")); - int dim = (int)getLayerAttr(layer, "axis").i(); - if (dim != 0) - CV_Error(Error::StsNotImplemented, "Unsupported mode of pack operation."); - - CV_Assert(hasLayerAttr(layer, "N")); - int num = (int)getLayerAttr(layer, "N").i(); - CV_CheckEQ(num_inputs, num, ""); - std::string base_name = name + "/reshape_"; - std::vector reshape_ids; - for (int i = 0; i < num; i++) { - std::ostringstream ss; - ss << i; - std::string reshape_name = base_name + ss.str(); - LayerParams reshapeLP; - reshapeLP.set("axis", dim); - reshapeLP.set("num_axes", 1); - int outShape[] = {1, -1}; - reshapeLP.set("dim", DictValue::arrayInt(&outShape[0], 2)); - int id = dstNet.addLayer(reshape_name, "Reshape", reshapeLP); - layer_id[reshape_name] = id; - reshape_ids.push_back(id); - connect(layer_id, dstNet, parsePin(layer.input(i)), id, 0); - } - - layerParams.set("axis", dim); - int id = dstNet.addLayer(name, "Concat", layerParams); - layer_id[name] = id; - - for (int li = 0; li < num; li++) - dstNet.connect(reshape_ids[li], 0, id, li); - } - else if (type == "ClipByValue") - { - // op: "ClipByValue" - // input: "input" - // input: "mix" - // input: "max" - CV_CheckEQ(num_inputs, 3, ""); - - Mat minValue = getTensorContent(getConstBlob(layer, value_id, 1)); - Mat maxValue = getTensorContent(getConstBlob(layer, value_id, 2)); - CV_CheckEQ(minValue.total(), (size_t)1, ""); CV_CheckTypeEQ(minValue.type(), CV_32FC1, ""); - CV_CheckEQ(maxValue.total(), (size_t)1, ""); CV_CheckTypeEQ(maxValue.type(), CV_32FC1, ""); - - layerParams.set("min_value", minValue.at(0)); - layerParams.set("max_value", maxValue.at(0)); - - int id = dstNet.addLayer(name, "ReLU6", layerParams); - layer_id[name] = id; - - connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); - } - else if (type == "LeakyRelu") - { - CV_CheckGT(num_inputs, 0, ""); - CV_Assert(hasLayerAttr(layer, "alpha")); - layerParams.set("negative_slope", getLayerAttr(layer, "alpha").f()); - - int id = dstNet.addLayer(name, "ReLU", layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else if (type == "Abs" || type == "Tanh" || type == "Sigmoid" || - type == "Relu" || type == "Elu" || type == "Exp" || - type == "Identity" || type == "Relu6") - { - CV_CheckGT(num_inputs, 0, ""); - std::string dnnType = type; - if (type == "Abs") dnnType = "AbsVal"; - else if (type == "Tanh") dnnType = "TanH"; - else if (type == "Relu") dnnType = "ReLU"; - else if (type == "Relu6") dnnType = "ReLU6"; - else if (type == "Elu") dnnType = "ELU"; - - int id = dstNet.addLayer(name, dnnType, layerParams); - layer_id[name] = id; - connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); - } - else - { - // Importer does not know how to map this TensorFlow's operation onto OpenCV's layer. - // However we create a layer with the same type and rely that user defined a custom layer. - - // All the attributes are added to LayerParams. - google::protobuf::Map attr = layer.attr(); - for (google::protobuf::Map::const_iterator ai = attr.begin(); - ai != attr.end(); ++ai) - { - if (ai->second.value_case() == tensorflow::AttrValue::kS) // string - layerParams.set(ai->first, ai->second.s()); - if (ai->second.value_case() == tensorflow::AttrValue::kI) // int64 - layerParams.set(ai->first, ai->second.i()); - if (ai->second.value_case() == tensorflow::AttrValue::kF) // float - layerParams.set(ai->first, ai->second.f()); - if (ai->second.value_case() == tensorflow::AttrValue::kB) // bool - layerParams.set(ai->first, ai->second.b()); - } - - // All the Const input nodes are added to layer's blobs. - std::vector inputsNames; - for (int i = 0; i < num_inputs; ++i) - { - // Check if input is a Const node. - if (value_id.find(layer.input(i)) != value_id.end()) - { - Mat blob = getTensorContent(getConstBlob(layer, value_id, i)); - layerParams.blobs.push_back(blob); - } - else - inputsNames.push_back(layer.input(i)); - } - int id = dstNet.addLayer(name, type, layerParams); - layer_id[name] = id; - - for (int i = 0; i < inputsNames.size(); ++i) - { - connect(layer_id, dstNet, parsePin(inputsNames[i]), id, i); - } + parseCustomLayer(net, layer, layerParams); } } catch (const std::exception& e) { - CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "'. Exception: " << e.what()); - throw; + CV_LOG_ERROR(NULL, "DNN/TF: Can't parse layer for node='" << name << "' of type='" << type + << "'. Exception: " << e.what()); + + if (DNN_DIAGNOSTICS_RUN) + { + layerHandler->handleFailed(layer); + } + else + { + throw; + } + } +} + +TFLayerHandler::TFLayerHandler(TFImporter* importer_) : importer(importer_) {} + +void TFLayerHandler::fillRegistry(const tensorflow::GraphDef& net) +{ + for (int li = 0; li < net.node_size(); li++) { + const tensorflow::NodeDef& layer = net.node(li); + + const std::string& name = layer.name(); + const std::string& type = layer.op(); + if (importer->dispatch.find(type) == importer->dispatch.end()) + { + addMissing(name, type); + } + } + printMissing(); +}; + +bool TFLayerHandler::handleMissing(const tensorflow::NodeDef& layer) +{ + bool unsupported = contains(layer.op()); + + if (unsupported) + { + handleFailed(layer); + } + + return unsupported; +} + +void TFLayerHandler::handleFailed(const tensorflow::NodeDef& layer) +{ + LayerParams lp = getNotImplementedParams(layer.name(), layer.op()); + + // the layer will be created or its params and type will be replaced + int id = importer->dstNet.addLayer(lp.name, lp.type, lp); + if (id != -1) // internal layer failure before the call to addLayer() + { + importer->layer_id[lp.name] = id; } } @@ -2570,17 +3118,13 @@ void TFImporter::parseNode(const tensorflow::NodeDef& layer_) Net readNetFromTensorflow(const String &model, const String &config) { - Net net; - TFImporter importer(net, model.c_str(), config.c_str()); - return net; + return detail::readNetDiagnostic(model.c_str(), config.c_str()); } Net readNetFromTensorflow(const char* bufferModel, size_t lenModel, const char* bufferConfig, size_t lenConfig) { - Net net; - TFImporter importer(net, bufferModel, lenModel, bufferConfig, lenConfig); - return net; + return detail::readNetDiagnostic(bufferModel, lenModel, bufferConfig, lenConfig); } Net readNetFromTensorflow(const std::vector& bufferModel, const std::vector& bufferConfig) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index aab4c6f507..e8c7e700f6 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -204,7 +204,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe) Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1.5e-2 : 0.0; float iouDiff = (target == DNN_TARGET_MYRIAD) ? 0.063 : 0.0; - float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.252 : FLT_MIN; + float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.262 : FLT_MIN; processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt", inp, "detection_out", "", scoreDiff, iouDiff, detectionConfThresh); expectNoFallbacksFromIE(net); @@ -359,8 +359,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_coco) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_VERSION); #endif - const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.0056 : 0.0; - const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.072 : 0.0; + const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.009 : 0.0; + const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.09 : 0.0; processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt", Size(46, 46), "", "", l1, lInf); expectNoFallbacksFromIE(net); @@ -380,8 +380,8 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi) #endif // output range: [-0.001, 0.97] - const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.012 : 0.0; - const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16) ? 0.16 : 0.0; + const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.02 : 0.0; + const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16) ? 0.2 : 0.0; processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt", Size(46, 46), "", "", l1, lInf); expectNoFallbacksFromIE(net); diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index f2b30c9b87..d822efa889 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -714,6 +714,84 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny) #endif } +TEST_P(Test_Darknet_nets, YOLOv4x_mish) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // NC_OUT_OF_MEMORY + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + // batchId, classId, confidence, left, top, right, bottom + const int N0 = 3; + const int N1 = 5; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.925536f, 0.17188f, 0.386832f, 0.406138f, 0.941696f, +0, 1, 0.912028f, 0.162125f, 0.208863f, 0.741316f, 0.729332f, +0, 7, 0.841018f, 0.608953f, 0.128653f, 0.900692f, 0.295657f, + +1, 2, 0.925697f, 0.650438f, 0.458118f, 0.813927f, 0.661775f, +1, 0, 0.882156f, 0.203644f, 0.365763f, 0.265473f, 0.632195f, +1, 2, 0.848857f, 0.451044f, 0.462997f, 0.496629f, 0.522719f, +1, 9, 0.736015f, 0.374503f, 0.316029f, 0.399358f, 0.392883f, +1, 9, 0.727129f, 0.662469f, 0.373687f, 0.687877f, 0.441335f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + double scoreDiff = 8e-5; + double iouDiff = 3e-4; + + if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CUDA_FP16) + { + scoreDiff = 0.006; + iouDiff = 0.042; + } + + std::string config_file = "yolov4x-mish.cfg"; + std::string weights_file = "yolov4x-mish.weights"; + +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + { + scoreDiff = 0.04; + iouDiff = 0.2; + } +#endif + + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets, dnnBackendsAndTargets()); diff --git a/modules/dnn/test/test_halide_layers.cpp b/modules/dnn/test/test_halide_layers.cpp index 165ee4d67b..a6c1b7d0f8 100644 --- a/modules/dnn/test/test_halide_layers.cpp +++ b/modules/dnn/test/test_halide_layers.cpp @@ -893,7 +893,7 @@ TEST_P(Eltwise, Accuracy) INSTANTIATE_TEST_CASE_P(Layer_Test_Halide, Eltwise, Combine( /*input size*/ Values(Vec3i(1, 4, 5), Vec3i(2, 8, 6)), -/*operation*/ Values("prod", "sum", "div", "max"), +/*operation*/ Values("prod", "sum", "div", "max", "min"), /*num convs*/ Values(1, 2, 3), /*weighted(for sum only)*/ Bool(), dnnBackendsAndTargetsWithHalide() diff --git a/modules/dnn/test/test_ie_models.cpp b/modules/dnn/test/test_ie_models.cpp index b285e91d96..2846f9ae76 100644 --- a/modules/dnn/test/test_ie_models.cpp +++ b/modules/dnn/test/test_ie_models.cpp @@ -112,6 +112,25 @@ static const std::map& getOpenVINOTestMo "intel/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013", "intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013" }}, +#endif +#if INF_ENGINE_RELEASE >= 2021020000 + // OMZ: 2020.2 + { "face-detection-0105", { + "intel/face-detection-0105/FP32/face-detection-0105", + "intel/face-detection-0105/FP16/face-detection-0105" + }}, + { "face-detection-0106", { + "intel/face-detection-0106/FP32/face-detection-0106", + "intel/face-detection-0106/FP16/face-detection-0106" + }}, +#endif +#if INF_ENGINE_RELEASE >= 2021040000 + // OMZ: 2021.4 + { "person-vehicle-bike-detection-2004", { + "intel/person-vehicle-bike-detection-2004/FP32/person-vehicle-bike-detection-2004", + "intel/person-vehicle-bike-detection-2004/FP16/person-vehicle-bike-detection-2004" + //"intel/person-vehicle-bike-detection-2004/FP16-INT8/person-vehicle-bike-detection-2004" + }}, #endif }; @@ -145,10 +164,22 @@ inline static std::string getOpenVINOModel(const std::string &modelName, bool is static inline void genData(const InferenceEngine::TensorDesc& desc, Mat& m, Blob::Ptr& dataPtr) { const std::vector& dims = desc.getDims(); - m.create(std::vector(dims.begin(), dims.end()), CV_32F); - randu(m, -1, 1); - - dataPtr = make_shared_blob(desc, (float*)m.data); + if (desc.getPrecision() == InferenceEngine::Precision::FP32) + { + m.create(std::vector(dims.begin(), dims.end()), CV_32F); + randu(m, -1, 1); + dataPtr = make_shared_blob(desc, (float*)m.data); + } + else if (desc.getPrecision() == InferenceEngine::Precision::I32) + { + m.create(std::vector(dims.begin(), dims.end()), CV_32S); + randu(m, -100, 100); + dataPtr = make_shared_blob(desc, (int*)m.data); + } + else + { + FAIL() << "Unsupported precision: " << desc.getPrecision(); + } } void runIE(Target target, const std::string& xmlPath, const std::string& binPath, @@ -254,7 +285,16 @@ void runIE(Target target, const std::string& xmlPath, const std::string& binPath BlobMap inputBlobs; for (auto& it : net.getInputsInfo()) { - genData(it.second->getTensorDesc(), inputsMap[it.first], inputBlobs[it.first]); + const InferenceEngine::TensorDesc& desc = it.second->getTensorDesc(); + genData(desc, inputsMap[it.first], inputBlobs[it.first]); + if (cvtest::debugLevel > 0) + { + const std::vector& dims = desc.getDims(); + std::cout << "Input: '" << it.first << "' precison=" << desc.getPrecision() << " dims=" << dims.size() << " ["; + for (auto d : dims) + std::cout << " " << d; + std::cout << "] ocv_mat=" << inputsMap[it.first].size << " of " << typeToString(inputsMap[it.first].type()) << std::endl; + } } infRequest.SetInput(inputBlobs); @@ -263,7 +303,16 @@ void runIE(Target target, const std::string& xmlPath, const std::string& binPath BlobMap outputBlobs; for (auto& it : net.getOutputsInfo()) { - genData(it.second->getTensorDesc(), outputsMap[it.first], outputBlobs[it.first]); + const InferenceEngine::TensorDesc& desc = it.second->getTensorDesc(); + genData(desc, outputsMap[it.first], outputBlobs[it.first]); + if (cvtest::debugLevel > 0) + { + const std::vector& dims = desc.getDims(); + std::cout << "Output: '" << it.first << "' precison=" << desc.getPrecision() << " dims=" << dims.size() << " ["; + for (auto d : dims) + std::cout << " " << d; + std::cout << "] ocv_mat=" << outputsMap[it.first].size << " of " << typeToString(outputsMap[it.first].type()) << std::endl; + } } infRequest.SetOutput(outputBlobs); @@ -284,6 +333,12 @@ void runCV(Backend backendId, Target targetId, const std::string& xmlPath, const net.setPreferableTarget(targetId); std::vector outNames = net.getUnconnectedOutLayersNames(); + if (cvtest::debugLevel > 0) + { + std::cout << "OpenCV output names: " << outNames.size() << std::endl; + for (auto name : outNames) + std::cout << "- " << name << std::endl; + } std::vector outs; net.forward(outs, outNames); @@ -307,6 +362,28 @@ TEST_P(DNNTestOpenVINO, models) ASSERT_FALSE(backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) << "Inference Engine backend is required"; +#if INF_ENGINE_VER_MAJOR_GE(2021030000) + if (targetId == DNN_TARGET_MYRIAD && (false + || modelName == "person-detection-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot + || modelName == "age-gender-recognition-retail-0013" // ncDeviceOpen:1013 Failed to find booted device after boot + || modelName == "face-detection-0105" // get_element_type() must be called on a node with exactly one output + || modelName == "face-detection-0106" // get_element_type() must be called on a node with exactly one output + || modelName == "person-vehicle-bike-detection-2004" // 2021.4+: ncDeviceOpen:1013 Failed to find booted device after boot + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (targetId == DNN_TARGET_OPENCL && (false + || modelName == "face-detection-0106" // Operation: 2278 of type ExperimentalDetectronPriorGridGenerator(op::v6) is not supported + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (targetId == DNN_TARGET_OPENCL_FP16 && (false + || modelName == "face-detection-0106" // Operation: 2278 of type ExperimentalDetectronPriorGridGenerator(op::v6) is not supported + ) + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + #if INF_ENGINE_VER_MAJOR_GE(2020020000) if (targetId == DNN_TARGET_MYRIAD && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) { @@ -343,6 +420,8 @@ TEST_P(DNNTestOpenVINO, models) if (targetId == DNN_TARGET_HDDL) releaseHDDLPlugin(); EXPECT_NO_THROW(runIE(targetId, xmlPath, binPath, inputsMap, ieOutputsMap)) << "runIE"; + if (targetId == DNN_TARGET_MYRIAD) + resetMyriadDevice(); EXPECT_NO_THROW(runCV(backendId, targetId, xmlPath, binPath, inputsMap, cvOutputsMap)) << "runCV"; double eps = 0; @@ -350,6 +429,14 @@ TEST_P(DNNTestOpenVINO, models) if (targetId == DNN_TARGET_CPU && checkHardwareSupport(CV_CPU_AVX_512F)) eps = 1e-5; #endif +#if INF_ENGINE_VER_MAJOR_GE(2021030000) + if (targetId == DNN_TARGET_CPU && modelName == "face-detection-0105") + eps = 2e-4; +#endif +#if INF_ENGINE_VER_MAJOR_GE(2021040000) + if (targetId == DNN_TARGET_CPU && modelName == "person-vehicle-bike-detection-2004") + eps = 1e-6; +#endif EXPECT_EQ(ieOutputsMap.size(), cvOutputsMap.size()); for (auto& srcIt : ieOutputsMap) diff --git a/modules/dnn/test/test_int8_layers.cpp b/modules/dnn/test/test_int8_layers.cpp new file mode 100644 index 0000000000..85c30a4271 --- /dev/null +++ b/modules/dnn/test/test_int8_layers.cpp @@ -0,0 +1,1351 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "npy_blob.hpp" +#include +#include +namespace opencv_test { namespace { + +template +static std::string _tf(TString filename) +{ + return (getOpenCVExtraDir() + "dnn/") + filename; +} + +class Test_Int8_layers : public DNNTestLayer +{ +public: + void testLayer(const String& basename, const String& importer, double l1, double lInf, + int numInps = 1, int numOuts = 1, bool useCaffeModel = false, + bool useCommonInputBlob = true, bool hasText = false) + { + CV_Assert_N(numInps >= 1, numInps <= 10, numOuts >= 1, numOuts <= 10); + std::vector inps(numInps), inps_int8(numInps); + std::vector refs(numOuts), outs_int8(numOuts), outs_dequantized(numOuts); + std::vector inputScale, outputScale; + std::vector inputZp, outputZp; + String inpPath, outPath; + Net net, qnet; + + if (importer == "Caffe") + { + String prototxt = _tf("layers/" + basename + ".prototxt"); + String caffemodel = _tf("layers/" + basename + ".caffemodel"); + net = readNetFromCaffe(prototxt, useCaffeModel ? caffemodel : String()); + + inpPath = _tf("layers/" + (useCommonInputBlob ? "blob" : basename + ".input")); + outPath = _tf("layers/" + basename); + } + else if (importer == "TensorFlow") + { + String netPath = _tf("tensorflow/" + basename + "_net.pb"); + String netConfig = hasText ? _tf("tensorflow/" + basename + "_net.pbtxt") : ""; + net = readNetFromTensorflow(netPath, netConfig); + + inpPath = _tf("tensorflow/" + basename + "_in"); + outPath = _tf("tensorflow/" + basename + "_out"); + } + else if (importer == "ONNX") + { + String onnxmodel = _tf("onnx/models/" + basename + ".onnx"); + net = readNetFromONNX(onnxmodel); + + inpPath = _tf("onnx/data/input_" + basename); + outPath = _tf("onnx/data/output_" + basename); + } + ASSERT_FALSE(net.empty()); + net.setPreferableBackend(backend); + net.setPreferableTarget(target); + + for (int i = 0; i < numInps; i++) + inps[i] = blobFromNPY(inpPath + ((numInps > 1) ? cv::format("_%d.npy", i) : ".npy")); + + for (int i = 0; i < numOuts; i++) + refs[i] = blobFromNPY(outPath + ((numOuts > 1) ? cv::format("_%d.npy", i) : ".npy")); + + qnet = net.quantize(inps, CV_8S, CV_8S); + qnet.getInputDetails(inputScale, inputZp); + qnet.getOutputDetails(outputScale, outputZp); + + // Quantize inputs to int8 + // int8_value = float_value/scale + zero-point + for (int i = 0; i < numInps; i++) + { + inps[i].convertTo(inps_int8[i], CV_8S, 1.f/inputScale[i], inputZp[i]); + String inp_name = numInps > 1 ? (importer == "Caffe" ? cv::format("input_%d", i) : cv::format("%d", i)) : ""; + qnet.setInput(inps_int8[i], inp_name); + } + qnet.forward(outs_int8); + + // Dequantize outputs and compare with reference outputs + // float_value = scale*(int8_value - zero-point) + for (int i = 0; i < numOuts; i++) + { + outs_int8[i].convertTo(outs_dequantized[i], CV_32F, outputScale[i], -(outputScale[i] * outputZp[i])); + normAssert(refs[i], outs_dequantized[i], "", l1, lInf); + } + } +}; + +TEST_P(Test_Int8_layers, Convolution1D) +{ + testLayer("conv1d", "ONNX", 0.00302, 0.00909); + testLayer("conv1d_bias", "ONNX", 0.00306, 0.00948); +} + +TEST_P(Test_Int8_layers, Convolution2D) +{ + testLayer("layer_convolution", "Caffe", 0.0174, 0.0758, 1, 1, true); + testLayer("single_conv", "TensorFlow", 0.00413, 0.02201); + testLayer("depthwise_conv2d", "TensorFlow", 0.0388, 0.169); + testLayer("atrous_conv2d_valid", "TensorFlow", 0.0193, 0.0633); + testLayer("atrous_conv2d_same", "TensorFlow", 0.0185, 0.1322); + testLayer("keras_atrous_conv2d_same", "TensorFlow", 0.0056, 0.0244); + testLayer("convolution", "ONNX", 0.0052, 0.01516); + testLayer("two_convolution", "ONNX", 0.00295, 0.00840); +} + +TEST_P(Test_Int8_layers, Convolution3D) +{ + testLayer("conv3d", "TensorFlow", 0.00734, 0.02434); + testLayer("conv3d", "ONNX", 0.00353, 0.00941); + testLayer("conv3d_bias", "ONNX", 0.00129, 0.00249); +} + +TEST_P(Test_Int8_layers, Flatten) +{ + testLayer("flatten", "TensorFlow", 0.0036, 0.0069, 1, 1, false, true, true); + testLayer("unfused_flatten", "TensorFlow", 0.0014, 0.0028); + testLayer("unfused_flatten_unknown_batch", "TensorFlow", 0.0043, 0.0051); +} + +TEST_P(Test_Int8_layers, Padding) +{ + testLayer("padding_valid", "TensorFlow", 0.0026, 0.0064); + testLayer("padding_same", "TensorFlow", 0.0081, 0.032); + testLayer("spatial_padding", "TensorFlow", 0.0078, 0.028); + testLayer("mirror_pad", "TensorFlow", 0.0064, 0.013); + testLayer("pad_and_concat", "TensorFlow", 0.0021, 0.0098); + testLayer("padding", "ONNX", 0.0005, 0.0069); + testLayer("ReflectionPad2d", "ONNX", 0.00062, 0.0018); + testLayer("ZeroPad2d", "ONNX", 0.00037, 0.0018); +} + +TEST_P(Test_Int8_layers, AvePooling) +{ + testLayer("layer_pooling_ave", "Caffe", 0.0021, 0.0075); + testLayer("ave_pool_same", "TensorFlow", 0.00153, 0.0041); + testLayer("average_pooling_1d", "ONNX", 0.002, 0.0048); + testLayer("average_pooling", "ONNX", 0.0014, 0.0032); + testLayer("average_pooling_dynamic_axes", "ONNX", 0.0014, 0.006); + + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("ave_pool3d", "TensorFlow", 0.00175, 0.0047); + testLayer("ave_pool3d", "ONNX", 0.00063, 0.0016); +} + +TEST_P(Test_Int8_layers, MaxPooling) +{ + testLayer("pool_conv_1d", "ONNX", 0.0006, 0.0015); + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("pool_conv_3d", "ONNX", 0.0033, 0.0124); + + /* All the below tests have MaxPooling as last layer, so computeMaxIdx is set to true + which is not supported by int8 maxpooling + testLayer("layer_pooling_max", "Caffe", 0.0021, 0.004); + testLayer("max_pool_even", "TensorFlow", 0.0048, 0.0139); + testLayer("max_pool_odd_valid", "TensorFlow", 0.0043, 0.012); + testLayer("conv_pool_nchw", "TensorFlow", 0.007, 0.025); + testLayer("max_pool3d", "TensorFlow", 0.0025, 0.0058); + testLayer("maxpooling_1d", "ONNX", 0.0018, 0.0037); + testLayer("two_maxpooling_1d", "ONNX", 0.0037, 0.0052); + testLayer("maxpooling", "ONNX", 0.0034, 0.0065); + testLayer("two_maxpooling", "ONNX", 0.0025, 0.0052); + testLayer("max_pool3d", "ONNX", 0.0028, 0.0069);*/ +} + +TEST_P(Test_Int8_layers, Reduce) +{ + testLayer("reduce_mean", "TensorFlow", 0.0005, 0.0014); + testLayer("reduce_mean", "ONNX", 0.00062, 0.0014); + testLayer("reduce_mean_axis1", "ONNX", 0.00032, 0.0007); + testLayer("reduce_mean_axis2", "ONNX", 0.00033, 0.001); + + testLayer("reduce_sum", "TensorFlow", 0.015, 0.031); + testLayer("reduce_sum_channel", "TensorFlow", 0.008, 0.019); + testLayer("sum_pool_by_axis", "TensorFlow", 0.012, 0.032); + testLayer("reduce_sum", "ONNX", 0.0025, 0.0048); + + testLayer("reduce_max", "ONNX", 0, 0); + testLayer("reduce_max_axis_0", "ONNX", 0.0042, 0.007); + testLayer("reduce_max_axis_1", "ONNX", 0.0018, 0.0036); + + if (target != DNN_TARGET_CPU) + throw SkipTestException("Only CPU is supported"); + testLayer("reduce_mean3d", "ONNX", 0.00048, 0.0016); +} + +TEST_P(Test_Int8_layers, ReLU) +{ + testLayer("layer_relu", "Caffe", 0.0005, 0.002); + testLayer("ReLU", "ONNX", 0.0012, 0.0047); +} + +TEST_P(Test_Int8_layers, LeakyReLU) +{ + testLayer("leaky_relu", "TensorFlow", 0.0002, 0.0004); +} + +TEST_P(Test_Int8_layers, ReLU6) +{ + testLayer("keras_relu6", "TensorFlow", 0.0018, 0.0062); + testLayer("keras_relu6", "TensorFlow", 0.0018, 0.0062, 1, 1, false, true, true); + testLayer("clip_by_value", "TensorFlow", 0.0009, 0.002); + testLayer("clip", "ONNX", 0.00006, 0.00037); +} + +TEST_P(Test_Int8_layers, Sigmoid) +{ + testLayer("maxpooling_sigmoid", "ONNX", 0.0011, 0.0032); + testLayer("maxpooling_sigmoid_dynamic_axes", "ONNX", 0.0011, 0.0032); + testLayer("maxpooling_sigmoid_1d", "ONNX", 0.0011, 0.0037); +} + +TEST_P(Test_Int8_layers, Mish) +{ + testLayer("mish", "ONNX", 0.0015, 0.0025); +} + +TEST_P(Test_Int8_layers, Softmax) +{ + testLayer("layer_softmax", "Caffe", 0.0011, 0.0036); + testLayer("keras_softmax", "TensorFlow", 0.00093, 0.0027); + testLayer("slim_softmax", "TensorFlow", 0.0016, 0.0034); + testLayer("slim_softmax_v2", "TensorFlow", 0.0029, 0.017); + testLayer("softmax", "ONNX", 0.0016, 0.0028); + testLayer("log_softmax", "ONNX", 0.014, 0.025); + testLayer("softmax_unfused", "ONNX", 0.0009, 0.0021); +} + +TEST_P(Test_Int8_layers, Concat) +{ + testLayer("layer_concat_shared_input", "Caffe", 0.0076, 0.029, 1, 1, true, false); + testLayer("concat_axis_1", "TensorFlow", 0.0056, 0.017); + testLayer("keras_pad_concat", "TensorFlow", 0.0032, 0.0089); + testLayer("concat_3d", "TensorFlow", 0.005, 0.014); + testLayer("concatenation", "ONNX", 0.0032, 0.009); +} + +TEST_P(Test_Int8_layers, BatchNorm) +{ + testLayer("layer_batch_norm", "Caffe", 0.0061, 0.019, 1, 1, true); + testLayer("fused_batch_norm", "TensorFlow", 0.0063, 0.02); + testLayer("batch_norm_text", "TensorFlow", 0.0048, 0.013, 1, 1, false, true, true); + testLayer("unfused_batch_norm", "TensorFlow", 0.0076, 0.019); + testLayer("fused_batch_norm_no_gamma", "TensorFlow", 0.0067, 0.015); + testLayer("unfused_batch_norm_no_gamma", "TensorFlow", 0.0123, 0.044); + testLayer("switch_identity", "TensorFlow", 0.0035, 0.011); + testLayer("batch_norm3d", "TensorFlow", 0.0077, 0.02); + testLayer("batch_norm", "ONNX", 0.0012, 0.0049); + testLayer("batch_norm_3d", "ONNX", 0.0039, 0.012); + testLayer("frozenBatchNorm2d", "ONNX", 0.001, 0.0018); + testLayer("batch_norm_subgraph", "ONNX", 0.0049, 0.0098); +} + +TEST_P(Test_Int8_layers, Scale) +{ + testLayer("batch_norm", "TensorFlow", 0.0028, 0.0098); + testLayer("scale", "ONNX", 0.0025, 0.0071); + testLayer("expand_hw", "ONNX", 0.0012, 0.0012); + testLayer("flatten_const", "ONNX", 0.0024, 0.0048); +} + +TEST_P(Test_Int8_layers, InnerProduct) +{ + testLayer("layer_inner_product", "Caffe", 0.005, 0.02, 1, 1, true); + testLayer("matmul", "TensorFlow", 0.0061, 0.019); + testLayer("nhwc_transpose_reshape_matmul", "TensorFlow", 0.0009, 0.0091); + testLayer("nhwc_reshape_matmul", "TensorFlow", 0.03, 0.071); + testLayer("matmul_layout", "TensorFlow", 0.035, 0.06); + testLayer("tf2_dense", "TensorFlow", 0, 0); + testLayer("matmul_add", "ONNX", 0.041, 0.082); + testLayer("linear", "ONNX", 0.0018, 0.0029); + testLayer("constant", "ONNX", 0.00021, 0.0006); + testLayer("lin_with_constant", "ONNX", 0.0011, 0.0016); +} + +TEST_P(Test_Int8_layers, Reshape) +{ + testLayer("reshape_layer", "TensorFlow", 0.0032, 0.0082); + testLayer("reshape_nchw", "TensorFlow", 0.0089, 0.029); + testLayer("reshape_conv", "TensorFlow", 0.035, 0.054); + testLayer("reshape_reduce", "TensorFlow", 0.0042, 0.0078); + testLayer("reshape_as_shape", "TensorFlow", 0.0014, 0.0028); + testLayer("reshape_no_reorder", "TensorFlow", 0.0014, 0.0028); + testLayer("shift_reshape_no_reorder", "TensorFlow", 0.0063, 0.014); + testLayer("dynamic_reshape", "ONNX", 0.0047, 0.0079); + testLayer("dynamic_reshape_opset_11", "ONNX", 0.0048, 0.0081); + testLayer("flatten_by_prod", "ONNX", 0.0048, 0.0081); + testLayer("squeeze", "ONNX", 0.0048, 0.0081); + testLayer("unsqueeze", "ONNX", 0.0033, 0.0053); + testLayer("squeeze_and_conv_dynamic_axes", "ONNX", 0.0054, 0.0154); + testLayer("unsqueeze_and_conv_dynamic_axes", "ONNX", 0.0037, 0.0151); +} + +TEST_P(Test_Int8_layers, Permute) +{ + testLayer("tf2_permute_nhwc_ncwh", "TensorFlow", 0.0028, 0.006); + testLayer("transpose", "ONNX", 0.0015, 0.0046); +} + +TEST_P(Test_Int8_layers, Identity) +{ + testLayer("expand_batch", "ONNX", 0.0027, 0.0036); + testLayer("expand_channels", "ONNX", 0.0013, 0.0019); + testLayer("expand_neg_batch", "ONNX", 0.00071, 0.0019); +} + +TEST_P(Test_Int8_layers, Slice) +{ + testLayer("split", "TensorFlow", 0.0033, 0.0056); + testLayer("slice_4d", "TensorFlow", 0.003, 0.0073); + testLayer("strided_slice", "TensorFlow", 0.008, 0.0142); + testLayer("slice", "ONNX", 0.0046, 0.0077); + testLayer("slice_dynamic_axes", "ONNX", 0.0039, 0.0084); + testLayer("slice_opset_11_steps_2d", "ONNX", 0.0052, 0.0124); + testLayer("slice_opset_11_steps_3d", "ONNX", 0.0068, 0.014); + testLayer("slice_opset_11_steps_4d", "ONNX", 0.0041, 0.008); + testLayer("slice_opset_11_steps_5d", "ONNX", 0.0085, 0.021); +} + +TEST_P(Test_Int8_layers, Dropout) +{ + testLayer("layer_dropout", "Caffe", 0.0021, 0.004); + testLayer("dropout", "ONNX", 0.0029, 0.004); +} + +TEST_P(Test_Int8_layers, Eltwise) +{ + testLayer("layer_eltwise", "Caffe", 0.062, 0.15); + testLayer("conv_2_inps", "Caffe", 0.0086, 0.0232, 2, 1, true, false); + testLayer("eltwise_sub", "TensorFlow", 0.015, 0.047); + testLayer("eltwise_add_vec", "TensorFlow", 0.037, 0.21); // tflite 0.0095, 0.0365 + testLayer("eltwise_mul_vec", "TensorFlow", 0.173, 1.14); // tflite 0.0028, 0.017 + testLayer("channel_broadcast", "TensorFlow", 0.0025, 0.0063); + testLayer("split_equals", "TensorFlow", 0.02, 0.065); + testLayer("mul", "ONNX", 0.0039, 0.014); + testLayer("split_max", "ONNX", 0.004, 0.012); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_layers, dnnBackendsAndTargets()); + +class Test_Int8_nets : public DNNTestLayer +{ +public: + void testClassificationNet(Net baseNet, const Mat& blob, const Mat& ref, double l1, double lInf) + { + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob); + Mat out = qnet.forward(); + normAssert(ref, out, "", l1, lInf); + } + + void testDetectionNet(Net baseNet, const Mat& blob, const Mat& ref, + double confThreshold, double scoreDiff, double iouDiff) + { + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob); + Mat out = qnet.forward(); + normAssertDetections(ref, out, "", confThreshold, scoreDiff, iouDiff); + } + + void testFaster(Net baseNet, const Mat& ref, double confThreshold, double scoreDiff, double iouDiff) + { + Mat inp = imread(_tf("dog416.png")); + resize(inp, inp, Size(800, 600)); + Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(102.9801, 115.9465, 122.7717), false, false); + Mat imInfo = (Mat_(1, 3) << inp.rows, inp.cols, 1.6f); + + Net qnet = baseNet.quantize(std::vector{blob, imInfo}, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + + qnet.setInput(blob, "data"); + qnet.setInput(imInfo, "im_info"); + Mat out = qnet.forward(); + normAssertDetections(ref, out, "", confThreshold, scoreDiff, iouDiff); + } + + void testONNXNet(const String& basename, double l1, double lInf, bool useSoftmax = false) + { + String onnxmodel = findDataFile("dnn/onnx/models/" + basename + ".onnx", false); + + Mat blob = readTensorFromONNX(findDataFile("dnn/onnx/data/input_" + basename + ".pb")); + Mat ref = readTensorFromONNX(findDataFile("dnn/onnx/data/output_" + basename + ".pb")); + Net baseNet = readNetFromONNX(onnxmodel); + baseNet.setPreferableBackend(backend); + baseNet.setPreferableTarget(target); + + Net qnet = baseNet.quantize(blob, CV_32F, CV_32F); + qnet.setInput(blob); + Mat out = qnet.forward(); + + if (useSoftmax) + { + LayerParams lp; + Net netSoftmax; + netSoftmax.addLayerToPrev("softmaxLayer", "Softmax", lp); + netSoftmax.setPreferableBackend(DNN_BACKEND_OPENCV); + + netSoftmax.setInput(out); + out = netSoftmax.forward(); + + netSoftmax.setInput(ref); + ref = netSoftmax.forward(); + } + + normAssert(ref, out, "", l1, lInf); + } + + void testDarknetModel(const std::string& cfg, const std::string& weights, + const cv::Mat& ref, double scoreDiff, double iouDiff, + float confThreshold = 0.24, float nmsThreshold = 0.4) + { + CV_Assert(ref.cols == 7); + std::vector > refClassIds; + std::vector > refScores; + std::vector > refBoxes; + for (int i = 0; i < ref.rows; ++i) + { + int batchId = static_cast(ref.at(i, 0)); + int classId = static_cast(ref.at(i, 1)); + float score = ref.at(i, 2); + float left = ref.at(i, 3); + float top = ref.at(i, 4); + float right = ref.at(i, 5); + float bottom = ref.at(i, 6); + Rect2d box(left, top, right - left, bottom - top); + if (batchId >= refClassIds.size()) + { + refClassIds.resize(batchId + 1); + refScores.resize(batchId + 1); + refBoxes.resize(batchId + 1); + } + refClassIds[batchId].push_back(classId); + refScores[batchId].push_back(score); + refBoxes[batchId].push_back(box); + } + + Mat img1 = imread(_tf("dog416.png")); + Mat img2 = imread(_tf("street.png")); + std::vector samples(2); + samples[0] = img1; samples[1] = img2; + + // determine test type, whether batch or single img + int batch_size = refClassIds.size(); + CV_Assert(batch_size == 1 || batch_size == 2); + samples.resize(batch_size); + + Mat inp = blobFromImages(samples, 1.0/255, Size(416, 416), Scalar(), true, false); + + Net baseNet = readNetFromDarknet(findDataFile("dnn/" + cfg), findDataFile("dnn/" + weights, false)); + Net qnet = baseNet.quantize(inp, CV_32F, CV_32F); + qnet.setPreferableBackend(backend); + qnet.setPreferableTarget(target); + qnet.setInput(inp); + std::vector outs; + qnet.forward(outs, qnet.getUnconnectedOutLayersNames()); + + for (int b = 0; b < batch_size; ++b) + { + std::vector classIds; + std::vector confidences; + std::vector boxes; + for (int i = 0; i < outs.size(); ++i) + { + Mat out; + if (batch_size > 1){ + // get the sample slice from 3D matrix (batch, box, classes+5) + Range ranges[3] = {Range(b, b+1), Range::all(), Range::all()}; + out = outs[i](ranges).reshape(1, outs[i].size[1]); + }else{ + out = outs[i]; + } + for (int j = 0; j < out.rows; ++j) + { + Mat scores = out.row(j).colRange(5, out.cols); + double confidence; + Point maxLoc; + minMaxLoc(scores, 0, &confidence, 0, &maxLoc); + + if (confidence > confThreshold) { + float* detection = out.ptr(j); + double centerX = detection[0]; + double centerY = detection[1]; + double width = detection[2]; + double height = detection[3]; + boxes.push_back(Rect2d(centerX - 0.5 * width, centerY - 0.5 * height, + width, height)); + confidences.push_back(confidence); + classIds.push_back(maxLoc.x); + } + } + } + + // here we need NMS of boxes + std::vector indices; + NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices); + + std::vector nms_classIds; + std::vector nms_confidences; + std::vector nms_boxes; + + for (size_t i = 0; i < indices.size(); ++i) + { + int idx = indices[i]; + Rect2d box = boxes[idx]; + float conf = confidences[idx]; + int class_id = classIds[idx]; + nms_boxes.push_back(box); + nms_confidences.push_back(conf); + nms_classIds.push_back(class_id); + } + + if (cvIsNaN(iouDiff)) + { + if (b == 0) + std::cout << "Skip accuracy checks" << std::endl; + continue; + } + + normAssertDetections(refClassIds[b], refScores[b], refBoxes[b], nms_classIds, nms_confidences, nms_boxes, + format("batch size %d, sample %d\n", batch_size, b).c_str(), confThreshold, scoreDiff, iouDiff); + } + } +}; + +TEST_P(Test_Int8_nets, AlexNet) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + if (backend != DNN_BACKEND_OPENCV) + throw SkipTestException("Only OpenCV backend is supported"); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_alexnet.prototxt"), + findDataFile("dnn/bvlc_alexnet.caffemodel", false)); + + Mat inp = imread(_tf("grace_hopper_227.png")); + Mat blob = blobFromImage(inp, 1.0, Size(227, 227), Scalar(), false); + Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy")); + + float l1 = 1e-4, lInf = 0.003; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, GoogLeNet) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromCaffe(findDataFile("dnn/bvlc_googlenet.prototxt"), + findDataFile("dnn/bvlc_googlenet.caffemodel", false)); + + std::vector inpMats; + inpMats.push_back( imread(_tf("googlenet_0.png")) ); + inpMats.push_back( imread(_tf("googlenet_1.png")) ); + Mat blob = blobFromImages(inpMats, 1.0, Size(224, 224), Scalar(), false); + Mat ref = blobFromNPY(_tf("googlenet_prob.npy")); + + float l1 = 2e-4, lInf = 0.06; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, ResNet50) +{ + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); + if (backend != DNN_BACKEND_OPENCV) + throw SkipTestException("Only OpenCV backend is supported"); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromCaffe(findDataFile("dnn/ResNet-50-deploy.prototxt"), + findDataFile("dnn/ResNet-50-model.caffemodel", false)); + + Mat inp = imread(_tf("googlenet_0.png")); + Mat blob = blobFromImage(inp, 1.0, Size(224, 224), Scalar(), false); + Mat ref = blobFromNPY(_tf("resnet50_prob.npy")); + + float l1 = 3e-4, lInf = 0.04; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, DenseNet121) +{ + applyTestTag(CV_TEST_TAG_MEMORY_512MB); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromCaffe(findDataFile("dnn/DenseNet_121.prototxt", false), + findDataFile("dnn/DenseNet_121.caffemodel", false)); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0 / 255.0, Size(224, 224), Scalar(), true, true); + Mat ref = blobFromNPY(_tf("densenet_121_output.npy")); + + float l1 = 0.76, lInf = 3.31; // seems wrong + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, SqueezeNet_v1_1) +{ + if(target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromCaffe(findDataFile("dnn/squeezenet_v1.1.prototxt"), + findDataFile("dnn/squeezenet_v1.1.caffemodel", false)); + + Mat inp = imread(_tf("googlenet_0.png")); + Mat blob = blobFromImage(inp, 1.0, Size(227, 227), Scalar(), false, true); + Mat ref = blobFromNPY(_tf("squeezenet_v1.1_prob.npy")); + + float l1 = 3e-4, lInf = 0.056; + testClassificationNet(net, blob, ref, l1, lInf); +} + +TEST_P(Test_Int8_nets, CaffeNet) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + float l1 = 4e-5, lInf = 0.0025; + testONNXNet("caffenet", l1, lInf); +} + +TEST_P(Test_Int8_nets, RCNN_ILSVRC13) +{ +#if defined(OPENCV_32BIT_CONFIGURATION) && (defined(HAVE_OPENCL) || defined(_WIN32)) + applyTestTag(CV_TEST_TAG_MEMORY_2GB); +#else + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#endif + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019030000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + float l1 = 0.02, lInf = 0.042; + testONNXNet("rcnn_ilsvrc13", l1, lInf); +} + +TEST_P(Test_Int8_nets, Inception_v2) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + testONNXNet("inception_v2", default_l1, default_lInf, true); +} + +TEST_P(Test_Int8_nets, MobileNet_v2) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + testONNXNet("mobilenetv2", default_l1, default_lInf, true); +} + +TEST_P(Test_Int8_nets, Shufflenet) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + } + testONNXNet("shufflenet", default_l1, default_lInf); +} + +TEST_P(Test_Int8_nets, MobileNet_SSD) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromCaffe(findDataFile("dnn/MobileNetSSD_deploy.prototxt", false), + findDataFile("dnn/MobileNetSSD_deploy.caffemodel", false)); + + Mat inp = imread(_tf("street.png")); + Mat blob = blobFromImage(inp, 1.0 / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false); + Mat ref = blobFromNPY(_tf("mobilenet_ssd_caffe_out.npy")); + + float confThreshold = FLT_MIN, scoreDiff = 0.059, iouDiff = 0.11; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, MobileNet_v1_SSD) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false), + findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.034, iouDiff = 0.13; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, MobileNet_v1_SSD_PPN) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, + CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false), + findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy")); + + float confThreshold = 0.51, scoreDiff = 0.05, iouDiff = 0.06; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, Inception_v2_SSD) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB); +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + Net net = readNetFromTensorflow(findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false), + findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt")); + + Mat inp = imread(_tf("street.png")); + Mat blob = blobFromImage(inp, 1.0, Size(300, 300), Scalar(), true, false); + Mat ref = (Mat_(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729, + 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131, + 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015, + 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527, + 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384); + + float confThreshold = 0.5, scoreDiff = 0.0114, iouDiff = 0.22; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, opencv_face_detector) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + Net net = readNetFromCaffe(findDataFile("dnn/opencv_face_detector.prototxt"), + findDataFile("dnn/opencv_face_detector.caffemodel", false)); + + Mat inp = imread(findDataFile("gpu/lbpcascade/er.png")); + Mat blob = blobFromImage(inp, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false); + Mat ref = (Mat_(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631, + 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168, + 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290, + 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477, + 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494, + 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801); + + float confThreshold = 0.5, scoreDiff = 0.002, iouDiff = 0.21; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, EfficientDet) +{ + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + if (target != DNN_TARGET_CPU) + { + if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + } + Net net = readNetFromTensorflow(findDataFile("dnn/efficientdet-d0.pb", false), + findDataFile("dnn/efficientdet-d0.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0/255, Size(512, 512), Scalar(123.675, 116.28, 103.53)); + Mat ref = (Mat_(3, 7) << 0, 1, 0.8437444, 0.153996080160141, 0.20534580945968628, 0.7463544607162476, 0.7414066195487976, + 0, 17, 0.8245924, 0.16657517850399017, 0.3996818959712982, 0.4111558794975281, 0.9306337833404541, + 0, 7, 0.8039304, 0.6118435263633728, 0.13175517320632935, 0.9065558314323425, 0.2943994700908661); + + float confThreshold = 0.65, scoreDiff = 0.17, iouDiff = 0.18; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_resnet50) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#ifdef INF_ENGINE_RELEASE + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && + (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (INF_ENGINE_VER_MAJOR_GT(2019030000) && + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pb", false), + findDataFile("dnn/faster_rcnn_resnet50_coco_2018_01_28.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_resnet50_coco_2018_01_28.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.05, iouDiff = 0.15; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_inceptionv2) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#ifdef INF_ENGINE_RELEASE + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && + (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (INF_ENGINE_VER_MAJOR_GT(2019030000) && + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + if (backend == DNN_BACKEND_CUDA && target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromTensorflow(findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false), + findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt")); + + Mat inp = imread(_tf("dog416.png")); + Mat blob = blobFromImage(inp, 1.0, Size(800, 600), Scalar(), true, false); + Mat ref = blobFromNPY(_tf("tensorflow/faster_rcnn_inception_v2_coco_2018_01_28.detection_out.npy")); + + float confThreshold = 0.5, scoreDiff = 0.21, iouDiff = 0.1; + testDetectionNet(net, blob, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_vgg16) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB), +#endif + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)) + applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); +#endif + + Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_vgg16.prototxt"), + findDataFile("dnn/VGG16_faster_rcnn_final.caffemodel", false)); + + Mat ref = (Mat_(3, 7) << 0, 2, 0.949398, 99.2454, 210.141, 601.205, 462.849, + 0, 7, 0.997022, 481.841, 92.3218, 722.685, 175.953, + 0, 12, 0.993028, 133.221, 189.377, 350.994, 563.166); + + float confThreshold = 0.8, scoreDiff = 0.024, iouDiff = 0.35; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, FasterRCNN_zf) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB), +#endif + CV_TEST_TAG_DEBUG_LONG + ); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + + if (target == DNN_TARGET_CUDA_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA_FP16); + + Net net = readNetFromCaffe(findDataFile("dnn/faster_rcnn_zf.prototxt"), + findDataFile("dnn/ZF_faster_rcnn_final.caffemodel", false)); + + Mat ref = (Mat_(3, 7) << 0, 2, 0.90121, 120.407, 115.83, 570.586, 528.395, + 0, 7, 0.988779, 469.849, 75.1756, 718.64, 186.762, + 0, 12, 0.967198, 138.588, 206.843, 329.766, 553.176); + + float confThreshold = 0.8, scoreDiff = 0.021, iouDiff = 0.1; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, RFCN) +{ + applyTestTag( + (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_2GB), + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG + ); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); + + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD); + + Net net = readNetFromCaffe(findDataFile("dnn/rfcn_pascal_voc_resnet50.prototxt"), + findDataFile("dnn/resnet50_rfcn_final.caffemodel", false)); + + Mat ref = (Mat_(2, 7) << 0, 7, 0.991359, 491.822, 81.1668, 702.573, 178.234, + 0, 12, 0.94786, 132.093, 223.903, 338.077, 566.16); + + float confThreshold = 0.8, scoreDiff = 0.017, iouDiff = 0.11; + testFaster(net, ref, confThreshold, scoreDiff, iouDiff); +} + +TEST_P(Test_Int8_nets, YoloVoc) +{ + applyTestTag( +#if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL) + CV_TEST_TAG_MEMORY_2GB, +#else + CV_TEST_TAG_MEMORY_1GB, +#endif + CV_TEST_TAG_LONG + ); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16); +#endif +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && + target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); +#endif + + Mat ref = (Mat_(6, 7) << 0, 6, 0.750469f, 0.577374f, 0.127391f, 0.902949f, 0.300809f, + 0, 1, 0.780879f, 0.270762f, 0.264102f, 0.732475f, 0.745412f, + 0, 11, 0.901615f, 0.1386f, 0.338509f, 0.421337f, 0.938789f, + 1, 14, 0.623813f, 0.183179f, 0.381921f, 0.247726f, 0.625847f, + 1, 6, 0.667770f, 0.446555f, 0.453578f, 0.499986f, 0.519167f, + 1, 6, 0.844947f, 0.637058f, 0.460398f, 0.828508f, 0.66427f); + + std::string config_file = "yolo-voc.cfg"; + std::string weights_file = "yolo-voc.weights"; + + double scoreDiff = 0.1, iouDiff = 0.3; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, 3), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, TinyYoloVoc) +{ + applyTestTag(CV_TEST_TAG_MEMORY_512MB); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && + target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); +#endif + + Mat ref = (Mat_(4, 7) << 0, 6, 0.761967f, 0.579042f, 0.159161f, 0.894482f, 0.31994f, + 0, 11, 0.780595f, 0.129696f, 0.386467f, 0.445275f, 0.920994f, + 1, 6, 0.651450f, 0.460526f, 0.458019f, 0.522527f, 0.5341f, + 1, 6, 0.928758f, 0.651024f, 0.463539f, 0.823784f, 0.654998f); + + std::string config_file = "tiny-yolo-voc.cfg"; + std::string weights_file = "tiny-yolo-voc.weights"; + + double scoreDiff = 0.043, iouDiff = 0.12; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, YOLOv3) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + + const int N0 = 3; + const int N1 = 6; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.998836f, 0.160024f, 0.389964f, 0.417885f, 0.943716f, +0, 1, 0.987908f, 0.150913f, 0.221933f, 0.742255f, 0.746261f, +0, 7, 0.952983f, 0.614621f, 0.150257f, 0.901368f, 0.289251f, + +1, 2, 0.997412f, 0.647584f, 0.459939f, 0.821037f, 0.663947f, +1, 2, 0.989633f, 0.450719f, 0.463353f, 0.496306f, 0.522258f, +1, 0, 0.980053f, 0.195856f, 0.378454f, 0.258626f, 0.629257f, +1, 9, 0.785341f, 0.665503f, 0.373543f, 0.688893f, 0.439244f, +1, 9, 0.733275f, 0.376029f, 0.315694f, 0.401776f, 0.395165f, +1, 9, 0.384815f, 0.659824f, 0.372389f, 0.673927f, 0.429412f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov3.cfg"; + std::string weights_file = "yolov3.weights"; + + double scoreDiff = 0.08, iouDiff = 0.21, confThreshold = 0.25; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); + } + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); + } +} + +TEST_P(Test_Int8_nets, YOLOv4) +{ + applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB)); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + const int N0 = 3; + const int N1 = 7; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 16, 0.992194f, 0.172375f, 0.402458f, 0.403918f, 0.932801f, +0, 1, 0.988326f, 0.166708f, 0.228236f, 0.737208f, 0.735803f, +0, 7, 0.94639f, 0.602523f, 0.130399f, 0.901623f, 0.298452f, + +1, 2, 0.99761f, 0.646556f, 0.45985f, 0.816041f, 0.659067f, +1, 0, 0.988913f, 0.201726f, 0.360282f, 0.266181f, 0.631728f, +1, 2, 0.98233f, 0.452007f, 0.462217f, 0.495612f, 0.521687f, +1, 9, 0.919195f, 0.374642f, 0.316524f, 0.398126f, 0.393714f, +1, 9, 0.856303f, 0.666842f, 0.372215f, 0.685539f, 0.44141f, +1, 9, 0.313516f, 0.656791f, 0.374734f, 0.671959f, 0.438371f, +1, 9, 0.256625f, 0.940232f, 0.326931f, 0.967586f, 0.374002f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov4.cfg"; + std::string weights_file = "yolov4.weights"; + double scoreDiff = 0.15, iouDiff = 0.2; + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff); + } + + { + SCOPED_TRACE("batch size 2"); + +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + { + if (target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + else if (target == DNN_TARGET_MYRIAD && + getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + } +#endif + + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff); + } +} + +TEST_P(Test_Int8_nets, YOLOv4_tiny) +{ + applyTestTag( + target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB + ); + + if (target == DNN_TARGET_OPENCL_FP16 && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + if (target == DNN_TARGET_OPENCL && !ocl::Device::getDefault().isIntel()) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL); + +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000) + if (target == DNN_TARGET_MYRIAD) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + const float confThreshold = 0.6; + + const int N0 = 2; + const int N1 = 3; + static const float ref_[/* (N0 + N1) * 7 */] = { +0, 7, 0.85935f, 0.593484f, 0.141211f, 0.920356f, 0.291593f, +0, 16, 0.795188f, 0.169207f, 0.386886f, 0.423753f, 0.933004f, + +1, 2, 0.996832f, 0.653802f, 0.464573f, 0.815193f, 0.653292f, +1, 2, 0.963325f, 0.451151f, 0.458915f, 0.496255f, 0.52241f, +1, 0, 0.926244f, 0.194851f, 0.361743f, 0.260277f, 0.632364f, + }; + Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_); + + std::string config_file = "yolov4-tiny.cfg"; + std::string weights_file = "yolov4-tiny.weights"; + double scoreDiff = 0.12; + double iouDiff = target == DNN_TARGET_OPENCL_FP16 ? 0.2 : 0.082; + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // bad accuracy + iouDiff = std::numeric_limits::quiet_NaN(); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) + iouDiff = std::numeric_limits::quiet_NaN(); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + iouDiff = std::numeric_limits::quiet_NaN(); +#endif + + { + SCOPED_TRACE("batch size 1"); + testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold); + } + + /* bad accuracy on second image + { + SCOPED_TRACE("batch size 2"); + testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold); + } + */ + +#if defined(INF_ENGINE_RELEASE) + if (target == DNN_TARGET_MYRIAD) // bad accuracy + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_Int8_nets, dnnBackendsAndTargets()); +}} // namespace diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp index 20d3fb41eb..ee0629b853 100644 --- a/modules/dnn/test/test_layers.cpp +++ b/modules/dnn/test/test_layers.cpp @@ -445,7 +445,7 @@ class Layer_LSTM_Test : public ::testing::Test { public: int numInp, numOut; - Mat Wh, Wx, b; + Mat Wh, Wx, b, h, c; Ptr layer; std::vector inputs, outputs; @@ -460,12 +460,17 @@ public: Wh = Mat::ones(4 * numOut, numOut, CV_32F); Wx = Mat::ones(4 * numOut, numInp, CV_32F); b = Mat::ones(4 * numOut, 1, CV_32F); + h = Mat::ones(4, numOut, CV_32F); + c = Mat::ones(4, numOut, CV_32F); LayerParams lp; - lp.blobs.resize(3); + lp.blobs.resize(5); lp.blobs[0] = Wh; lp.blobs[1] = Wx; lp.blobs[2] = b; + lp.blobs[3] = h; + lp.blobs[4] = c; + lp.set("produce_cell_output", produceCellOutput); lp.set("use_timestamp_dim", useTimestampDim); @@ -513,10 +518,12 @@ TEST_F(Layer_LSTM_Test, get_set_test) TEST(Layer_LSTM_Test_Accuracy_with_, CaffeRecurrent) { LayerParams lp; - lp.blobs.resize(3); + lp.blobs.resize(5); lp.blobs[0] = blobFromNPY(_tf("lstm.prototxt.w_2.npy")); // Wh lp.blobs[1] = blobFromNPY(_tf("lstm.prototxt.w_0.npy")); // Wx lp.blobs[2] = blobFromNPY(_tf("lstm.prototxt.w_1.npy")); // bias + lp.blobs[3] = Mat::zeros(2, 17, CV_32F); // h_0 + lp.blobs[4] = Mat::zeros(2, 17, CV_32F); // c_0 Ptr layer = LSTMLayer::create(lp); Mat inp = blobFromNPY(_tf("recurrent.input.npy")); @@ -527,6 +534,97 @@ TEST(Layer_LSTM_Test_Accuracy_with_, CaffeRecurrent) normAssert(h_t_reference, outputs[0]); } +TEST(Layer_LSTM_Test_Accuracy_with_, HiddenParams) +{ + Mat Wx = blobFromNPY(_tf("lstm.hidden.W.npy")); + Mat Wh = blobFromNPY(_tf("lstm.hidden.R.npy")); + Mat b = blobFromNPY(_tf("lstm.hidden.B.npy")); + Mat h0 = blobFromNPY(_tf("lstm.hidden.h0.npy")); + Mat c0 = blobFromNPY(_tf("lstm.hidden.c0.npy")); + + const int numHidden = 3; + const int numDirs = Wx.size[0]; + const int numFeatures = Wx.size[2]; + + b = b.reshape(1, b.size[0]); + Mat bx = b.colRange(0, b.cols / 2); + Mat bh = b.colRange(b.cols / 2, b.cols); + b = bx + bh; + + // IFGO->IGFO + for (int k = 0; k < numDirs; ++k) + { + float* WxData = Wx.ptr(k); + float* WhData = Wh.ptr(k); + float* biasData = b.ptr(k); + for (int j = 0; j < numHidden; ++j) + { + for (int i = 0; i < numFeatures; ++i) + { + std::swap(WxData[(numHidden + j) * numFeatures + i], + WxData[(numHidden * 2 + j) * numFeatures + i]); + } + for (int i = 0; i < numHidden; ++i) + { + std::swap(WhData[(numHidden + j) * numHidden + i], + WhData[(numHidden * 2 + j) * numHidden + i]); + } + std::swap(biasData[numHidden + j], biasData[numHidden * 2 + j]); + } + } + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + c0 = c0.reshape(1, c0.size[0] * c0.size[1]); + + LayerParams lstmParams; + lstmParams.blobs.resize(5); + lstmParams.blobs[0] = Wh; + lstmParams.blobs[1] = Wx; + lstmParams.blobs[2] = b; + lstmParams.blobs[3] = h0; + lstmParams.blobs[4] = c0; + lstmParams.set("bidirectional", false); + Ptr layer = LSTMLayer::create(lstmParams); + + Mat inp = blobFromNPY(_tf("lstm.hidden.input.npy")); + std::vector inputs(1, inp), outputs; + runLayer(layer, inputs, outputs); + + Mat h_t_reference = blobFromNPY(_tf("lstm.hidden.output.npy")); + normAssert(h_t_reference, outputs[0]); +} + +TEST(Layer_GRU_Test_Accuracy_with_, Pytorch) +{ + Mat Wx = blobFromNPY(_tf("gru.W.npy")); + Mat Wh = blobFromNPY(_tf("gru.R.npy")); + Mat b = blobFromNPY(_tf("gru.B.npy")); + Mat h0 = blobFromNPY(_tf("gru.h0.npy")); + + Wx = Wx.reshape(1, Wx.size[0] * Wx.size[1]); + Wh = Wh.reshape(1, Wh.size[0] * Wh.size[1]); + h0 = h0.reshape(1, h0.size[0] * h0.size[1]); + b = b.reshape(1, b.size[0]); + + LayerParams gruParams; + gruParams.blobs.resize(4); + gruParams.blobs[0] = Wh; + gruParams.blobs[1] = Wx; + gruParams.blobs[2] = b; + gruParams.blobs[3] = h0; + gruParams.set("bidirectional", false); + Ptr layer = GRULayer::create(gruParams); + + Mat inp = blobFromNPY(_tf("gru.input.npy")); + std::vector inputs(1, inp), outputs; + runLayer(layer, inputs, outputs); + + Mat h_t_reference = blobFromNPY(_tf("gru.output.npy")); + normAssert(h_t_reference, outputs[0]); +} + TEST(Layer_RNN_Test_Accuracy_with_, CaffeRecurrent) { Ptr layer = RNNLayer::create(LayerParams()); @@ -571,6 +669,9 @@ TEST(Layer_LSTM_Test_Accuracy_, Reverse) bias.at(2, 0) = 1e10f; // Output gate - always output everything bias.at(3, 0) = 0.f; // Update signal + cv::Mat hInternal = cv::Mat::zeros(1, 1, CV_32FC1); + cv::Mat cInternal = cv::Mat::zeros(1, 1, CV_32FC1); + LayerParams lp; lp.set("reverse", true); lp.set("use_timestamp_dim", true); @@ -578,6 +679,8 @@ TEST(Layer_LSTM_Test_Accuracy_, Reverse) lp.blobs.push_back(Wh); lp.blobs.push_back(Wx); lp.blobs.push_back(bias); + lp.blobs.push_back(hInternal); + lp.blobs.push_back(cInternal); cv::Ptr layer = LSTMLayer::create(lp); std::vector outputs; @@ -2237,7 +2340,7 @@ public: static testing::internal::ParamGenerator eltwiseOpList() { // TODO: automate list generation - return Values("sum", "max", "prod", "div"); + return Values("sum", "max", "min", "prod", "div"); } static testing::internal::ParamGenerator activationLayersList() diff --git a/modules/dnn/test/test_model.cpp b/modules/dnn/test/test_model.cpp index f7befa9937..6ac9702c69 100644 --- a/modules/dnn/test/test_model.cpp +++ b/modules/dnn/test/test_model.cpp @@ -615,6 +615,25 @@ TEST_P(Test_Model, TextRecognition) testTextRecognitionModel(weightPath, "", imgPath, seq, decodeType, vocabulary, size, mean, scale); } +TEST_P(Test_Model, TextRecognitionWithCTCPrefixBeamSearch) +{ + if (target == DNN_TARGET_OPENCL_FP16) + applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16); + + std::string imgPath = _tf("text_rec_test.png"); + std::string weightPath = _tf("onnx/models/crnn.onnx", false); + std::string seq = "welcome"; + + Size size{100, 32}; + double scale = 1.0 / 127.5; + Scalar mean = Scalar(127.5); + std::string decodeType = "CTC-prefix-beam-search"; + std::vector vocabulary = {"0","1","2","3","4","5","6","7","8","9", + "a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"}; + + testTextRecognitionModel(weightPath, "", imgPath, seq, decodeType, vocabulary, size, mean, scale); +} + TEST_P(Test_Model, TextDetectionByDB) { if (target == DNN_TARGET_OPENCL_FP16) diff --git a/modules/dnn/test/test_nms.cpp b/modules/dnn/test/test_nms.cpp index 53cb3500ec..5ba240287c 100644 --- a/modules/dnn/test/test_nms.cpp +++ b/modules/dnn/test/test_nms.cpp @@ -37,4 +37,41 @@ TEST(NMS, Accuracy) ASSERT_EQ(indices[i], ref_indices[i]); } +TEST(SoftNMS, Accuracy) +{ + //reference results are obtained using TF v2.7 tf.image.non_max_suppression_with_scores + std::string dataPath = findDataFile("dnn/soft_nms_reference.yml"); + FileStorage fs(dataPath, FileStorage::READ); + + std::vector bboxes; + std::vector scores; + std::vector ref_indices; + std::vector ref_updated_scores; + + fs["boxes"] >> bboxes; + fs["probs"] >> scores; + fs["indices"] >> ref_indices; + fs["updated_scores"] >> ref_updated_scores; + + std::vector updated_scores; + const float score_thresh = .01f; + const float nms_thresh = .5f; + std::vector indices; + const size_t top_k = 0; + const float sigma = 1.; // sigma in TF is being multiplied by 2, so 0.5 should be passed there + cv::dnn::softNMSBoxes(bboxes, scores, updated_scores, score_thresh, nms_thresh, indices, top_k, sigma); + + ASSERT_EQ(ref_indices.size(), indices.size()); + for(size_t i = 0; i < indices.size(); i++) + { + ASSERT_EQ(indices[i], ref_indices[i]); + } + + ASSERT_EQ(ref_updated_scores.size(), updated_scores.size()); + for(size_t i = 0; i < updated_scores.size(); i++) + { + EXPECT_NEAR(updated_scores[i], ref_updated_scores[i], 1e-7); + } +} + }} // namespace diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 22a504df69..5909f2ce6e 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -112,6 +112,7 @@ TEST_P(Test_ONNX_layers, MaxPooling_2) TEST_P(Test_ONNX_layers, Convolution) { testONNXModels("convolution"); + testONNXModels("conv_asymmetric_pads"); } TEST_P(Test_ONNX_layers, Convolution_variable_weight) @@ -266,6 +267,11 @@ TEST_P(Test_ONNX_layers, ReLU) testONNXModels("ReLU"); } +TEST_P(Test_ONNX_layers, PReLU) +{ + testONNXModels("PReLU_slope"); +} + TEST_P(Test_ONNX_layers, Clip) { testONNXModels("clip", npy); @@ -295,11 +301,18 @@ TEST_P(Test_ONNX_layers, ReduceMax) testONNXModels("reduce_max_axis_1"); } +TEST_P(Test_ONNX_layers, Min) +{ + testONNXModels("min", npy, 0, 0, false, true, 2); +} + TEST_P(Test_ONNX_layers, Scale) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); testONNXModels("scale"); + testONNXModels("scale_broadcast", npy, 0, 0, false, true, 3); + testONNXModels("scale_broadcast_mid", npy, 0, 0, false, true, 2); } TEST_P(Test_ONNX_layers, ReduceMean3D) @@ -349,6 +362,7 @@ TEST_P(Test_ONNX_layers, Concatenation) if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); } testONNXModels("concatenation"); + testONNXModels("concat_const_blobs"); } TEST_P(Test_ONNX_layers, Eltwise3D) @@ -502,6 +516,8 @@ TEST_P(Test_ONNX_layers, MatMulAdd) TEST_P(Test_ONNX_layers, Expand) { + testONNXModels("expand"); + testONNXModels("expand_identity"); testONNXModels("expand_batch"); testONNXModels("expand_channels"); testONNXModels("expand_neg_batch"); @@ -584,6 +600,11 @@ TEST_P(Test_ONNX_layers, DynamicResize) testONNXModels("dynamic_resize_scale_11", npy, 0, 0, false, true, 2); } +TEST_P(Test_ONNX_layers, Resize_HumanSeg) +{ + testONNXModels("resize_humanseg"); +} + TEST_P(Test_ONNX_layers, Div) { const String model = _tf("models/div.onnx"); @@ -624,6 +645,7 @@ TEST_P(Test_ONNX_layers, DynamicReshape) TEST_P(Test_ONNX_layers, Reshape) { testONNXModels("unsqueeze"); + testONNXModels("unsqueeze_opset_13"); } TEST_P(Test_ONNX_layers, Squeeze) @@ -638,6 +660,7 @@ TEST_P(Test_ONNX_layers, ReduceL2) testONNXModels("reduceL2"); testONNXModels("reduceL2_subgraph"); testONNXModels("reduceL2_subgraph_2"); + testONNXModels("reduceL2_subgraph2_2"); } TEST_P(Test_ONNX_layers, Split) @@ -650,6 +673,8 @@ TEST_P(Test_ONNX_layers, Split) testONNXModels("split_2"); testONNXModels("split_3"); testONNXModels("split_4"); + testONNXModels("split_sizes"); + testONNXModels("split_neg_axis"); } TEST_P(Test_ONNX_layers, Slice) @@ -658,6 +683,7 @@ TEST_P(Test_ONNX_layers, Slice) testONNXModels("slice", npy, 0, 0, false, false); #else testONNXModels("slice"); + testONNXModels("slice_neg_starts"); testONNXModels("slice_opset_11"); #endif } @@ -698,6 +724,11 @@ TEST_P(Test_ONNX_layers, Split_EltwiseMax) testONNXModels("split_max"); } +TEST_P(Test_ONNX_layers, LSTM_Activations) +{ + testONNXModels("lstm_cntk_tanh", pb, 0, 0, false, false); +} + TEST_P(Test_ONNX_layers, LSTM) { testONNXModels("lstm", npy, 0, 0, false, false); @@ -708,6 +739,26 @@ TEST_P(Test_ONNX_layers, LSTM_bidirectional) testONNXModels("lstm_bidirectional", npy, 0, 0, false, false); } +TEST_P(Test_ONNX_layers, LSTM_hidden) +{ + testONNXModels("hidden_lstm", npy, 0, 0, false, false); +} + +TEST_P(Test_ONNX_layers, LSTM_hidden_bidirectional) +{ + testONNXModels("hidden_lstm_bi", npy, 0, 0, false, false); +} + +TEST_P(Test_ONNX_layers, GRU) +{ + testONNXModels("gru", npy, 0, 0, false, false); +} + +TEST_P(Test_ONNX_layers, GRU_bidirectional) +{ + testONNXModels("gru_bi", npy, 0, 0, false, false); +} + TEST_P(Test_ONNX_layers, Pad2d_Unfused) { testONNXModels("ReflectionPad2d"); @@ -853,6 +904,7 @@ TEST_P(Test_ONNX_layers, DynamicAxes) testONNXModels("resize_opset11_torch1.6_dynamic_axes"); testONNXModels("average_pooling_dynamic_axes"); testONNXModels("maxpooling_sigmoid_dynamic_axes"); + testONNXModels("dynamic_batch"); } TEST_P(Test_ONNX_layers, MaxPool1d) @@ -939,6 +991,120 @@ TEST_P(Test_ONNX_layers, ConvResizePool1d) testONNXModels("conv_resize_pool_1d"); } +TEST_P(Test_ONNX_layers, SubFromConst) +{ + testONNXModels("sub_from_const1"); + testONNXModels("sub_from_const_eltwise"); + testONNXModels("sub_from_const_broadcast"); +} + + +TEST_P(Test_ONNX_layers, Quantized_Convolution) +{ + testONNXModels("quantized_conv_uint8_weights", npy, 0.004, 0.02); + testONNXModels("quantized_conv_int8_weights", npy, 0.03, 0.5); + testONNXModels("quantized_conv_per_channel_weights", npy, 0.06, 0.4); +} + +TEST_P(Test_ONNX_layers, Quantized_MatMul) +{ + testONNXModels("quantized_matmul_uint8_weights", npy, 0.005, 0.007); + testONNXModels("quantized_matmul_int8_weights", npy, 0.06, 0.2); + testONNXModels("quantized_matmul_per_channel_weights", npy, 0.06, 0.22); +} + +TEST_P(Test_ONNX_layers, Quantized_MatMul_Variable_Weights) +{ + // Unsupported + EXPECT_THROW( + { + testONNXModels("quantized_matmul_variable_inputs"); + }, cv::Exception); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise) +{ + testONNXModels("quantized_eltwise"); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise_Scalar) +{ + testONNXModels("quantized_eltwise_scalar"); +} + +TEST_P(Test_ONNX_layers, Quantized_Eltwise_Broadcast) +{ + testONNXModels("quantized_eltwise_broadcast"); +} + +TEST_P(Test_ONNX_layers, Quantized_LeakyReLU) +{ + testONNXModels("quantized_leaky_relu"); +} + +TEST_P(Test_ONNX_layers, Quantized_Sigmoid) +{ + testONNXModels("quantized_sigmoid"); +} + +TEST_P(Test_ONNX_layers, Quantized_MaxPool) +{ + testONNXModels("quantized_maxpool"); +} + +TEST_P(Test_ONNX_layers, Quantized_AvgPool) +{ + testONNXModels("quantized_avgpool"); +} + +TEST_P(Test_ONNX_layers, Quantized_Split) +{ + testONNXModels("quantized_split"); +} + +TEST_P(Test_ONNX_layers, Quantized_Pad) +{ + testONNXModels("quantized_padding"); +} + +TEST_P(Test_ONNX_layers, Quantized_Reshape) +{ + testONNXModels("quantized_reshape"); +} + +TEST_P(Test_ONNX_layers, Quantized_Transpose) +{ + testONNXModels("quantized_transpose"); +} + +TEST_P(Test_ONNX_layers, Quantized_Squeeze) +{ + testONNXModels("quantized_squeeze"); +} + +TEST_P(Test_ONNX_layers, Quantized_Unsqueeze) +{ + testONNXModels("quantized_unsqueeze"); +} + +TEST_P(Test_ONNX_layers, Quantized_Resize) +{ + testONNXModels("quantized_resize_nearest"); + testONNXModels("quantized_resize_bilinear", npy, 2e-4, 0.003); + testONNXModels("quantized_resize_bilinear_align", npy, 3e-4, 0.003); +} + +TEST_P(Test_ONNX_layers, Quantized_Concat) +{ + testONNXModels("quantized_concat"); + testONNXModels("quantized_concat_const_blob"); +} + +TEST_P(Test_ONNX_layers, Quantized_Constant) +{ + testONNXModels("quantized_constant", npy, 0.002, 0.008); +} + INSTANTIATE_TEST_CASE_P(/*nothing*/, Test_ONNX_layers, dnnBackendsAndTargets()); class Test_ONNX_nets : public Test_ONNX_layers @@ -1075,6 +1241,11 @@ TEST_P(Test_ONNX_nets, ResNet50v1) testONNXModels("resnet50v1", pb, default_l1, default_lInf, true, target != DNN_TARGET_MYRIAD); } +TEST_P(Test_ONNX_nets, ResNet50_Int8) +{ + testONNXModels("resnet50_int8", pb, default_l1, default_lInf, true); +} + TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC) { applyTestTag(CV_TEST_TAG_VERYLONG); @@ -1335,6 +1506,15 @@ TEST_P(Test_ONNX_nets, Resnet34_kinetics) expectNoFallbacksFromIE(net); } +TEST_P(Test_ONNX_layers, CumSum) +{ + testONNXModels("cumsum_1d_exclusive_1"); + testONNXModels("cumsum_1d_reverse"); + testONNXModels("cumsum_1d_exclusive_1_reverse"); + testONNXModels("cumsum_2d_dim_1"); + testONNXModels("cumsum_3d_dim_2"); +} + INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_nets, dnnBackendsAndTargets()); }} // namespace diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index 2c36134724..8e7f969480 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -13,6 +13,7 @@ Test for Tensorflow models loading #include "npy_blob.hpp" #include // CV_DNN_REGISTER_LAYER_CLASS +#include namespace opencv_test { @@ -128,6 +129,13 @@ TEST_P(Test_TensorFlow_layers, reduce_mean) runTensorFlowNet("global_pool_by_axis"); } +TEST_P(Test_TensorFlow_layers, reduce_max) +{ + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + runTensorFlowNet("max_pool_by_axis"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) @@ -135,11 +143,21 @@ TEST_P(Test_TensorFlow_layers, reduce_sum) runTensorFlowNet("sum_pool_by_axis"); } +TEST_P(Test_TensorFlow_layers, reduce_max_channel) +{ + runTensorFlowNet("reduce_max_channel"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum_channel) { runTensorFlowNet("reduce_sum_channel"); } +TEST_P(Test_TensorFlow_layers, reduce_max_channel_keep_dims) +{ + runTensorFlowNet("reduce_max_channel", false, 0.0, 0.0, false, "_keep_dims"); +} + TEST_P(Test_TensorFlow_layers, reduce_sum_channel_keep_dims) { runTensorFlowNet("reduce_sum_channel", false, 0.0, 0.0, false, "_keep_dims"); @@ -203,6 +221,16 @@ TEST_P(Test_TensorFlow_layers, padding) runTensorFlowNet("keras_pad_concat"); } +TEST_P(Test_TensorFlow_layers, padding_asymmetric) +{ + runTensorFlowNet("conv2d_asymmetric_pads_nchw"); + runTensorFlowNet("conv2d_asymmetric_pads_nhwc"); + runTensorFlowNet("max_pool2d_asymmetric_pads_nchw"); + runTensorFlowNet("max_pool2d_asymmetric_pads_nhwc"); + runTensorFlowNet("conv2d_backprop_input_asymmetric_pads_nchw"); + runTensorFlowNet("conv2d_backprop_input_asymmetric_pads_nhwc"); +} + TEST_P(Test_TensorFlow_layers, padding_same) { // Reference output values are in range [0.0006, 2.798] @@ -385,6 +413,11 @@ TEST_P(Test_TensorFlow_layers, pooling_reduce_mean) runTensorFlowNet("reduce_mean"); // an average pooling over all spatial dimensions. } +TEST_P(Test_TensorFlow_layers, pooling_reduce_max) +{ + runTensorFlowNet("reduce_max"); // a MAX pooling over all spatial dimensions. +} + TEST_P(Test_TensorFlow_layers, pooling_reduce_sum) { runTensorFlowNet("reduce_sum"); // a SUM pooling over all spatial dimensions. @@ -548,6 +581,31 @@ TEST_P(Test_TensorFlow_layers, l2_normalize) runTensorFlowNet("l2_normalize"); } +TEST_P(Test_TensorFlow_layers, BiasAdd) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + runTensorFlowNet("bias_add_1"); +} + +TEST_P(Test_TensorFlow_layers, ExpandDims) +{ +#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD + && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X + ) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif + + runTensorFlowNet("expand_dims_1"); + runTensorFlowNet("expand_dims_2"); +} + // TODO: fix it and add to l2_normalize TEST_P(Test_TensorFlow_layers, l2_normalize_3d) { @@ -568,6 +626,41 @@ TEST_P(Test_TensorFlow_layers, l2_normalize_3d) runTensorFlowNet("l2_normalize_3d"); } +class Test_TensorFlow_diagnostics : public DNNTestLayer { +public: + Test_TensorFlow_diagnostics() + { + enableModelDiagnostics(true); + skipModelImport(true); + } + + ~Test_TensorFlow_diagnostics() + { + enableModelDiagnostics(false); + skipModelImport(false); + } + + void runFailingTensorFlowNet(const std::string& prefix, bool hasText = false) + { + std::string netPath = path(prefix + "_net.pb"); + std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : ""); + + Net net = readNetFromTensorflow(netPath, netConfig); + } +}; + +TEST_P(Test_TensorFlow_diagnostics, not_implemented_layer) +{ + runFailingTensorFlowNet("not_implemented_layer"); +} + +TEST_P(Test_TensorFlow_diagnostics, broken_parameters) +{ + runFailingTensorFlowNet("broken_layer"); +} + +INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_diagnostics, dnnBackendsAndTargets()); + class Test_TensorFlow_nets : public DNNTestLayer {}; TEST_P(Test_TensorFlow_nets, MobileNet_SSD) @@ -1174,6 +1267,11 @@ TEST_P(Test_TensorFlow_layers, resize_bilinear_down) runTensorFlowNet("resize_bilinear_down"); } +TEST_P(Test_TensorFlow_layers, resize_concat_optimization) +{ + runTensorFlowNet("resize_concat_optimization"); +} + TEST_P(Test_TensorFlow_layers, tf2_dense) { runTensorFlowNet("tf2_dense"); diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index f1d636895b..48725f244e 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -290,9 +290,14 @@ TEST_P(Test_Torch_layers, net_padding) TEST_P(Test_Torch_layers, net_non_spatial) { -#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000) +#if defined(INF_ENGINE_RELEASE) && ( \ + INF_ENGINE_VER_MAJOR_EQ(2021030000) || \ + INF_ENGINE_VER_MAJOR_EQ(2021040000) \ +) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // crash + // 2021.3: crash + // 2021.4: [ GENERAL_ERROR ] AssertionFailed: !out.networkInputs.empty() + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); // exception if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16) diff --git a/modules/features2d/include/opencv2/features2d.hpp b/modules/features2d/include/opencv2/features2d.hpp index 8a8ff04625..afd164ecb6 100644 --- a/modules/features2d/include/opencv2/features2d.hpp +++ b/modules/features2d/include/opencv2/features2d.hpp @@ -1094,7 +1094,7 @@ public: that is, copies both parameters and train data. If emptyTrainData is true, the method creates an object copy with the current parameters but with empty train data. */ - CV_WRAP virtual Ptr clone( bool emptyTrainData=false ) const = 0; + CV_WRAP CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const = 0; /** @brief Creates a descriptor matcher of a given type with the default parameters (using default constructor). @@ -1154,7 +1154,7 @@ protected: static bool isPossibleMatch( InputArray mask, int queryIdx, int trainIdx ); static bool isMaskedOut( InputArrayOfArrays masks, int queryIdx ); - static Mat clone_op( Mat m ) { return m.clone(); } + CV_NODISCARD_STD static Mat clone_op( Mat m ) { return m.clone(); } void checkMasks( InputArrayOfArrays masks, int queryDescriptorsCount ) const; //! Collection of descriptors from train images. @@ -1195,7 +1195,7 @@ public: */ CV_WRAP static Ptr create( int normType=NORM_L2, bool crossCheck=false ) ; - virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; + CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; protected: virtual void knnMatchImpl( InputArray queryDescriptors, std::vector >& matches, int k, InputArrayOfArrays masks=noArray(), bool compactResult=false ) CV_OVERRIDE; @@ -1234,7 +1234,7 @@ public: CV_WRAP static Ptr create(); - virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; + CV_NODISCARD_STD virtual Ptr clone( bool emptyTrainData=false ) const CV_OVERRIDE; protected: static void convertToDMatches( const DescriptorCollection& descriptors, const Mat& indices, const Mat& distances, @@ -1326,6 +1326,13 @@ CV_EXPORTS_W void drawMatches( InputArray img1, const std::vector& key const std::vector& matchesMask=std::vector(), DrawMatchesFlags flags=DrawMatchesFlags::DEFAULT ); /** @overload */ +CV_EXPORTS_W void drawMatches( InputArray img1, const std::vector& keypoints1, + InputArray img2, const std::vector& keypoints2, + const std::vector& matches1to2, InputOutputArray outImg, + const int matchesThickness, const Scalar& matchColor=Scalar::all(-1), + const Scalar& singlePointColor=Scalar::all(-1), const std::vector& matchesMask=std::vector(), + DrawMatchesFlags flags=DrawMatchesFlags::DEFAULT ); + CV_EXPORTS_AS(drawMatchesKnn) void drawMatches( InputArray img1, const std::vector& keypoints1, InputArray img2, const std::vector& keypoints2, const std::vector >& matches1to2, InputOutputArray outImg, diff --git a/modules/features2d/src/blobdetector.cpp b/modules/features2d/src/blobdetector.cpp index c2215cd57c..fb79831807 100644 --- a/modules/features2d/src/blobdetector.cpp +++ b/modules/features2d/src/blobdetector.cpp @@ -44,6 +44,8 @@ #include #include +#include + // Requires CMake flag: DEBUG_opencv_features2d=ON //#define DEBUG_BLOB_DETECTOR @@ -317,6 +319,19 @@ void SimpleBlobDetectorImpl::detect(InputArray image, std::vector& CV_Error(Error::StsUnsupportedFormat, "Blob detector only supports 8-bit images!"); } + CV_CheckGT(params.thresholdStep, 0.0f, ""); + if (params.minThreshold + params.thresholdStep >= params.maxThreshold) + { + // https://github.com/opencv/opencv/issues/6667 + CV_LOG_ONCE_INFO(NULL, "SimpleBlobDetector: params.minDistBetweenBlobs is ignored for case with single threshold"); +#if 0 // OpenCV 5.0 + CV_CheckEQ(params.minRepeatability, 1u, "Incompatible parameters for case with single threshold"); +#else + if (params.minRepeatability != 1) + CV_LOG_WARNING(NULL, "SimpleBlobDetector: params.minRepeatability=" << params.minRepeatability << " is incompatible for case with single threshold. Empty result is expected."); +#endif + } + std::vector < std::vector
> centers; for (double thresh = params.minThreshold; thresh < params.maxThreshold; thresh += params.thresholdStep) { @@ -325,19 +340,13 @@ void SimpleBlobDetectorImpl::detect(InputArray image, std::vector& std::vector < Center > curCenters; findBlobs(grayscaleImage, binarizedImage, curCenters); - if(params.maxThreshold - params.minThreshold <= params.thresholdStep) { - // if the difference between min and max threshold is less than the threshold step - // we're only going to enter the loop once, so we need to add curCenters - // to ensure we still use minDistBetweenBlobs - centers.push_back(curCenters); - } std::vector < std::vector
> newCenters; for (size_t i = 0; i < curCenters.size(); i++) { bool isNew = true; for (size_t j = 0; j < centers.size(); j++) { - double dist = norm(centers[j][centers[j].size() / 2 ].location - curCenters[i].location); + double dist = norm(centers[j][ centers[j].size() / 2 ].location - curCenters[i].location); isNew = dist >= params.minDistBetweenBlobs && dist >= centers[j][ centers[j].size() / 2 ].radius && dist >= curCenters[i].radius; if (!isNew) { diff --git a/modules/features2d/src/draw.cpp b/modules/features2d/src/draw.cpp index 84fb0aca39..86f06dc885 100644 --- a/modules/features2d/src/draw.cpp +++ b/modules/features2d/src/draw.cpp @@ -183,7 +183,8 @@ static void _prepareImgAndDrawKeypoints( InputArray img1, const std::vector& keypoints1, @@ -207,6 +208,21 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, const std::vector& matches1to2, InputOutputArray outImg, const Scalar& matchColor, const Scalar& singlePointColor, const std::vector& matchesMask, DrawMatchesFlags flags ) +{ + drawMatches( img1, keypoints1, + img2, keypoints2, + matches1to2, outImg, + 1, matchColor, + singlePointColor, matchesMask, + flags); +} + +void drawMatches( InputArray img1, const std::vector& keypoints1, + InputArray img2, const std::vector& keypoints2, + const std::vector& matches1to2, InputOutputArray outImg, + const int matchesThickness, const Scalar& matchColor, + const Scalar& singlePointColor, const std::vector& matchesMask, + DrawMatchesFlags flags ) { if( !matchesMask.empty() && matchesMask.size() != matches1to2.size() ) CV_Error( Error::StsBadSize, "matchesMask must have the same size as matches1to2" ); @@ -226,11 +242,12 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, CV_Assert(i2 >= 0 && i2 < static_cast(keypoints2.size())); const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; - _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags, matchesThickness ); } } } + void drawMatches( InputArray img1, const std::vector& keypoints1, InputArray img2, const std::vector& keypoints2, const std::vector >& matches1to2, InputOutputArray outImg, @@ -254,7 +271,7 @@ void drawMatches( InputArray img1, const std::vector& keypoints1, if( matchesMask.empty() || matchesMask[i][j] ) { const KeyPoint &kp1 = keypoints1[i1], &kp2 = keypoints2[i2]; - _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags ); + _drawMatch( outImg, outImg1, outImg2, kp1, kp2, matchColor, flags, 1 ); } } } diff --git a/modules/features2d/src/kaze/KAZEFeatures.cpp b/modules/features2d/src/kaze/KAZEFeatures.cpp index 58f0937f00..ab591d417c 100644 --- a/modules/features2d/src/kaze/KAZEFeatures.cpp +++ b/modules/features2d/src/kaze/KAZEFeatures.cpp @@ -311,7 +311,7 @@ private: void KAZEFeatures::Determinant_Hessian(std::vector& kpts) { int level = 0; - float dist = 0.0, smax = 3.0; + float smax = 3.0; int npoints = 0, id_repeated = 0; int left_x = 0, right_x = 0, up_y = 0, down_y = 0; bool is_extremum = false, is_repeated = false, is_out = false; @@ -338,17 +338,24 @@ void KAZEFeatures::Determinant_Hessian(std::vector& kpts) for (int j = 0; j < (int)kpts_par_[i].size(); j++) { level = i + 1; + const TEvolution& evolution_level = evolution_[level]; + is_extremum = true; is_repeated = false; is_out = false; - // Check in case we have the same point as maxima in previous evolution levels - for (int ik = 0; ik < (int)kpts.size(); ik++) { - if (kpts[ik].class_id == level || kpts[ik].class_id == level + 1 || kpts[ik].class_id == level - 1) { - dist = pow(kpts_par_[i][j].pt.x - kpts[ik].pt.x, 2) + pow(kpts_par_[i][j].pt.y - kpts[ik].pt.y, 2); + const KeyPoint& kpts_par_ij = kpts_par_[i][j]; - if (dist < evolution_[level].sigma_size*evolution_[level].sigma_size) { - if (kpts_par_[i][j].response > kpts[ik].response) { + // Check in case we have the same point as maxima in previous evolution levels + for (int ik = 0; ik < (int)kpts.size(); ik++) + { + const KeyPoint& kpts_ik = kpts[ik]; + if (kpts_ik.class_id == level || kpts_ik.class_id == level + 1 || kpts_ik.class_id == level - 1) { + Point2f diff = kpts_par_ij.pt - kpts_ik.pt; + float dist = diff.dot(diff); + + if (dist < evolution_level.sigma_size*evolution_level.sigma_size) { + if (kpts_par_ij.response > kpts_ik.response) { id_repeated = ik; is_repeated = true; } @@ -363,23 +370,23 @@ void KAZEFeatures::Determinant_Hessian(std::vector& kpts) if (is_extremum == true) { // Check that the point is under the image limits for the descriptor computation - left_x = cvRound(kpts_par_[i][j].pt.x - smax*kpts_par_[i][j].size); - right_x = cvRound(kpts_par_[i][j].pt.x + smax*kpts_par_[i][j].size); - up_y = cvRound(kpts_par_[i][j].pt.y - smax*kpts_par_[i][j].size); - down_y = cvRound(kpts_par_[i][j].pt.y + smax*kpts_par_[i][j].size); + left_x = cvRound(kpts_par_ij.pt.x - smax*kpts_par_ij.size); + right_x = cvRound(kpts_par_ij.pt.x + smax*kpts_par_ij.size); + up_y = cvRound(kpts_par_ij.pt.y - smax*kpts_par_ij.size); + down_y = cvRound(kpts_par_ij.pt.y + smax*kpts_par_ij.size); - if (left_x < 0 || right_x >= evolution_[level].Ldet.cols || - up_y < 0 || down_y >= evolution_[level].Ldet.rows) { + if (left_x < 0 || right_x >= evolution_level.Ldet.cols || + up_y < 0 || down_y >= evolution_level.Ldet.rows) { is_out = true; } if (is_out == false) { if (is_repeated == false) { - kpts.push_back(kpts_par_[i][j]); + kpts.push_back(kpts_par_ij); npoints++; } else { - kpts[id_repeated] = kpts_par_[i][j]; + kpts[id_repeated] = kpts_par_ij; } } } @@ -513,16 +520,16 @@ public: if (options_.upright) { kpts[i].angle = 0.0; - if (options_.extended) + if (options_.extended) Get_KAZE_Upright_Descriptor_128(kpts[i], desc.ptr((int)i)); else Get_KAZE_Upright_Descriptor_64(kpts[i], desc.ptr((int)i)); } else { - KAZEFeatures::Compute_Main_Orientation(kpts[i], evolution, options_); + KAZEFeatures::Compute_Main_Orientation(kpts[i], evolution, options_); - if (options_.extended) + if (options_.extended) Get_KAZE_Descriptor_128(kpts[i], desc.ptr((int)i)); else Get_KAZE_Descriptor_64(kpts[i], desc.ptr((int)i)); @@ -712,26 +719,26 @@ void KAZE_Descriptor_Invoker::Get_KAZE_Upright_Descriptor_64(const KeyPoint &kpt y1 = (int)(sample_y - 0.5f); x1 = (int)(sample_x - 0.5f); - checkDescriptorLimits(x1, y1, options_.img_width, options_.img_height); + checkDescriptorLimits(x1, y1, options_.img_width, options_.img_height); y2 = (int)(sample_y + 0.5f); x2 = (int)(sample_x + 0.5f); - checkDescriptorLimits(x2, y2, options_.img_width, options_.img_height); + checkDescriptorLimits(x2, y2, options_.img_width, options_.img_height); fx = sample_x - x1; fy = sample_y - y1; - res1 = *(evolution[level].Lx.ptr(y1)+x1); - res2 = *(evolution[level].Lx.ptr(y1)+x2); - res3 = *(evolution[level].Lx.ptr(y2)+x1); - res4 = *(evolution[level].Lx.ptr(y2)+x2); + res1 = *(evolution[level].Lx.ptr(y1)+x1); + res2 = *(evolution[level].Lx.ptr(y1)+x2); + res3 = *(evolution[level].Lx.ptr(y2)+x1); + res4 = *(evolution[level].Lx.ptr(y2)+x2); rx = (1.0f - fx)*(1.0f - fy)*res1 + fx*(1.0f - fy)*res2 + (1.0f - fx)*fy*res3 + fx*fy*res4; - res1 = *(evolution[level].Ly.ptr(y1)+x1); - res2 = *(evolution[level].Ly.ptr(y1)+x2); - res3 = *(evolution[level].Ly.ptr(y2)+x1); - res4 = *(evolution[level].Ly.ptr(y2)+x2); + res1 = *(evolution[level].Ly.ptr(y1)+x1); + res2 = *(evolution[level].Ly.ptr(y1)+x2); + res3 = *(evolution[level].Ly.ptr(y2)+x1); + res4 = *(evolution[level].Ly.ptr(y2)+x2); ry = (1.0f - fx)*(1.0f - fy)*res1 + fx*(1.0f - fy)*res2 + (1.0f - fx)*fy*res3 + fx*fy*res4; rx = gauss_s1*rx; diff --git a/modules/features2d/src/orb.cpp b/modules/features2d/src/orb.cpp index 85d17cdd0d..ae1e611127 100644 --- a/modules/features2d/src/orb.cpp +++ b/modules/features2d/src/orb.cpp @@ -131,12 +131,17 @@ static void HarrisResponses(const Mat& img, const std::vector& layerinfo, std::vector& pts, int blockSize, float harris_k) { - CV_Assert( img.type() == CV_8UC1 && blockSize*blockSize <= 2048 ); + CV_CheckTypeEQ(img.type(), CV_8UC1, ""); + CV_CheckGT(blockSize, 0, ""); + CV_CheckLE(blockSize*blockSize, 2048, ""); size_t ptidx, ptsize = pts.size(); const uchar* ptr00 = img.ptr(); - int step = (int)(img.step/img.elemSize1()); + size_t size_t_step = img.step; + CV_CheckLE(size_t_step * blockSize + blockSize + 1, (size_t)INT_MAX, ""); // ofs computation, step+1 + int step = static_cast(size_t_step); + int r = blockSize/2; float scale = 1.f/((1 << 2) * blockSize * 255.f); @@ -154,7 +159,7 @@ HarrisResponses(const Mat& img, const std::vector& layerinfo, int y0 = cvRound(pts[ptidx].pt.y); int z = pts[ptidx].octave; - const uchar* ptr0 = ptr00 + (y0 - r + layerinfo[z].y)*step + x0 - r + layerinfo[z].x; + const uchar* ptr0 = ptr00 + (y0 - r + layerinfo[z].y)*size_t_step + (x0 - r + layerinfo[z].x); int a = 0, b = 0, c = 0; for( int k = 0; k < blockSize*blockSize; k++ ) diff --git a/modules/features2d/src/sift.simd.hpp b/modules/features2d/src/sift.simd.hpp index b5033459b9..60129b1535 100644 --- a/modules/features2d/src/sift.simd.hpp +++ b/modules/features2d/src/sift.simd.hpp @@ -450,31 +450,184 @@ public: const sift_wt* currptr = img.ptr(r); const sift_wt* prevptr = prev.ptr(r); const sift_wt* nextptr = next.ptr(r); + int c = SIFT_IMG_BORDER; - for( int c = SIFT_IMG_BORDER; c < cols-SIFT_IMG_BORDER; c++) +#if CV_SIMD && !(DoG_TYPE_SHORT) + const int vecsize = v_float32::nlanes; + for( ; c <= cols-SIFT_IMG_BORDER - vecsize; c += vecsize) + { + v_float32 val = vx_load(&currptr[c]); + v_float32 _00,_01,_02; + v_float32 _10, _12; + v_float32 _20,_21,_22; + + v_float32 vmin,vmax; + + + v_float32 cond = v_abs(val) > vx_setall_f32((float)threshold); + if (!v_check_any(cond)) + { + continue; + } + + _00 = vx_load(&currptr[c-step-1]); _01 = vx_load(&currptr[c-step]); _02 = vx_load(&currptr[c-step+1]); + _10 = vx_load(&currptr[c -1]); _12 = vx_load(&currptr[c +1]); + _20 = vx_load(&currptr[c+step-1]); _21 = vx_load(&currptr[c+step]); _22 = vx_load(&currptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + v_float32 condp = cond & (val > vx_setall_f32(0)) & (val >= vmax); + v_float32 condm = cond & (val < vx_setall_f32(0)) & (val <= vmin); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + _00 = vx_load(&prevptr[c-step-1]); _01 = vx_load(&prevptr[c-step]); _02 = vx_load(&prevptr[c-step+1]); + _10 = vx_load(&prevptr[c -1]); _12 = vx_load(&prevptr[c +1]); + _20 = vx_load(&prevptr[c+step-1]); _21 = vx_load(&prevptr[c+step]); _22 = vx_load(&prevptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + condp &= (val >= vmax); + condm &= (val <= vmin); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + v_float32 _11p = vx_load(&prevptr[c]); + v_float32 _11n = vx_load(&nextptr[c]); + + v_float32 max_middle = v_max(_11n,_11p); + v_float32 min_middle = v_min(_11n,_11p); + + _00 = vx_load(&nextptr[c-step-1]); _01 = vx_load(&nextptr[c-step]); _02 = vx_load(&nextptr[c-step+1]); + _10 = vx_load(&nextptr[c -1]); _12 = vx_load(&nextptr[c +1]); + _20 = vx_load(&nextptr[c+step-1]); _21 = vx_load(&nextptr[c+step]); _22 = vx_load(&nextptr[c+step+1]); + + vmax = v_max(v_max(v_max(_00,_01),v_max(_02,_10)),v_max(v_max(_12,_20),v_max(_21,_22))); + vmin = v_min(v_min(v_min(_00,_01),v_min(_02,_10)),v_min(v_min(_12,_20),v_min(_21,_22))); + + condp &= (val >= v_max(vmax,max_middle)); + condm &= (val <= v_min(vmin,min_middle)); + + cond = condp | condm; + if (!v_check_any(cond)) + { + continue; + } + + int mask = v_signmask(cond); + for (int k = 0; k 0 ? j - 1 : n - 1; + int r2 = j < n-1 ? j + 1 : 0; + + if( hist[j] > hist[l] && hist[j] > hist[r2] && hist[j] >= mag_thr ) + { + float bin = j + 0.5f * (hist[l]-hist[r2]) / (hist[l] - 2*hist[j] + hist[r2]); + bin = bin < 0 ? n + bin : bin >= n ? bin - n : bin; + kpt.angle = 360.f - (float)((360.f/n) * bin); + if(std::abs(kpt.angle - 360.f) < FLT_EPSILON) + kpt.angle = 0.f; + + kpts_.push_back(kpt); + } + } + } + } + +#endif //CV_SIMD && !(DoG_TYPE_SHORT) + + // vector loop reminder, better predictibility and less branch density + for( ; c < cols-SIFT_IMG_BORDER; c++) { sift_wt val = currptr[c]; + if (std::abs(val) <= threshold) + continue; - // find local extrema with pixel accuracy - if( std::abs(val) > threshold && - ((val > 0 && val >= currptr[c-1] && val >= currptr[c+1] && - val >= currptr[c-step-1] && val >= currptr[c-step] && val >= currptr[c-step+1] && - val >= currptr[c+step-1] && val >= currptr[c+step] && val >= currptr[c+step+1] && - val >= nextptr[c] && val >= nextptr[c-1] && val >= nextptr[c+1] && - val >= nextptr[c-step-1] && val >= nextptr[c-step] && val >= nextptr[c-step+1] && - val >= nextptr[c+step-1] && val >= nextptr[c+step] && val >= nextptr[c+step+1] && - val >= prevptr[c] && val >= prevptr[c-1] && val >= prevptr[c+1] && - val >= prevptr[c-step-1] && val >= prevptr[c-step] && val >= prevptr[c-step+1] && - val >= prevptr[c+step-1] && val >= prevptr[c+step] && val >= prevptr[c+step+1]) || - (val < 0 && val <= currptr[c-1] && val <= currptr[c+1] && - val <= currptr[c-step-1] && val <= currptr[c-step] && val <= currptr[c-step+1] && - val <= currptr[c+step-1] && val <= currptr[c+step] && val <= currptr[c+step+1] && - val <= nextptr[c] && val <= nextptr[c-1] && val <= nextptr[c+1] && - val <= nextptr[c-step-1] && val <= nextptr[c-step] && val <= nextptr[c-step+1] && - val <= nextptr[c+step-1] && val <= nextptr[c+step] && val <= nextptr[c+step+1] && - val <= prevptr[c] && val <= prevptr[c-1] && val <= prevptr[c+1] && - val <= prevptr[c-step-1] && val <= prevptr[c-step] && val <= prevptr[c-step+1] && - val <= prevptr[c+step-1] && val <= prevptr[c+step] && val <= prevptr[c+step+1]))) + sift_wt _00,_01,_02; + sift_wt _10, _12; + sift_wt _20,_21,_22; + _00 = currptr[c-step-1]; _01 = currptr[c-step]; _02 = currptr[c-step+1]; + _10 = currptr[c -1]; _12 = currptr[c +1]; + _20 = currptr[c+step-1]; _21 = currptr[c+step]; _22 = currptr[c+step+1]; + + bool calculate = false; + if (val > 0) + { + sift_wt vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + _00 = prevptr[c-step-1]; _01 = prevptr[c-step]; _02 = prevptr[c-step+1]; + _10 = prevptr[c -1]; _12 = prevptr[c +1]; + _20 = prevptr[c+step-1]; _21 = prevptr[c+step]; _22 = prevptr[c+step+1]; + vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + _00 = nextptr[c-step-1]; _01 = nextptr[c-step]; _02 = nextptr[c-step+1]; + _10 = nextptr[c -1]; _12 = nextptr[c +1]; + _20 = nextptr[c+step-1]; _21 = nextptr[c+step]; _22 = nextptr[c+step+1]; + vmax = std::max(std::max(std::max(_00,_01),std::max(_02,_10)),std::max(std::max(_12,_20),std::max(_21,_22))); + if (val >= vmax) + { + sift_wt _11p = prevptr[c], _11n = nextptr[c]; + calculate = (val >= std::max(_11p,_11n)); + } + } + } + + } else { // val cant be zero here (first abs took care of zero), must be negative + sift_wt vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + _00 = prevptr[c-step-1]; _01 = prevptr[c-step]; _02 = prevptr[c-step+1]; + _10 = prevptr[c -1]; _12 = prevptr[c +1]; + _20 = prevptr[c+step-1]; _21 = prevptr[c+step]; _22 = prevptr[c+step+1]; + vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + _00 = nextptr[c-step-1]; _01 = nextptr[c-step]; _02 = nextptr[c-step+1]; + _10 = nextptr[c -1]; _12 = nextptr[c +1]; + _20 = nextptr[c+step-1]; _21 = nextptr[c+step]; _22 = nextptr[c+step+1]; + vmin = std::min(std::min(std::min(_00,_01),std::min(_02,_10)),std::min(std::min(_12,_20),std::min(_21,_22))); + if (val <= vmin) + { + sift_wt _11p = prevptr[c], _11n = nextptr[c]; + calculate = (val <= std::min(_11p,_11n)); + } + } + } + } + + if (calculate) { CV_TRACE_REGION("pixel_candidate"); diff --git a/modules/features2d/test/test_blobdetector.cpp b/modules/features2d/test/test_blobdetector.cpp index 56b7145862..c6ed09e6a2 100644 --- a/modules/features2d/test/test_blobdetector.cpp +++ b/modules/features2d/test/test_blobdetector.cpp @@ -12,6 +12,7 @@ TEST(Features2d_BlobDetector, bug_6667) SimpleBlobDetector::Params params; params.minThreshold = 250; params.maxThreshold = 260; + params.minRepeatability = 1; // https://github.com/opencv/opencv/issues/6667 std::vector keypoints; Ptr detector = SimpleBlobDetector::create(params); diff --git a/modules/features2d/test/test_orb.cpp b/modules/features2d/test/test_orb.cpp index 8a4d9776b3..89e2f7d78b 100644 --- a/modules/features2d/test/test_orb.cpp +++ b/modules/features2d/test/test_orb.cpp @@ -141,5 +141,31 @@ TEST(Features2D_ORB, regression_16197) ASSERT_NO_THROW(orbPtr->detectAndCompute(img, noArray(), kps, fv)); } +// https://github.com/opencv/opencv-python/issues/537 +BIGDATA_TEST(Features2D_ORB, regression_opencv_python_537) // memory usage: ~3 Gb +{ + applyTestTag( + CV_TEST_TAG_LONG, + CV_TEST_TAG_DEBUG_VERYLONG, + CV_TEST_TAG_MEMORY_6GB + ); + + const int width = 25000; + const int height = 25000; + Mat img(Size(width, height), CV_8UC1, Scalar::all(0)); + + const int border = 23, num_lines = 23; + for (int i = 0; i < num_lines; i++) + { + cv::Point2i point1(border + i * 100, border + i * 100); + cv::Point2i point2(width - border - i * 100, height - border * i * 100); + cv::line(img, point1, point2, 255, 1, LINE_AA); + } + + Ptr orbPtr = ORB::create(31); + std::vector kps; + Mat fv; + ASSERT_NO_THROW(orbPtr->detectAndCompute(img, noArray(), kps, fv)); +} }} // namespace diff --git a/modules/flann/include/opencv2/flann/heap.h b/modules/flann/include/opencv2/flann/heap.h index ee1c682cfe..8cace20449 100644 --- a/modules/flann/include/opencv2/flann/heap.h +++ b/modules/flann/include/opencv2/flann/heap.h @@ -36,9 +36,21 @@ #include #include +#include + namespace cvflann { +// TODO: Define x > y operator and use std::greater instead +template +struct greater +{ + bool operator()(const T& x, const T& y) const + { + return y < x; + } +}; + /** * Priority Queue Implementation * @@ -49,117 +61,180 @@ namespace cvflann template class Heap { - /** * Storage array for the heap. * Type T must be comparable. */ std::vector heap; - int length; - - /** - * Number of element in the heap - */ - int count; - - - public: /** - * Constructor. + * \brief Constructs a heap with a pre-allocated capacity * - * Params: - * sz = heap size + * \param capacity heap maximum capacity */ - - Heap(int sz) + Heap(const int capacity) { - length = sz; - heap.reserve(length); - count = 0; + reserve(capacity); + } + + /** + * \brief Move-constructs a heap from an external vector + * + * \param vec external vector + */ + Heap(std::vector&& vec) + : heap(std::move(vec)) + { + std::make_heap(heap.begin(), heap.end(), greater()); } /** * - * Returns: heap size + * \returns heap size */ - int size() + int size() const { - return count; + return (int)heap.size(); } /** - * Tests if the heap is empty * - * Returns: true is heap empty, false otherwise + * \returns heap capacity + */ + int capacity() const + { + return (int)heap.capacity(); + } + + /** + * \brief Tests if the heap is empty + * + * \returns true is heap empty, false otherwise */ bool empty() { - return size()==0; + return heap.empty(); } /** - * Clears the heap. + * \brief Clears the heap. */ void clear() { heap.clear(); - count = 0; } - struct CompareT + /** + * \brief Sets the heap maximum capacity. + * + * \param capacity heap maximum capacity + */ + void reserve(const int capacity) { - bool operator()(const T& t_1, const T& t_2) const - { - return t_2 < t_1; - } - }; + heap.reserve(capacity); + } /** - * Insert a new element in the heap. + * \brief Inserts a new element in the heap. * * We select the next empty leaf node, and then keep moving any larger * parents down until the right location is found to store this element. * - * Params: - * value = the new element to be inserted in the heap + * \param value the new element to be inserted in the heap */ void insert(T value) { /* If heap is full, then return without adding this element. */ - if (count == length) { + if (size() == capacity()) { return; } heap.push_back(value); - static CompareT compareT; - std::push_heap(heap.begin(), heap.end(), compareT); - ++count; + std::push_heap(heap.begin(), heap.end(), greater()); } - - /** - * Returns the node of minimum value from the heap (top of the heap). + * \brief Returns the node of minimum value from the heap (top of the heap). * - * Params: - * value = out parameter used to return the min element - * Returns: false if heap empty + * \param[out] value parameter used to return the min element + * \returns false if heap empty */ bool popMin(T& value) { - if (count == 0) { + if (empty()) { return false; } value = heap[0]; - static CompareT compareT; - std::pop_heap(heap.begin(), heap.end(), compareT); + std::pop_heap(heap.begin(), heap.end(), greater()); heap.pop_back(); - --count; return true; /* Return old last node. */ } + + /** + * \brief Returns a shared heap for the given memory pool ID. + * + * It constructs the heap if it does not already exists. + * + * \param poolId a user-chosen hashable ID for identifying the heap. + * For thread-safe operations, using current thread ID is a good choice. + * \param capacity heap maximum capacity + * \param iterThreshold remove heaps that were not reused for more than specified iterations count + * if iterThreshold value is less 2, it will be internally adjusted to twice the number of CPU threads + * \returns pointer to the heap + */ + template + static cv::Ptr> getPooledInstance( + const HashableT& poolId, const int capacity, int iterThreshold = 0) + { + static cv::Mutex mutex; + const cv::AutoLock lock(mutex); + + struct HeapMapValueType { + cv::Ptr> heapPtr; + int iterCounter; + }; + typedef std::unordered_map HeapMapType; + + static HeapMapType heapsPool; + typename HeapMapType::iterator heapIt = heapsPool.find(poolId); + + if (heapIt == heapsPool.end()) + { + // Construct the heap as it does not already exists + HeapMapValueType heapAndTimePair = {cv::makePtr>(capacity), 0}; + const std::pair& emplaceResult = heapsPool.emplace(poolId, std::move(heapAndTimePair)); + CV_CheckEQ(static_cast(emplaceResult.second), 1, "Failed to insert the heap into its memory pool"); + heapIt = emplaceResult.first; + } + else + { + CV_CheckEQ(heapIt->second.heapPtr.use_count(), 1, "Cannot modify a heap that is currently accessed by another caller"); + heapIt->second.heapPtr->clear(); + heapIt->second.heapPtr->reserve(capacity); + heapIt->second.iterCounter = 0; + } + + if (iterThreshold <= 1) { + iterThreshold = 2 * cv::getNumThreads(); + } + + // Remove heaps that were not reused for more than given iterThreshold + typename HeapMapType::iterator cleanupIt = heapsPool.begin(); + while (cleanupIt != heapsPool.end()) + { + if (cleanupIt->second.iterCounter++ > iterThreshold) + { + CV_Assert(cleanupIt != heapIt); + cleanupIt = heapsPool.erase(cleanupIt); + continue; + } + ++cleanupIt; + } + + return heapIt->second.heapPtr; + } }; } diff --git a/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h b/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h index 2d39d4f0f6..60662e7714 100644 --- a/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h +++ b/modules/flann/include/opencv2/flann/hierarchical_clustering_index.h @@ -532,7 +532,7 @@ public: const bool explore_all_trees = get_param(searchParams,"explore_all_trees",false); // Priority queue storing intermediate branches in the best-bin-first search - Heap* heap = new Heap((int)size_); + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); std::vector checked(size_,false); int checks = 0; @@ -548,8 +548,6 @@ public: findNN(node, result, vec, checks, maxChecks, heap, checked, false); } - delete heap; - CV_Assert(result.full()); } @@ -742,7 +740,7 @@ private: void findNN(NodePtr node, ResultSet& result, const ElementType* vec, int& checks, int maxChecks, - Heap* heap, std::vector& checked, bool explore_all_trees = false) + const cv::Ptr>& heap, std::vector& checked, bool explore_all_trees = false) { if (node->childs==NULL) { if (!explore_all_trees && (checks>=maxChecks) && result.full()) { diff --git a/modules/flann/include/opencv2/flann/kdtree_index.h b/modules/flann/include/opencv2/flann/kdtree_index.h index 603fdbd421..8245f7db79 100644 --- a/modules/flann/include/opencv2/flann/kdtree_index.h +++ b/modules/flann/include/opencv2/flann/kdtree_index.h @@ -445,11 +445,12 @@ private: { int i; BranchSt branch; - int checkCount = 0; - Heap* heap = new Heap((int)size_); DynamicBitset checked(size_); + // Priority queue storing intermediate branches in the best-bin-first search + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); + /* Search once through each tree down to root. */ for (i = 0; i < trees_; ++i) { searchLevel(result, vec, tree_roots_[i], 0, checkCount, maxCheck, @@ -464,8 +465,6 @@ private: epsError, heap, checked, false); } - delete heap; - CV_Assert(result.full()); } @@ -476,7 +475,7 @@ private: * at least "mindistsq". */ void searchLevel(ResultSet& result_set, const ElementType* vec, NodePtr node, DistanceType mindist, int& checkCount, int maxCheck, - float epsError, Heap* heap, DynamicBitset& checked, bool explore_all_trees = false) + float epsError, const cv::Ptr>& heap, DynamicBitset& checked, bool explore_all_trees = false) { if (result_set.worstDist()* heap = new Heap((int)size_); + const cv::Ptr>& heap = Heap::getPooledInstance(cv::utils::getThreadID(), (int)size_); int checks = 0; for (int i=0; i& result, const ElementType* vec, int& checks, int maxChecks, - Heap* heap) + const cv::Ptr>& heap) { // Ignore those clusters that are too far away { @@ -1577,7 +1575,7 @@ private: * distances = array with the distances to each child node. * Returns: */ - int exploreNodeBranches(KMeansNodePtr node, const ElementType* q, DistanceType* domain_distances, Heap* heap) + int exploreNodeBranches(KMeansNodePtr node, const ElementType* q, DistanceType* domain_distances, const cv::Ptr>& heap) { int best_index = 0; diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index b26b613e72..50c8f42717 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -163,6 +163,22 @@ set(gapi_srcs src/backends/ie/bindings_ie.cpp src/backends/python/gpythonbackend.cpp + # Streaming source + src/streaming/onevpl/source.cpp + src/streaming/onevpl/source_priv.cpp + src/streaming/onevpl/file_data_provider.cpp + src/streaming/onevpl/cfg_params.cpp + src/streaming/onevpl/data_provider_interface_exception.cpp + src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp + src/streaming/onevpl/accelerators/surface/surface.cpp + src/streaming/onevpl/accelerators/surface/surface_pool.cpp + src/streaming/onevpl/accelerators/accel_policy_cpu.cpp + src/streaming/onevpl/accelerators/accel_policy_dx11.cpp + src/streaming/onevpl/engine/engine_session.cpp + src/streaming/onevpl/engine/processing_engine_base.cpp + src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp + src/streaming/onevpl/engine/decode/decode_session.cpp + # Utils (ITT tracing) src/utils/itt.cpp ) @@ -234,6 +250,20 @@ if(HAVE_PLAIDML) ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${PLAIDML_INCLUDE_DIRS}) endif() +if(HAVE_GAPI_ONEVPL) + if(TARGET opencv_test_gapi) + ocv_target_compile_definitions(opencv_test_gapi PRIVATE -DHAVE_ONEVPL) + ocv_target_link_libraries(opencv_test_gapi PRIVATE ${VPL_IMPORTED_TARGETS}) + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(opencv_test_gapi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() + endif() + ocv_target_compile_definitions(${the_module} PRIVATE -DHAVE_ONEVPL) + ocv_target_link_libraries(${the_module} PRIVATE ${VPL_IMPORTED_TARGETS}) + if(HAVE_D3D11 AND HAVE_OPENCL) + ocv_target_include_directories(${the_module} SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + endif() +endif() if(WIN32) # Required for htonl/ntohl on Windows diff --git a/modules/gapi/cmake/init.cmake b/modules/gapi/cmake/init.cmake index 4c25c75f55..1c464328ca 100644 --- a/modules/gapi/cmake/init.cmake +++ b/modules/gapi/cmake/init.cmake @@ -32,3 +32,10 @@ if(WITH_PLAIDML) set(HAVE_PLAIDML TRUE) endif() endif() + +if(WITH_GAPI_ONEVPL) + find_package(VPL) + if(VPL_FOUND) + set(HAVE_GAPI_ONEVPL TRUE) + endif() +endif() diff --git a/modules/gapi/cmake/standalone.cmake b/modules/gapi/cmake/standalone.cmake index d08eda1be5..f81c1c8a85 100644 --- a/modules/gapi/cmake/standalone.cmake +++ b/modules/gapi/cmake/standalone.cmake @@ -6,6 +6,13 @@ if (NOT TARGET ade ) find_package(ade 0.1.0 REQUIRED) endif() +if (WITH_GAPI_ONEVPL) + find_package(VPL) + if(VPL_FOUND) + set(HAVE_GAPI_ONEVPL TRUE) + endif() +endif() + set(FLUID_TARGET fluid) set(FLUID_ROOT "${CMAKE_CURRENT_LIST_DIR}/../") diff --git a/modules/gapi/include/opencv2/gapi.hpp b/modules/gapi/include/opencv2/gapi.hpp index e4b2021479..f10dfd471d 100644 --- a/modules/gapi/include/opencv2/gapi.hpp +++ b/modules/gapi/include/opencv2/gapi.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_HPP @@ -19,6 +19,7 @@ @} @defgroup gapi_std_backends G-API Standard Backends @defgroup gapi_compile_args G-API Graph Compilation Arguments + @defgroup gapi_serialization G-API Serialization functionality @} */ diff --git a/modules/gapi/include/opencv2/gapi/core.hpp b/modules/gapi/include/opencv2/gapi/core.hpp index cb8a6127d7..b7f65b6719 100644 --- a/modules/gapi/include/opencv2/gapi/core.hpp +++ b/modules/gapi/include/opencv2/gapi/core.hpp @@ -29,6 +29,10 @@ */ namespace cv { namespace gapi { +/** + * @brief This namespace contains G-API Operation Types for OpenCV + * Core module functionality. + */ namespace core { using GMat2 = std::tuple; using GMat3 = std::tuple; // FIXME: how to avoid this? diff --git a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp index 5539e244ba..009c19688b 100644 --- a/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp +++ b/modules/gapi/include/opencv2/gapi/cpu/gcpukernel.hpp @@ -40,6 +40,10 @@ namespace gimpl namespace gapi { +/** + * @brief This namespace contains G-API CPU backend functions, + * structures, and symbols. + */ namespace cpu { /** @@ -492,7 +496,7 @@ public: #define GAPI_OCV_KERNEL_ST(Name, API, State) \ struct Name: public cv::GCPUStKernelImpl \ - +/// @private class gapi::cpu::GOCVFunctor : public gapi::GFunctor { public: diff --git a/modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp b/modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp index dbb36d801d..53b68c4e21 100644 --- a/modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/fluid/gfluidkernel.hpp @@ -25,6 +25,9 @@ namespace cv { namespace gapi { +/** + * @brief This namespace contains G-API Fluid backend functions, structures, and symbols. + */ namespace fluid { /** diff --git a/modules/gapi/include/opencv2/gapi/garg.hpp b/modules/gapi/include/opencv2/gapi/garg.hpp index 20f2233bf9..ee6ee81e1c 100644 --- a/modules/gapi/include/opencv2/gapi/garg.hpp +++ b/modules/gapi/include/opencv2/gapi/garg.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018-2020 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_GARG_HPP @@ -171,7 +171,7 @@ using GRunArgs = std::vector; * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GRunArgs usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GRunArgs usage * */ inline GRunArgs& operator += (GRunArgs &lhs, const GRunArgs &rhs) @@ -223,7 +223,7 @@ using GRunArgsP = std::vector; * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GRunArgsP usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GRunArgsP usage * */ inline GRunArgsP& operator += (GRunArgsP &lhs, const GRunArgsP &rhs) @@ -235,8 +235,39 @@ inline GRunArgsP& operator += (GRunArgsP &lhs, const GRunArgsP &rhs) namespace gapi { - GAPI_EXPORTS cv::GRunArgsP bind(cv::GRunArgs &results); - GAPI_EXPORTS cv::GRunArg bind(cv::GRunArgP &out); // FIXME: think more about it +/** + * \addtogroup gapi_serialization + * @{ + * + * @brief G-API functions and classes for serialization and deserialization. + */ +/** @brief Wraps deserialized output GRunArgs to GRunArgsP which can be used by GCompiled. + * + * Since it's impossible to get modifiable output arguments from deserialization + * it needs to be wrapped by this function. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp bind after deserialization + * + * @param out_args deserialized GRunArgs. + * @return the same GRunArgs wrapped in GRunArgsP. + * @see deserialize + */ +GAPI_EXPORTS cv::GRunArgsP bind(cv::GRunArgs &out_args); +/** @brief Wraps output GRunArgsP available during graph execution to GRunArgs which can be serialized. + * + * GRunArgsP is pointer-to-value, so to be serialized they need to be binded to real values + * which this function does. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp bind before serialization + * + * @param out output GRunArgsP available during graph execution. + * @return the same GRunArgsP wrapped in serializable GRunArgs. + * @see serialize + */ +GAPI_EXPORTS cv::GRunArg bind(cv::GRunArgP &out); // FIXME: think more about it +/** @} */ } template inline GRunArgs gin(const Ts&... args) diff --git a/modules/gapi/include/opencv2/gapi/garray.hpp b/modules/gapi/include/opencv2/gapi/garray.hpp index 32799bc07e..d7b365f7f9 100644 --- a/modules/gapi/include/opencv2/gapi/garray.hpp +++ b/modules/gapi/include/opencv2/gapi/garray.hpp @@ -340,21 +340,79 @@ namespace detail /** \addtogroup gapi_data_objects * @{ */ - +/** + * @brief `cv::GArray` template class represents a list of objects + * of class `T` in the graph. + * + * `cv::GArray` describes a functional relationship between + * operations consuming and producing arrays of objects of class + * `T`. The primary purpose of `cv::GArray` is to represent a + * dynamic list of objects -- where the size of the list is not known + * at the graph construction or compile time. Examples include: corner + * and feature detectors (`cv::GArray`), object detection + * and tracking results (`cv::GArray`). Programmers can use + * their own types with `cv::GArray` in the custom operations. + * + * Similar to `cv::GScalar`, `cv::GArray` may be value-initialized + * -- in this case a graph-constant value is associated with the object. + * + * `GArray` is a virtual counterpart of `std::vector`, which is + * usually used to represent the `GArray` data in G-API during the + * execution. + * + * @sa `cv::GOpaque` + */ template class GArray { public: // Host type (or Flat type) - the type this GArray is actually // specified to. + /// @private using HT = typename detail::flatten_g::type>::type; + /** + * @brief Constructs a value-initialized `cv::GArray` + * + * `cv::GArray` objects may have their values + * be associated at graph construction time. It is useful when + * some operation has a `cv::GArray` input which doesn't change during + * the program execution, and is set only once. In this case, + * there is no need to declare such `cv::GArray` as a graph input. + * + * @note The value of `cv::GArray` may be overwritten by assigning some + * other `cv::GArray` to the object using `operator=` -- on the + * assigment, the old association or value is discarded. + * + * @param v a std::vector to associate with this + * `cv::GArray` object. Vector data is copied into the + * `cv::GArray` (no reference to the passed data is held). + */ explicit GArray(const std::vector& v) // Constant value constructor : m_ref(detail::GArrayU(detail::VectorRef(v))) { putDetails(); } + + /** + * @overload + * @brief Constructs a value-initialized `cv::GArray` + * + * @param v a std::vector to associate with this + * `cv::GArray` object. Vector data is moved into the `cv::GArray`. + */ explicit GArray(std::vector&& v) // Move-constructor : m_ref(detail::GArrayU(detail::VectorRef(std::move(v)))) { putDetails(); } - GArray() { putDetails(); } // Empty constructor - explicit GArray(detail::GArrayU &&ref) // GArrayU-based constructor - : m_ref(ref) { putDetails(); } // (used by GCall, not for users) + + /** + * @brief Constructs an empty `cv::GArray` + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty `cv::GArray` is assigned to a result + * of some operation, it obtains a functional link to this + * operation (and is not empty anymore). + */ + GArray() { putDetails(); } // Empty constructor + + /// @private + explicit GArray(detail::GArrayU &&ref) // GArrayU-based constructor + : m_ref(ref) { putDetails(); } // (used by GCall, not for users) /// @private detail::GArrayU strip() const { diff --git a/modules/gapi/include/opencv2/gapi/gasync_context.hpp b/modules/gapi/include/opencv2/gapi/gasync_context.hpp index 69ce530fc9..f49b59822d 100644 --- a/modules/gapi/include/opencv2/gapi/gasync_context.hpp +++ b/modules/gapi/include/opencv2/gapi/gasync_context.hpp @@ -17,6 +17,13 @@ namespace cv { namespace gapi{ + +/** + * @brief This namespace contains experimental G-API functionality, + * functions or structures in this namespace are subjects to change or + * removal in the future releases. This namespace also contains + * functions which API is not stabilized yet. + */ namespace wip { /** diff --git a/modules/gapi/include/opencv2/gapi/gcommon.hpp b/modules/gapi/include/opencv2/gapi/gcommon.hpp index a9cb015901..d3c280816f 100644 --- a/modules/gapi/include/opencv2/gapi/gcommon.hpp +++ b/modules/gapi/include/opencv2/gapi/gcommon.hpp @@ -44,6 +44,7 @@ namespace detail CV_UNKNOWN, // Unknown, generic, opaque-to-GAPI data type unsupported in graph seriallization CV_BOOL, // bool user G-API data CV_INT, // int user G-API data + CV_INT64, // int64_t user G-API data CV_DOUBLE, // double user G-API data CV_FLOAT, // float user G-API data CV_UINT64, // uint64_t user G-API data @@ -61,6 +62,7 @@ namespace detail template struct GOpaqueTraits; template struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_UNKNOWN; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_INT; }; + template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_INT64; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_DOUBLE; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_FLOAT; }; template<> struct GOpaqueTraits { static constexpr const OpaqueKind kind = OpaqueKind::CV_UINT64; }; diff --git a/modules/gapi/include/opencv2/gapi/gcomputation.hpp b/modules/gapi/include/opencv2/gapi/gcomputation.hpp index a3566fb495..7e07a587e1 100644 --- a/modules/gapi/include/opencv2/gapi/gcomputation.hpp +++ b/modules/gapi/include/opencv2/gapi/gcomputation.hpp @@ -437,11 +437,7 @@ public: * * @sa @ref gapi_compile_args */ - GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {}); - - /// @private -- Exclude this function from OpenCV documentation - GAPI_WRAP GStreamingCompiled compileStreaming(const cv::detail::ExtractMetaCallback &callback, - GCompileArgs &&args = {}); + GAPI_WRAP GStreamingCompiled compileStreaming(GMetaArgs &&in_metas, GCompileArgs &&args = {}); /** * @brief Compile the computation for streaming mode. @@ -464,6 +460,10 @@ public: */ GAPI_WRAP GStreamingCompiled compileStreaming(GCompileArgs &&args = {}); + /// @private -- Exclude this function from OpenCV documentation + GAPI_WRAP GStreamingCompiled compileStreaming(const cv::detail::ExtractMetaCallback &callback, + GCompileArgs &&args = {}); + // 2. Direct metadata version /** * @overload diff --git a/modules/gapi/include/opencv2/gapi/gframe.hpp b/modules/gapi/include/opencv2/gapi/gframe.hpp index 13fd5d6d29..96913dc4cc 100644 --- a/modules/gapi/include/opencv2/gapi/gframe.hpp +++ b/modules/gapi/include/opencv2/gapi/gframe.hpp @@ -28,14 +28,54 @@ struct GOrigin; /** \addtogroup gapi_data_objects * @{ */ +/** + * @brief GFrame class represents an image or media frame in the graph. + * + * GFrame doesn't store any data itself, instead it describes a + * functional relationship between operations consuming and producing + * GFrame objects. + * + * GFrame is introduced to handle various media formats (e.g., NV12 or + * I420) under the same type. Various image formats may differ in the + * number of planes (e.g. two for NV12, three for I420) and the pixel + * layout inside. GFrame type allows to handle these media formats in + * the graph uniformly -- the graph structure will not change if the + * media format changes, e.g. a different camera or decoder is used + * with the same graph. G-API provides a number of operations which + * operate directly on GFrame, like `infer<>()` or + * renderFrame(); these operations are expected to handle different + * media formats inside. There is also a number of accessor + * operations like BGR(), Y(), UV() -- these operations provide + * access to frame's data in the familiar cv::GMat form, which can be + * used with the majority of the existing G-API operations. These + * accessor functions may perform color space converion on the fly if + * the image format of the GFrame they are applied to differs from the + * operation's semantic (e.g. the BGR() accessor is called on an NV12 + * image frame). + * + * GFrame is a virtual counterpart of cv::MediaFrame. + * + * @sa cv::MediaFrame, cv::GFrameDesc, BGR(), Y(), UV(), infer<>(). + */ class GAPI_EXPORTS_W_SIMPLE GFrame { public: - GAPI_WRAP GFrame(); // Empty constructor - GFrame(const GNode &n, std::size_t out); // Operation result constructor + /** + * @brief Constructs an empty GFrame + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty GFrame is assigned to a result of some + * operation, it obtains a functional link to this operation (and + * is not empty anymore). + */ + GAPI_WRAP GFrame(); // Empty constructor - GOrigin& priv(); // Internal use only - const GOrigin& priv() const; // Internal use only + /// @private + GFrame(const GNode &n, std::size_t out); // Operation result constructor + /// @private + GOrigin& priv(); // Internal use only + /// @private + const GOrigin& priv() const; // Internal use only private: std::shared_ptr m_priv; diff --git a/modules/gapi/include/opencv2/gapi/gkernel.hpp b/modules/gapi/include/opencv2/gapi/gkernel.hpp index f70e50253d..cfac552bfe 100644 --- a/modules/gapi/include/opencv2/gapi/gkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/gkernel.hpp @@ -372,6 +372,7 @@ namespace gapi { // Prework: model "Device" API before it gets to G-API headers. // FIXME: Don't mix with internal Backends class! + /// @private class GAPI_EXPORTS GBackend { public: @@ -412,6 +413,7 @@ namespace std namespace cv { namespace gapi { + /// @private class GFunctor { public: diff --git a/modules/gapi/include/opencv2/gapi/gmat.hpp b/modules/gapi/include/opencv2/gapi/gmat.hpp index 11c3071f03..7bea97bbc5 100644 --- a/modules/gapi/include/opencv2/gapi/gmat.hpp +++ b/modules/gapi/include/opencv2/gapi/gmat.hpp @@ -30,29 +30,57 @@ struct GOrigin; * @brief G-API data objects used to build G-API expressions. * * These objects do not own any particular data (except compile-time - * associated values like with cv::GScalar) and are used to construct - * graphs. + * associated values like with cv::GScalar or `cv::GArray`) and are + * used only to construct graphs. * * Every graph in G-API starts and ends with data objects. * * Once constructed and compiled, G-API operates with regular host-side * data instead. Refer to the below table to find the mapping between - * G-API and regular data types. + * G-API and regular data types when passing input and output data + * structures to G-API: * * G-API data type | I/O data type * ------------------ | ------------- - * cv::GMat | cv::Mat + * cv::GMat | cv::Mat, cv::UMat, cv::RMat * cv::GScalar | cv::Scalar * `cv::GArray` | std::vector * `cv::GOpaque` | T + * cv::GFrame | cv::MediaFrame + */ +/** + * @brief GMat class represents image or tensor data in the + * graph. + * + * GMat doesn't store any data itself, instead it describes a + * functional relationship between operations consuming and producing + * GMat objects. + * + * GMat is a virtual counterpart of Mat and UMat, but it + * doesn't mean G-API use Mat or UMat objects internally to represent + * GMat objects -- the internal data representation may be + * backend-specific or optimized out at all. + * + * @sa Mat, GMatDesc */ class GAPI_EXPORTS_W_SIMPLE GMat { public: + /** + * @brief Constructs an empty GMat + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty GMat is assigned to a result of some + * operation, it obtains a functional link to this operation (and + * is not empty anymore). + */ GAPI_WRAP GMat(); // Empty constructor - GMat(const GNode &n, std::size_t out); // Operation result constructor + /// @private + GMat(const GNode &n, std::size_t out); // Operation result constructor + /// @private GOrigin& priv(); // Internal use only + /// @private const GOrigin& priv() const; // Internal use only private: diff --git a/modules/gapi/include/opencv2/gapi/gopaque.hpp b/modules/gapi/include/opencv2/gapi/gopaque.hpp index 00f0718422..979a9db8c4 100644 --- a/modules/gapi/include/opencv2/gapi/gopaque.hpp +++ b/modules/gapi/include/opencv2/gapi/gopaque.hpp @@ -307,15 +307,40 @@ namespace detail /** \addtogroup gapi_data_objects * @{ */ - +/** + * @brief `cv::GOpaque` template class represents an object of + * class `T` in the graph. + * + * `cv::GOpaque` describes a functional relationship between operations + * consuming and producing object of class `T`. `cv::GOpaque` is + * designed to extend G-API with user-defined data types, which are + * often required with user-defined operations. G-API can't apply any + * optimizations to user-defined types since these types are opaque to + * the framework. However, there is a number of G-API operations + * declared with `cv::GOpaque` as a return type, + * e.g. cv::gapi::streaming::timestamp() or cv::gapi::streaming::size(). + * + * @sa `cv::GArray` + */ template class GOpaque { public: // Host type (or Flat type) - the type this GOpaque is actually // specified to. + /// @private using HT = typename detail::flatten_g>::type; + /** + * @brief Constructs an empty `cv::GOpaque` + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty `cv::GOpaque` is assigned to a result + * of some operation, it obtains a functional link to this + * operation (and is not empty anymore). + */ GOpaque() { putDetails(); } // Empty constructor + + /// @private explicit GOpaque(detail::GOpaqueU &&ref) // GOpaqueU-based constructor : m_ref(ref) { putDetails(); } // (used by GCall, not for users) diff --git a/modules/gapi/include/opencv2/gapi/gproto.hpp b/modules/gapi/include/opencv2/gapi/gproto.hpp index fbcccb38ea..6271e470b0 100644 --- a/modules/gapi/include/opencv2/gapi/gproto.hpp +++ b/modules/gapi/include/opencv2/gapi/gproto.hpp @@ -71,7 +71,7 @@ public: * It's an ordinary overload of addition assignment operator. * * Example of usage: - * @snippet dynamic_graph.cpp GIOProtoArgs usage + * @snippet modules/gapi/samples/dynamic_graph.cpp GIOProtoArgs usage * */ template diff --git a/modules/gapi/include/opencv2/gapi/gscalar.hpp b/modules/gapi/include/opencv2/gapi/gscalar.hpp index d4af2cab5d..c6edc33b75 100644 --- a/modules/gapi/include/opencv2/gapi/gscalar.hpp +++ b/modules/gapi/include/opencv2/gapi/gscalar.hpp @@ -25,18 +25,83 @@ struct GOrigin; /** \addtogroup gapi_data_objects * @{ */ - +/** + * @brief GScalar class represents cv::Scalar data in the graph. + * + * GScalar may be associated with a cv::Scalar value, which becomes + * its constant value bound in graph compile time. cv::GScalar describes a + * functional relationship between operations consuming and producing + * GScalar objects. + * + * GScalar is a virtual counterpart of cv::Scalar, which is usually used + * to represent the GScalar data in G-API during the execution. + * + * @sa Scalar + */ class GAPI_EXPORTS_W_SIMPLE GScalar { public: - GAPI_WRAP GScalar(); // Empty constructor - explicit GScalar(const cv::Scalar& s); // Constant value constructor from cv::Scalar + /** + * @brief Constructs an empty GScalar + * + * Normally, empty G-API data objects denote a starting point of + * the graph. When an empty GScalar is assigned to a result of some + * operation, it obtains a functional link to this operation (and + * is not empty anymore). + */ + GAPI_WRAP GScalar(); + + /** + * @brief Constructs a value-initialized GScalar + * + * In contrast with GMat (which can be either an explicit graph input + * or a result of some operation), GScalars may have their values + * be associated at graph construction time. It is useful when + * some operation has a GScalar input which doesn't change during + * the program execution, and is set only once. In this case, + * there is no need to declare such GScalar as a graph input. + * + * @note The value of GScalar may be overwritten by assigning some + * other GScalar to the object using `operator=` -- on the + * assigment, the old GScalar value is discarded. + * + * @param s a cv::Scalar value to associate with this GScalar object. + */ + explicit GScalar(const cv::Scalar& s); + + /** + * @overload + * @brief Constructs a value-initialized GScalar + * + * @param s a cv::Scalar value to associate with this GScalar object. + */ explicit GScalar(cv::Scalar&& s); // Constant value move-constructor from cv::Scalar + /** + * @overload + * @brief Constructs a value-initialized GScalar + * + * @param v0 A `double` value to associate with this GScalar. Note + * that only the first component of a four-component cv::Scalar is + * set to this value, with others remain zeros. + * + * This constructor overload is not marked `explicit` and can be + * used in G-API expression code like this: + * + * @snippet modules/gapi/samples/api_ref_snippets.cpp gscalar_implicit + * + * Here operator+(GMat,GScalar) is used to wrap cv::gapi::addC() + * and a value-initialized GScalar is created on the fly. + * + * @overload + */ GScalar(double v0); // Constant value constructor from double - GScalar(const GNode &n, std::size_t out); // Operation result constructor + /// @private + GScalar(const GNode &n, std::size_t out); // Operation result constructor + /// @private GOrigin& priv(); // Internal use only + /// @private const GOrigin& priv() const; // Internal use only private: diff --git a/modules/gapi/include/opencv2/gapi/gstreaming.hpp b/modules/gapi/include/opencv2/gapi/gstreaming.hpp index 371581345f..890eb584fb 100644 --- a/modules/gapi/include/opencv2/gapi/gstreaming.hpp +++ b/modules/gapi/include/opencv2/gapi/gstreaming.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2018 Intel Corporation +// Copyright (C) 2018-2021 Intel Corporation #ifndef OPENCV_GAPI_GSTREAMING_COMPILED_HPP @@ -65,12 +65,23 @@ using OptionalOpaqueRef = OptRef; using GOptRunArgP = util::variant< optional*, optional*, + optional*, optional*, cv::detail::OptionalVectorRef, cv::detail::OptionalOpaqueRef >; using GOptRunArgsP = std::vector; +using GOptRunArg = util::variant< + optional, + optional, + optional, + optional, + optional, + optional +>; +using GOptRunArgs = std::vector; + namespace detail { template inline GOptRunArgP wrap_opt_arg(optional& arg) { @@ -86,6 +97,14 @@ template<> inline GOptRunArgP wrap_opt_arg(optional &m) { return GOptRunArgP{&m}; } +template<> inline GOptRunArgP wrap_opt_arg(optional &m) { + return GOptRunArgP{&m}; +} + +template<> inline GOptRunArgP wrap_opt_arg(optional &f) { + return GOptRunArgP{&f}; +} + template<> inline GOptRunArgP wrap_opt_arg(optional &s) { return GOptRunArgP{&s}; } @@ -196,7 +215,7 @@ public: * @param s a shared pointer to IStreamSource representing the * input video stream. */ - GAPI_WRAP void setSource(const gapi::wip::IStreamSource::Ptr& s); + void setSource(const gapi::wip::IStreamSource::Ptr& s); /** * @brief Constructs and specifies an input video stream for a @@ -255,7 +274,7 @@ public: // NB: Used from python /// @private -- Exclude this function from OpenCV documentation - GAPI_WRAP std::tuple pull(); + GAPI_WRAP std::tuple> pull(); /** * @brief Get some next available data from the pipeline. @@ -372,6 +391,14 @@ protected: /** @} */ namespace gapi { + +/** + * @brief This namespace contains G-API functions, structures, and + * symbols related to the Streaming execution mode. + * + * Some of the operations defined in this namespace (e.g. size(), + * BGR(), etc.) can be used in the traditional execution mode too. + */ namespace streaming { /** * @brief Specify queue capacity for streaming execution. @@ -379,9 +406,11 @@ namespace streaming { * In the streaming mode the pipeline steps are connected with queues * and this compile argument controls every queue's size. */ -struct GAPI_EXPORTS queue_capacity +struct GAPI_EXPORTS_W_SIMPLE queue_capacity { + GAPI_WRAP explicit queue_capacity(size_t cap = 1) : capacity(cap) { }; + GAPI_PROP_RW size_t capacity; }; /** @} */ diff --git a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp index 0b11b18485..2e8dcb1aec 100644 --- a/modules/gapi/include/opencv2/gapi/gtype_traits.hpp +++ b/modules/gapi/include/opencv2/gapi/gtype_traits.hpp @@ -43,19 +43,6 @@ namespace detail GOPAQUE, // a cv::GOpaqueU (note - exactly GOpaqueU, not GOpaque!) }; - template - constexpr const char* meta_to_string() noexcept; - template<> - constexpr const char* meta_to_string() noexcept { return "GMatDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GScalarDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GArrayDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GOpaqueDesc"; } - template<> - constexpr const char* meta_to_string() noexcept { return "GFrameDesc";} - // Describe G-API types (G-types) with traits. Mostly used by // cv::GArg to store meta information about types passed into // operation arguments. Please note that cv::GComputation is diff --git a/modules/gapi/include/opencv2/gapi/gtyped.hpp b/modules/gapi/include/opencv2/gapi/gtyped.hpp index 27d9777944..6fe52a62e1 100644 --- a/modules/gapi/include/opencv2/gapi/gtyped.hpp +++ b/modules/gapi/include/opencv2/gapi/gtyped.hpp @@ -35,7 +35,6 @@ namespace detail template<> struct ProtoToMeta { using type = cv::GScalarDesc; }; template struct ProtoToMeta > { using type = cv::GArrayDesc; }; template struct ProtoToMeta > { using type = cv::GOpaqueDesc; }; - template<> struct ProtoToMeta { using type = cv::GFrameDesc; }; template using ProtoToMetaT = typename ProtoToMeta::type; //workaround for MSVC 19.0 bug diff --git a/modules/gapi/include/opencv2/gapi/imgproc.hpp b/modules/gapi/include/opencv2/gapi/imgproc.hpp index 2dbe626ff1..5c4c6f7031 100644 --- a/modules/gapi/include/opencv2/gapi/imgproc.hpp +++ b/modules/gapi/include/opencv2/gapi/imgproc.hpp @@ -47,6 +47,10 @@ void validateFindingContoursMeta(const int depth, const int chan, const int mode namespace cv { namespace gapi { +/** + * @brief This namespace contains G-API Operation Types for OpenCV + * ImgProc module functionality. + */ namespace imgproc { using GMat2 = std::tuple; using GMat3 = std::tuple; // FIXME: how to avoid this? diff --git a/modules/gapi/include/opencv2/gapi/infer.hpp b/modules/gapi/include/opencv2/gapi/infer.hpp index eb9b9768c9..807c82d31f 100644 --- a/modules/gapi/include/opencv2/gapi/infer.hpp +++ b/modules/gapi/include/opencv2/gapi/infer.hpp @@ -136,11 +136,12 @@ public: } template - void setInput(const std::string& name, U in) + GInferInputsTyped& setInput(const std::string& name, U in) { m_priv->blobs.emplace(std::piecewise_construct, std::forward_as_tuple(name), std::forward_as_tuple(in)); + return *this; } using StorageT = cv::util::variant; @@ -653,7 +654,8 @@ namespace gapi { // A type-erased form of network parameters. // Similar to how a type-erased GKernel is represented and used. -struct GAPI_EXPORTS GNetParam { +/// @private +struct GAPI_EXPORTS_W_SIMPLE GNetParam { std::string tag; // FIXME: const? GBackend backend; // Specifies the execution model util::any params; // Backend-interpreted parameter structure @@ -664,12 +666,13 @@ struct GAPI_EXPORTS GNetParam { */ /** * @brief A container class for network configurations. Similar to - * GKernelPackage.Use cv::gapi::networks() to construct this object. + * GKernelPackage. Use cv::gapi::networks() to construct this object. * * @sa cv::gapi::networks */ struct GAPI_EXPORTS_W_SIMPLE GNetPackage { GAPI_WRAP GNetPackage() = default; + GAPI_WRAP explicit GNetPackage(std::vector nets); explicit GNetPackage(std::initializer_list ii); std::vector backends() const; std::vector networks; diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp index fdd4128b1a..92ef2101a1 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_ie.hpp @@ -22,17 +22,28 @@ namespace ie { // This class can be marked as SIMPLE, because it's implemented as pimpl class GAPI_EXPORTS_W_SIMPLE PyParams { public: + GAPI_WRAP PyParams() = default; + GAPI_WRAP PyParams(const std::string &tag, const std::string &model, const std::string &weights, const std::string &device); + GAPI_WRAP PyParams(const std::string &tag, const std::string &model, const std::string &device); + GAPI_WRAP + PyParams& constInput(const std::string &layer_name, + const cv::Mat &data, + TraitAs hint = TraitAs::TENSOR); + + GAPI_WRAP + PyParams& cfgNumRequests(size_t nireq); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index 0bcb8e6210..2be739e518 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -24,6 +24,11 @@ namespace cv { namespace gapi { // FIXME: introduce a new sub-namespace for NN? + +/** + * @brief This namespace contains G-API OpenVINO backend functions, + * structures, and symbols. + */ namespace ie { GAPI_EXPORTS cv::gapi::GBackend backend(); @@ -69,7 +74,11 @@ struct ParamDesc { std::map> reshape_table; std::unordered_set layer_names_to_reshape; + // NB: Number of asyncrhonious infer requests size_t nireq; + + // NB: An optional config to setup RemoteContext for IE + cv::util::any context_config; }; } // namespace detail @@ -110,7 +119,8 @@ public: , {} , {} , {} - , 1u} { + , 1u + , {}} { }; /** @overload @@ -130,7 +140,8 @@ public: , {} , {} , {} - , 1u} { + , 1u + , {}} { }; /** @brief Specifies sequence of network input layers names for inference. @@ -212,6 +223,30 @@ public: return *this; } + /** @brief Specifies configuration for RemoteContext in InferenceEngine. + + When RemoteContext is configured the backend imports the networks using the context. + It also expects cv::MediaFrames to be actually remote, to operate with blobs via the context. + + @param ctx_cfg cv::util::any value which holds InferenceEngine::ParamMap. + @return reference to this parameter structure. + */ + Params& cfgContextParams(const cv::util::any& ctx_cfg) { + desc.context_config = ctx_cfg; + return *this; + } + + /** @overload + Function with an rvalue parameter. + + @param ctx_cfg cv::util::any value which holds InferenceEngine::ParamMap. + @return reference to this parameter structure. + */ + Params& cfgContextParams(cv::util::any&& ctx_cfg) { + desc.context_config = std::move(ctx_cfg); + return *this; + } + /** @brief Specifies number of asynchronous inference requests. @param nireq Number of inference asynchronous requests. @@ -313,7 +348,10 @@ public: const std::string &model, const std::string &weights, const std::string &device) - : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u}, m_tag(tag) { + : desc{ model, weights, device, {}, {}, {}, 0u, 0u, + detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u, + {}}, + m_tag(tag) { }; /** @overload @@ -328,7 +366,10 @@ public: Params(const std::string &tag, const std::string &model, const std::string &device) - : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u}, m_tag(tag) { + : desc{ model, {}, device, {}, {}, {}, 0u, 0u, + detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u, + {}}, + m_tag(tag) { }; /** @see ie::Params::pluginConfig. */ diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index 761f688669..bb5ef6c59e 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -20,6 +20,10 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API ONNX Runtime backend functions, structures, and symbols. + */ namespace onnx { GAPI_EXPORTS cv::gapi::GBackend backend(); diff --git a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp index 3225c73831..c7308dd39f 100644 --- a/modules/gapi/include/opencv2/gapi/infer/parsers.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/parsers.hpp @@ -64,12 +64,13 @@ detection is smaller than confidence threshold, detection is rejected. given label will get to the output. @return a tuple with a vector of detected boxes and a vector of appropriate labels. */ -GAPI_EXPORTS std::tuple, GArray> parseSSD(const GMat& in, - const GOpaque& inSz, - const float confidenceThreshold = 0.5f, - const int filterLabel = -1); +GAPI_EXPORTS_W std::tuple, GArray> parseSSD(const GMat& in, + const GOpaque& inSz, + const float confidenceThreshold = 0.5f, + const int filterLabel = -1); + +/** @brief Parses output of SSD network. -/** @overload Extracts detection information (box, confidence) from SSD output and filters it by given confidence and by going out of bounds. @@ -87,9 +88,9 @@ the larger side of the rectangle. */ GAPI_EXPORTS_W GArray parseSSD(const GMat& in, const GOpaque& inSz, - const float confidenceThreshold = 0.5f, - const bool alignmentToSquare = false, - const bool filterOutOfBounds = false); + const float confidenceThreshold, + const bool alignmentToSquare, + const bool filterOutOfBounds); /** @brief Parses output of Yolo network. @@ -112,12 +113,12 @@ If 1.f, nms is not performed and no boxes are rejected. documentation. @return a tuple with a vector of detected boxes and a vector of appropriate labels. */ -GAPI_EXPORTS std::tuple, GArray> parseYolo(const GMat& in, - const GOpaque& inSz, - const float confidenceThreshold = 0.5f, - const float nmsThreshold = 0.5f, - const std::vector& anchors - = nn::parsers::GParseYolo::defaultAnchors()); +GAPI_EXPORTS_W std::tuple, GArray> parseYolo(const GMat& in, + const GOpaque& inSz, + const float confidenceThreshold = 0.5f, + const float nmsThreshold = 0.5f, + const std::vector& anchors + = nn::parsers::GParseYolo::defaultAnchors()); } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/media.hpp b/modules/gapi/include/opencv2/gapi/media.hpp index 61a7098b55..19aaef3fd1 100644 --- a/modules/gapi/include/opencv2/gapi/media.hpp +++ b/modules/gapi/include/opencv2/gapi/media.hpp @@ -15,30 +15,119 @@ #include #include +// Forward declaration +namespace cv { +namespace gapi { +namespace s11n { +struct IOStream; +struct IIStream; +} // namespace s11n +} // namespace gapi +} // namespace cv + namespace cv { +/** \addtogroup gapi_data_structures + * @{ + * + * @brief Extra G-API data structures used to pass input/output data + * to the graph for processing. + */ +/** + * @brief cv::MediaFrame class represents an image/media frame + * obtained from an external source. + * + * cv::MediaFrame represents image data as specified in + * cv::MediaFormat. cv::MediaFrame is designed to be a thin wrapper over some + * external memory of buffer; the class itself provides an uniform + * interface over such types of memory. cv::MediaFrame wraps data from + * a camera driver or from a media codec and provides an abstraction + * layer over this memory to G-API. MediaFrame defines a compact interface + * to access and manage the underlying data; the implementation is + * fully defined by the associated Adapter (which is usually + * user-defined). + * + * @sa cv::RMat + */ class GAPI_EXPORTS MediaFrame { public: - enum class Access { R, W }; + /// This enum defines different types of cv::MediaFrame provided + /// access to the underlying data. Note that different flags can't + /// be combined in this version. + enum class Access { + R, ///< Access data for reading + W, ///< Access data for writing + }; class IAdapter; class View; using AdapterPtr = std::unique_ptr; + /** + * @brief Constructs an empty MediaFrame + * + * The constructed object has no any data associated with it. + */ MediaFrame(); - explicit MediaFrame(AdapterPtr &&); - template static cv::MediaFrame Create(Args&&...); - View access(Access) const; + /** + * @brief Constructs a MediaFrame with the given + * Adapter. MediaFrame takes ownership over the passed adapter. + * + * @param p an unique pointer to instance of IAdapter derived class. + */ + explicit MediaFrame(AdapterPtr &&p); + + /** + * @overload + * @brief Constructs a MediaFrame with the given parameters for + * the Adapter. The adapter of type `T` is costructed on the fly. + * + * @param args list of arguments to construct an adapter of type + * `T`. + */ + template static cv::MediaFrame Create(Args&&... args); + + /** + * @brief Obtain access to the underlying data with the given + * mode. + * + * Depending on the associated Adapter and the data wrapped, this + * method may be cheap (e.g., the underlying memory is local) or + * costly (if the underlying memory is external or device + * memory). + * + * @param mode an access mode flag + * @return a MediaFrame::View object. The views should be handled + * carefully, refer to the MediaFrame::View documentation for details. + */ + View access(Access mode) const; + + /** + * @brief Returns a media frame descriptor -- the information + * about the media format, dimensions, etc. + * @return a cv::GFrameDesc + */ cv::GFrameDesc desc() const; // FIXME: design a better solution // Should be used only if the actual adapter provides implementation + /// @private -- exclude from the OpenCV documentation for now. cv::util::any blobParams() const; - // Cast underlying MediaFrame adapter to the particular adapter type, - // return nullptr if underlying type is different - template T* get() const - { + /** + * @brief Casts and returns the associated MediaFrame adapter to + * the particular adapter type `T`, returns nullptr if the type is + * different. + * + * This method may be useful if the adapter type is known by the + * caller, and some lower level access to the memory is required. + * Depending on the memory type, it may be more efficient than + * access(). + * + * @return a pointer to the adapter object, nullptr if the adapter + * type is different. + */ + template T* get() const { static_assert(std::is_base_of::value, "T is not derived from cv::MediaFrame::IAdapter!"); auto* adapter = getAdapter(); @@ -46,6 +135,16 @@ public: return dynamic_cast(adapter); } + /** + * @brief Serialize MediaFrame's data to a byte array. + * + * @note The actual logic is implemented by frame's adapter class. + * Does nothing by default. + * + * @param os Bytestream to store serialized MediaFrame data in. + */ + void serialize(cv::gapi::s11n::IOStream& os) const; + private: struct Priv; std::shared_ptr m; @@ -58,6 +157,43 @@ inline cv::MediaFrame cv::MediaFrame::Create(Args&&... args) { return cv::MediaFrame(std::move(ptr)); } +/** + * @brief Provides access to the MediaFrame's underlying data. + * + * This object contains the necessary information to access the pixel + * data of the associated MediaFrame: arrays of pointers and strides + * (distance between every plane row, in bytes) for every image + * plane, as defined in cv::MediaFormat. + * There may be up to four image planes in MediaFrame. + * + * Depending on the MediaFrame::Access flag passed in + * MediaFrame::access(), a MediaFrame::View may be read- or + * write-only. + * + * Depending on the MediaFrame::IAdapter implementation associated + * with the parent MediaFrame, writing to memory with + * MediaFrame::Access::R flag may have no effect or lead to + * undefined behavior. Same applies to reading the memory with + * MediaFrame::Access::W flag -- again, depending on the IAdapter + * implementation, the host-side buffer the view provides access to + * may have no current data stored in (so in-place editing of the + * buffer contents may not be possible). + * + * MediaFrame::View objects must be handled carefully, as an external + * resource associated with MediaFrame may be locked for the time the + * MediaFrame::View object exists. Obtaining MediaFrame::View should + * be seen as "map" and destroying it as "unmap" in the "map/unmap" + * idiom (applicable to OpenCL, device memory, remote + * memory). + * + * When a MediaFrame buffer is accessed for writing, and the memory + * under MediaFrame::View::Ptrs is altered, the data synchronization + * of a host-side and device/remote buffer is not guaranteed until the + * MediaFrame::View is destroyed. In other words, the real data on the + * device or in a remote target may be updated at the MediaFrame::View + * destruction only -- but it depends on the associated + * MediaFrame::IAdapter implementation. + */ class GAPI_EXPORTS MediaFrame::View final { public: static constexpr const size_t MAX_PLANES = 4; @@ -65,19 +201,38 @@ public: using Strides = std::array; // in bytes using Callback = std::function; + /// @private View(Ptrs&& ptrs, Strides&& strs, Callback &&cb = [](){}); + + /// @private View(const View&) = delete; + + /// @private View(View&&) = default; + + /// @private View& operator = (const View&) = delete; + ~View(); - Ptrs ptr; - Strides stride; + Ptrs ptr; ///< Array of image plane pointers + Strides stride; ///< Array of image plane strides, in bytes. private: Callback m_cb; }; +/** + * @brief An interface class for MediaFrame data adapters. + * + * Implement this interface to wrap media data in the MediaFrame. It + * makes sense to implement this class if there is a custom + * cv::gapi::wip::IStreamSource defined -- in this case, a stream + * source can produce MediaFrame objects with this adapter and the + * media data may be passed to graph without any copy. For example, a + * GStreamer-based stream source can implement an adapter over + * `GstBuffer` and G-API will transparently use it in the graph. + */ class GAPI_EXPORTS MediaFrame::IAdapter { public: virtual ~IAdapter() = 0; @@ -86,7 +241,16 @@ public: // FIXME: design a better solution // The default implementation does nothing virtual cv::util::any blobParams() const; + virtual void serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert(false && "Generic serialize method of MediaFrame::IAdapter does nothing by default. " + "Please, implement it in derived class to properly serialize the object."); + } + virtual void deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "Generic deserialize method of MediaFrame::IAdapter does nothing by default. " + "Please, implement it in derived class to properly deserialize the object."); + } }; +/** @} */ } //namespace cv diff --git a/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp b/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp index 8ec388d588..6a2f1df769 100644 --- a/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp @@ -29,6 +29,9 @@ namespace gimpl namespace gapi { +/** + * @brief This namespace contains G-API OpenCL backend functions, structures, and symbols. + */ namespace ocl { /** diff --git a/modules/gapi/include/opencv2/gapi/own/exports.hpp b/modules/gapi/include/opencv2/gapi/own/exports.hpp index 1978991b75..c36f4003d0 100644 --- a/modules/gapi/include/opencv2/gapi/own/exports.hpp +++ b/modules/gapi/include/opencv2/gapi/own/exports.hpp @@ -13,11 +13,13 @@ # define GAPI_EXPORTS CV_EXPORTS /* special informative macros for wrapper generators */ # define GAPI_PROP CV_PROP +# define GAPI_PROP_RW CV_PROP_RW # define GAPI_WRAP CV_WRAP # define GAPI_EXPORTS_W_SIMPLE CV_EXPORTS_W_SIMPLE # define GAPI_EXPORTS_W CV_EXPORTS_W # else # define GAPI_PROP +# define GAPI_PROP_RW # define GAPI_WRAP # define GAPI_EXPORTS # define GAPI_EXPORTS_W_SIMPLE diff --git a/modules/gapi/include/opencv2/gapi/own/types.hpp b/modules/gapi/include/opencv2/gapi/own/types.hpp index c77a62ca53..6bd6880982 100644 --- a/modules/gapi/include/opencv2/gapi/own/types.hpp +++ b/modules/gapi/include/opencv2/gapi/own/types.hpp @@ -15,6 +15,11 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API own data structures used in + * its standalone mode build. + */ namespace own { @@ -22,7 +27,7 @@ class Point { public: Point() = default; - Point(int _x, int _y) : x(_x), y(_y) {}; + Point(int _x, int _y) : x(_x), y(_y) {} int x = 0; int y = 0; @@ -32,7 +37,7 @@ class Point2f { public: Point2f() = default; - Point2f(float _x, float _y) : x(_x), y(_y) {}; + Point2f(float _x, float _y) : x(_x), y(_y) {} float x = 0.f; float y = 0.f; @@ -42,9 +47,9 @@ class Rect { public: Rect() = default; - Rect(int _x, int _y, int _width, int _height) : x(_x), y(_y), width(_width), height(_height) {}; + Rect(int _x, int _y, int _width, int _height) : x(_x), y(_y), width(_width), height(_height) {} #if !defined(GAPI_STANDALONE) - Rect(const cv::Rect& other) : x(other.x), y(other.y), width(other.width), height(other.height) {}; + Rect(const cv::Rect& other) : x(other.x), y(other.y), width(other.width), height(other.height) {} inline Rect& operator=(const cv::Rect& other) { x = other.x; @@ -99,9 +104,9 @@ class Size { public: Size() = default; - Size(int _width, int _height) : width(_width), height(_height) {}; + Size(int _width, int _height) : width(_width), height(_height) {} #if !defined(GAPI_STANDALONE) - Size(const cv::Size& other) : width(other.width), height(other.height) {}; + Size(const cv::Size& other) : width(other.width), height(other.height) {} inline Size& operator=(const cv::Size& rhs) { width = rhs.width; diff --git a/modules/gapi/include/opencv2/gapi/plaidml/plaidml.hpp b/modules/gapi/include/opencv2/gapi/plaidml/plaidml.hpp index bd12d25832..3207a8cb2e 100644 --- a/modules/gapi/include/opencv2/gapi/plaidml/plaidml.hpp +++ b/modules/gapi/include/opencv2/gapi/plaidml/plaidml.hpp @@ -15,6 +15,11 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API PlaidML backend functions, + * structures, and symbols. + */ namespace plaidml { diff --git a/modules/gapi/include/opencv2/gapi/python/python.hpp b/modules/gapi/include/opencv2/gapi/python/python.hpp index 1c85d69d9f..0e20bbbb59 100644 --- a/modules/gapi/include/opencv2/gapi/python/python.hpp +++ b/modules/gapi/include/opencv2/gapi/python/python.hpp @@ -13,6 +13,15 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API Python backend functions, + * structures, and symbols. + * + * This functionality is required to enable G-API custom operations + * and kernels when using G-API from Python, no need to use it in the + * C++ form. + */ namespace python { GAPI_EXPORTS cv::gapi::GBackend backend(); diff --git a/modules/gapi/include/opencv2/gapi/render/render.hpp b/modules/gapi/include/opencv2/gapi/render/render.hpp index a84c26c810..5375412224 100644 --- a/modules/gapi/include/opencv2/gapi/render/render.hpp +++ b/modules/gapi/include/opencv2/gapi/render/render.hpp @@ -81,9 +81,9 @@ using GMatDesc2 = std::tuple; @param prims vector of drawing primitivies @param args graph compile time parameters */ -void GAPI_EXPORTS render(cv::Mat& bgr, - const Prims& prims, - cv::GCompileArgs&& args = {}); +void GAPI_EXPORTS_W render(cv::Mat& bgr, + const Prims& prims, + cv::GCompileArgs&& args = {}); /** @brief The function renders on two NV12 planes passed drawing primitivies @@ -92,10 +92,10 @@ void GAPI_EXPORTS render(cv::Mat& bgr, @param prims vector of drawing primitivies @param args graph compile time parameters */ -void GAPI_EXPORTS render(cv::Mat& y_plane, - cv::Mat& uv_plane, - const Prims& prims, - cv::GCompileArgs&& args = {}); +void GAPI_EXPORTS_W render(cv::Mat& y_plane, + cv::Mat& uv_plane, + const Prims& prims, + cv::GCompileArgs&& args = {}); /** @brief The function renders on the input media frame passed drawing primitivies @@ -139,7 +139,7 @@ Output image must be 8-bit unsigned planar 3-channel image @param src input image: 8-bit unsigned 3-channel image @ref CV_8UC3 @param prims draw primitives */ -GAPI_EXPORTS GMat render3ch(const GMat& src, const GArray& prims); +GAPI_EXPORTS_W GMat render3ch(const GMat& src, const GArray& prims); /** @brief Renders on two planes @@ -150,9 +150,9 @@ uv image must be 8-bit unsigned planar 2-channel image @ref CV_8UC2 @param uv input image: 8-bit unsigned 2-channel image @ref CV_8UC2 @param prims draw primitives */ -GAPI_EXPORTS GMat2 renderNV12(const GMat& y, - const GMat& uv, - const GArray& prims); +GAPI_EXPORTS_W GMat2 renderNV12(const GMat& y, + const GMat& uv, + const GArray& prims); /** @brief Renders Media Frame @@ -169,11 +169,15 @@ GAPI_EXPORTS GFrame renderFrame(const GFrame& m_frame, } // namespace draw } // namespace wip +/** + * @brief This namespace contains G-API CPU rendering backend functions, + * structures, and symbols. See @ref gapi_draw for details. + */ namespace render { namespace ocv { - GAPI_EXPORTS cv::gapi::GKernelPackage kernels(); + GAPI_EXPORTS_W cv::gapi::GKernelPackage kernels(); } // namespace ocv } // namespace render diff --git a/modules/gapi/include/opencv2/gapi/render/render_types.hpp b/modules/gapi/include/opencv2/gapi/render/render_types.hpp index ca403be361..6d70e3a877 100644 --- a/modules/gapi/include/opencv2/gapi/render/render_types.hpp +++ b/modules/gapi/include/opencv2/gapi/render/render_types.hpp @@ -41,7 +41,7 @@ struct freetype_font * * Parameters match cv::putText(). */ -struct Text +struct GAPI_EXPORTS_W_SIMPLE Text { /** * @brief Text constructor @@ -55,6 +55,7 @@ struct Text * @param lt_ The line type. See #LineTypes * @param bottom_left_origin_ When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner */ + GAPI_WRAP Text(const std::string& text_, const cv::Point& org_, int ff_, @@ -68,17 +69,18 @@ struct Text { } + GAPI_WRAP Text() = default; /*@{*/ - std::string text; //!< The text string to be drawn - cv::Point org; //!< The bottom-left corner of the text string in the image - int ff; //!< The font type, see #HersheyFonts - double fs; //!< The font scale factor that is multiplied by the font-specific base size - cv::Scalar color; //!< The text color - int thick; //!< The thickness of the lines used to draw a text - int lt; //!< The line type. See #LineTypes - bool bottom_left_origin; //!< When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner + GAPI_PROP_RW std::string text; //!< The text string to be drawn + GAPI_PROP_RW cv::Point org; //!< The bottom-left corner of the text string in the image + GAPI_PROP_RW int ff; //!< The font type, see #HersheyFonts + GAPI_PROP_RW double fs; //!< The font scale factor that is multiplied by the font-specific base size + GAPI_PROP_RW cv::Scalar color; //!< The text color + GAPI_PROP_RW int thick; //!< The thickness of the lines used to draw a text + GAPI_PROP_RW int lt; //!< The line type. See #LineTypes + GAPI_PROP_RW bool bottom_left_origin; //!< When true, the image data origin is at the bottom-left corner. Otherwise, it is at the top-left corner /*@{*/ }; @@ -122,7 +124,7 @@ struct FText * * Parameters match cv::rectangle(). */ -struct Rect +struct GAPI_EXPORTS_W_SIMPLE Rect { /** * @brief Rect constructor @@ -142,14 +144,15 @@ struct Rect { } + GAPI_WRAP Rect() = default; /*@{*/ - cv::Rect rect; //!< Coordinates of the rectangle - cv::Scalar color; //!< The rectangle color or brightness (grayscale image) - int thick; //!< The thickness of lines that make up the rectangle. Negative values, like #FILLED, mean that the function has to draw a filled rectangle - int lt; //!< The type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinates + GAPI_PROP_RW cv::Rect rect; //!< Coordinates of the rectangle + GAPI_PROP_RW cv::Scalar color; //!< The rectangle color or brightness (grayscale image) + GAPI_PROP_RW int thick; //!< The thickness of lines that make up the rectangle. Negative values, like #FILLED, mean that the function has to draw a filled rectangle + GAPI_PROP_RW int lt; //!< The type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinates /*@{*/ }; @@ -158,7 +161,7 @@ struct Rect * * Parameters match cv::circle(). */ -struct Circle +struct GAPI_EXPORTS_W_SIMPLE Circle { /** * @brief Circle constructor @@ -170,6 +173,7 @@ struct Circle * @param lt_ The Type of the circle boundary. See #LineTypes * @param shift_ The Number of fractional bits in the coordinates of the center and in the radius value */ + GAPI_WRAP Circle(const cv::Point& center_, int radius_, const cv::Scalar& color_, @@ -180,15 +184,16 @@ struct Circle { } + GAPI_WRAP Circle() = default; /*@{*/ - cv::Point center; //!< The center of the circle - int radius; //!< The radius of the circle - cv::Scalar color; //!< The color of the circle - int thick; //!< The thickness of the circle outline, if positive. Negative values, like #FILLED, mean that a filled circle is to be drawn - int lt; //!< The Type of the circle boundary. See #LineTypes - int shift; //!< The Number of fractional bits in the coordinates of the center and in the radius value + GAPI_PROP_RW cv::Point center; //!< The center of the circle + GAPI_PROP_RW int radius; //!< The radius of the circle + GAPI_PROP_RW cv::Scalar color; //!< The color of the circle + GAPI_PROP_RW int thick; //!< The thickness of the circle outline, if positive. Negative values, like #FILLED, mean that a filled circle is to be drawn + GAPI_PROP_RW int lt; //!< The Type of the circle boundary. See #LineTypes + GAPI_PROP_RW int shift; //!< The Number of fractional bits in the coordinates of the center and in the radius value /*@{*/ }; @@ -197,7 +202,7 @@ struct Circle * * Parameters match cv::line(). */ -struct Line +struct GAPI_EXPORTS_W_SIMPLE Line { /** * @brief Line constructor @@ -209,6 +214,7 @@ struct Line * @param lt_ The Type of the line. See #LineTypes * @param shift_ The number of fractional bits in the point coordinates */ + GAPI_WRAP Line(const cv::Point& pt1_, const cv::Point& pt2_, const cv::Scalar& color_, @@ -219,15 +225,16 @@ struct Line { } + GAPI_WRAP Line() = default; /*@{*/ - cv::Point pt1; //!< The first point of the line segment - cv::Point pt2; //!< The second point of the line segment - cv::Scalar color; //!< The line color - int thick; //!< The thickness of line - int lt; //!< The Type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinates + GAPI_PROP_RW cv::Point pt1; //!< The first point of the line segment + GAPI_PROP_RW cv::Point pt2; //!< The second point of the line segment + GAPI_PROP_RW cv::Scalar color; //!< The line color + GAPI_PROP_RW int thick; //!< The thickness of line + GAPI_PROP_RW int lt; //!< The Type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinates /*@{*/ }; @@ -236,7 +243,7 @@ struct Line * * Mosaicing is a very basic method to obfuscate regions in the image. */ -struct Mosaic +struct GAPI_EXPORTS_W_SIMPLE Mosaic { /** * @brief Mosaic constructor @@ -252,12 +259,13 @@ struct Mosaic { } + GAPI_WRAP Mosaic() : cellSz(0), decim(0) {} /*@{*/ - cv::Rect mos; //!< Coordinates of the mosaic - int cellSz; //!< Cell size (same for X, Y) - int decim; //!< Decimation (0 stands for no decimation) + GAPI_PROP_RW cv::Rect mos; //!< Coordinates of the mosaic + GAPI_PROP_RW int cellSz; //!< Cell size (same for X, Y) + GAPI_PROP_RW int decim; //!< Decimation (0 stands for no decimation) /*@{*/ }; @@ -266,7 +274,7 @@ struct Mosaic * * Image is blended on a frame using the specified mask. */ -struct Image +struct GAPI_EXPORTS_W_SIMPLE Image { /** * @brief Mosaic constructor @@ -275,6 +283,7 @@ struct Image * @param img_ Image to draw * @param alpha_ Alpha channel for image to draw (same size and number of channels) */ + GAPI_WRAP Image(const cv::Point& org_, const cv::Mat& img_, const cv::Mat& alpha_) : @@ -282,19 +291,20 @@ struct Image { } + GAPI_WRAP Image() = default; /*@{*/ - cv::Point org; //!< The bottom-left corner of the image - cv::Mat img; //!< Image to draw - cv::Mat alpha; //!< Alpha channel for image to draw (same size and number of channels) + GAPI_PROP_RW cv::Point org; //!< The bottom-left corner of the image + GAPI_PROP_RW cv::Mat img; //!< Image to draw + GAPI_PROP_RW cv::Mat alpha; //!< Alpha channel for image to draw (same size and number of channels) /*@{*/ }; /** * @brief This structure represents a polygon to draw. */ -struct Poly +struct GAPI_EXPORTS_W_SIMPLE Poly { /** * @brief Mosaic constructor @@ -305,6 +315,7 @@ struct Poly * @param lt_ The Type of the line. See #LineTypes * @param shift_ The number of fractional bits in the point coordinate */ + GAPI_WRAP Poly(const std::vector& points_, const cv::Scalar& color_, int thick_ = 1, @@ -314,14 +325,15 @@ struct Poly { } + GAPI_WRAP Poly() = default; /*@{*/ - std::vector points; //!< Points to connect - cv::Scalar color; //!< The line color - int thick; //!< The thickness of line - int lt; //!< The Type of the line. See #LineTypes - int shift; //!< The number of fractional bits in the point coordinate + GAPI_PROP_RW std::vector points; //!< Points to connect + GAPI_PROP_RW cv::Scalar color; //!< The line color + GAPI_PROP_RW int thick; //!< The thickness of line + GAPI_PROP_RW int lt; //!< The Type of the line. See #LineTypes + GAPI_PROP_RW int shift; //!< The number of fractional bits in the point coordinate /*@{*/ }; @@ -336,7 +348,7 @@ using Prim = util::variant , Poly >; -using Prims = std::vector; +using Prims = std::vector; //! @} gapi_draw_prims } // namespace draw diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp index f50bd08b65..6b289001e7 100644 --- a/modules/gapi/include/opencv2/gapi/rmat.hpp +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -14,8 +14,8 @@ namespace cv { namespace gapi { namespace s11n { - struct IOStream; - struct IIStream; +struct IOStream; +struct IIStream; } // namespace s11n } // namespace gapi } // namespace cv @@ -42,6 +42,9 @@ namespace cv { // performCalculations(in_view, out_view); // // data from out_view is transferred to the device when out_view is destroyed // } +/** \addtogroup gapi_data_structures + * @{ + */ class GAPI_EXPORTS RMat { public: @@ -108,10 +111,12 @@ public: // is transferred to the device when the view is destroyed virtual View access(Access) = 0; virtual void serialize(cv::gapi::s11n::IOStream&) { - GAPI_Assert(false && "Generic serialize method should never be called for RMat adapter"); + GAPI_Assert(false && "Generic serialize method of RMat::Adapter does nothing by default. " + "Please, implement it in derived class to properly serialize the object."); } virtual void deserialize(cv::gapi::s11n::IIStream&) { - GAPI_Assert(false && "Generic deserialize method should never be called for RMat adapter"); + GAPI_Assert(false && "Generic deserialize method of RMat::Adapter does nothing by default. " + "Please, implement it in derived class to properly deserialize the object."); } }; using AdapterP = std::shared_ptr; @@ -146,6 +151,7 @@ private: template RMat make_rmat(Ts&&... args) { return { std::make_shared(std::forward(args)...) }; } +/** @} */ } //namespace cv diff --git a/modules/gapi/include/opencv2/gapi/s11n.hpp b/modules/gapi/include/opencv2/gapi/s11n.hpp index 5a64410e5a..53800970d1 100644 --- a/modules/gapi/include/opencv2/gapi/s11n.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n.hpp @@ -13,69 +13,145 @@ #include #include #include +#include +#include + +// FIXME: caused by deserialize_runarg +#if (defined _WIN32 || defined _WIN64) && defined _MSC_VER +#pragma warning(disable: 4702) +#endif namespace cv { namespace gapi { +/** +* \addtogroup gapi_serialization +* @{ +*/ + namespace detail { - GAPI_EXPORTS cv::GComputation getGraph(const std::vector &p); + GAPI_EXPORTS cv::GComputation getGraph(const std::vector &bytes); - GAPI_EXPORTS cv::GMetaArgs getMetaArgs(const std::vector &p); + GAPI_EXPORTS cv::GMetaArgs getMetaArgs(const std::vector &bytes); - GAPI_EXPORTS cv::GRunArgs getRunArgs(const std::vector &p); + GAPI_EXPORTS cv::GRunArgs getRunArgs(const std::vector &bytes); - GAPI_EXPORTS std::vector getVectorOfStrings(const std::vector &p); + GAPI_EXPORTS std::vector getVectorOfStrings(const std::vector &bytes); template - cv::GCompileArgs getCompileArgs(const std::vector &p); + cv::GCompileArgs getCompileArgs(const std::vector &bytes); - template - cv::GRunArgs getRunArgsWithRMats(const std::vector &p); + template + cv::GRunArgs getRunArgsWithAdapters(const std::vector &bytes); } // namespace detail +/** @brief Serialize a graph represented by GComputation into an array of bytes. + * + * Check different overloads for more examples. + * @param c GComputation to serialize. + * @return serialized vector of bytes. + */ GAPI_EXPORTS std::vector serialize(const cv::GComputation &c); -//namespace{ +/** @overload + * @param ca GCompileArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GCompileArgs& ca); + +/** @overload + * @param ma GMetaArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GMetaArgs& ma); + +/** @overload + * @param ra GRunArgs to serialize. + */ +GAPI_EXPORTS std::vector serialize(const cv::GRunArgs& ra); + +/** @overload + * @param vs std::vector to serialize. + */ +GAPI_EXPORTS std::vector serialize(const std::vector& vs); + +/** + * @private + */ template static inline -T deserialize(const std::vector &p); - -//} //ananymous namespace - -GAPI_EXPORTS std::vector serialize(const cv::GCompileArgs&); -GAPI_EXPORTS std::vector serialize(const cv::GMetaArgs&); -GAPI_EXPORTS std::vector serialize(const cv::GRunArgs&); -GAPI_EXPORTS std::vector serialize(const std::vector&); +T deserialize(const std::vector &bytes); +/** @brief Deserialize GComputation from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GComputation object. + */ template<> inline -cv::GComputation deserialize(const std::vector &p) { - return detail::getGraph(p); +cv::GComputation deserialize(const std::vector &bytes) { + return detail::getGraph(bytes); } +/** @brief Deserialize GMetaArgs from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GMetaArgs object. + */ template<> inline -cv::GMetaArgs deserialize(const std::vector &p) { - return detail::getMetaArgs(p); +cv::GMetaArgs deserialize(const std::vector &bytes) { + return detail::getMetaArgs(bytes); } +/** @brief Deserialize GRunArgs from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized GRunArgs object. + */ template<> inline -cv::GRunArgs deserialize(const std::vector &p) { - return detail::getRunArgs(p); +cv::GRunArgs deserialize(const std::vector &bytes) { + return detail::getRunArgs(bytes); } +/** @brief Deserialize std::vector from a byte array. + * + * Check different overloads for more examples. + * @param bytes serialized vector of bytes. + * @return deserialized std::vector object. + */ template<> inline -std::vector deserialize(const std::vector &p) { - return detail::getVectorOfStrings(p); +std::vector deserialize(const std::vector &bytes) { + return detail::getVectorOfStrings(bytes); } +/** + * @brief Deserialize GCompileArgs which types were specified in the template from a byte array. + * + * @note cv::gapi::s11n::detail::S11N template specialization must be provided to make a custom type + * in GCompileArgs deserializable. + * + * @param bytes vector of bytes to deserialize GCompileArgs object from. + * @return GCompileArgs object. + * @see GCompileArgs cv::gapi::s11n::detail::S11N + */ template inline typename std::enable_if::value, GCompileArgs>:: -type deserialize(const std::vector &p) { - return detail::getCompileArgs(p); +type deserialize(const std::vector &bytes) { + return detail::getCompileArgs(bytes); } -template inline +/** + * @brief Deserialize GRunArgs including RMat and MediaFrame objects if any from a byte array. + * + * Adapter types are specified in the template. + * @note To be used properly specified adapter types must overload their deserialize() method. + * @param bytes vector of bytes to deserialize GRunArgs object from. + * @return GRunArgs including RMat and MediaFrame objects if any. + * @see RMat MediaFrame + */ +template inline typename std::enable_if::value, GRunArgs>:: -type deserialize(const std::vector &p) { - return detail::getRunArgsWithRMats(p); +type deserialize(const std::vector &bytes) { + return detail::getRunArgsWithAdapters(bytes); } } // namespace gapi } // namespace cv @@ -83,6 +159,17 @@ type deserialize(const std::vector &p) { namespace cv { namespace gapi { namespace s11n { + +/** @brief This structure is an interface for serialization routines. + * + * It's main purpose is to provide multiple overloads for operator<<() + * with basic C++ in addition to OpenCV/G-API types. + * + * This sctructure can be inherited and further extended with additional types. + * + * For example, it is utilized in cv::gapi::s11n::detail::S11N as input parameter + * in serialize() method. + */ struct GAPI_EXPORTS IOStream { virtual ~IOStream() = default; // Define the native support for basic C++ types at the API level: @@ -99,6 +186,16 @@ struct GAPI_EXPORTS IOStream { virtual IOStream& operator<< (const std::string&) = 0; }; +/** @brief This structure is an interface for deserialization routines. + * + * It's main purpose is to provide multiple overloads for operator>>() + * with basic C++ in addition to OpenCV/G-API types. + * + * This structure can be inherited and further extended with additional types. + * + * For example, it is utilized in cv::gapi::s11n::detail::S11N as input parameter + * in deserialize() method. + */ struct GAPI_EXPORTS IIStream { virtual ~IIStream() = default; virtual IIStream& operator>> (bool &) = 0; @@ -116,7 +213,7 @@ struct GAPI_EXPORTS IIStream { }; namespace detail { -GAPI_EXPORTS std::unique_ptr getInStream(const std::vector &p); +GAPI_EXPORTS std::unique_ptr getInStream(const std::vector &bytes); } // namespace detail //////////////////////////////////////////////////////////////////////////////// @@ -146,24 +243,26 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::Mat &m); // FIXME: for GRunArgs serailization #if !defined(GAPI_STANDALONE) -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::UMat & um); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::UMat & um); #endif // !defined(GAPI_STANDALONE) GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::RMat &r); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::RMat &r); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gapi::wip::IStreamSource::Ptr &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::IStreamSource::Ptr &issptr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gapi::wip::IStreamSource::Ptr &issptr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::VectorRef &vr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::VectorRef &vr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::detail::OpaqueRef &opr); +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::detail::OpaqueRef &opr); -GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &); -GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &); +/// @private -- Exclude this function from OpenCV documentation +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::MediaFrame &mf); +/// @private -- Exclude this function from OpenCV documentation +GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::MediaFrame &mf); // Generic STL types //////////////////////////////////////////////////////////////// template @@ -186,6 +285,7 @@ IIStream& operator>> (IIStream& is, std::map &m) { } return is; } + template IOStream& operator<< (IOStream& os, const std::unordered_map &m) { const uint32_t sz = static_cast(m.size()); @@ -206,6 +306,7 @@ IIStream& operator>> (IIStream& is, std::unordered_map &m) { } return is; } + template IOStream& operator<< (IOStream& os, const std::vector &ts) { const uint32_t sz = static_cast(ts.size()); @@ -233,16 +334,19 @@ template IOStream& put_v(IOStream&, const V&, std::size_t) { GAPI_Assert(false && "variant>>: requested index is invalid"); }; + template IOStream& put_v(IOStream& os, const V& v, std::size_t x) { return (x == 0u) ? os << cv::util::get(v) : put_v(os, v, x-1); } + template IIStream& get_v(IIStream&, V&, std::size_t, std::size_t) { GAPI_Assert(false && "variant<<: requested index is invalid"); } + template IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) { if (i == gi) { @@ -254,11 +358,13 @@ IIStream& get_v(IIStream& is, V& v, std::size_t i, std::size_t gi) { } } // namespace detail +//! @overload template IOStream& operator<< (IOStream& os, const cv::util::variant &v) { os << static_cast(v.index()); return detail::put_v, Ts...>(os, v, v.index()); } +//! @overload template IIStream& operator>> (IIStream& is, cv::util::variant &v) { int idx = -1; @@ -268,6 +374,7 @@ IIStream& operator>> (IIStream& is, cv::util::variant &v) { } // FIXME: consider a better solution +/// @private -- Exclude this function from OpenCV documentation template void getRunArgByIdx (IIStream& is, cv::util::variant &v, uint32_t idx) { is = detail::get_v, Ts...>(is, v, 0u, idx); @@ -298,16 +405,39 @@ static cv::util::optional exec(const std::string& tag, cv::gapi::s1 } }; -template struct deserialize_runarg; +template +struct deserialize_arg_with_adapter; -template +template +struct deserialize_arg_with_adapter { +static GRunArg exec(cv::gapi::s11n::IIStream& is) { + std::unique_ptr ptr(new TA); + ptr->deserialize(is); + return GRunArg { RA(std::move(ptr)) }; +} +}; + +template +struct deserialize_arg_with_adapter { +static GRunArg exec(cv::gapi::s11n::IIStream&) { + GAPI_Assert(false && "No suitable adapter class found during RMat/MediaFrame deserialization. " + "Please, make sure you've passed them in cv::gapi::deserialize() template"); + return GRunArg{}; +} +}; + +template struct deserialize_runarg { static GRunArg exec(cv::gapi::s11n::IIStream& is, uint32_t idx) { if (idx == GRunArg::index_of()) { - auto ptr = std::make_shared(); - ptr->deserialize(is); - return GRunArg { RMat(std::move(ptr)) }; - } else { // non-RMat arg - use default deserialization + // Type or void (if not found) + using TA = typename cv::util::find_adapter_impl::type; + return deserialize_arg_with_adapter::exec(is); + } else if (idx == GRunArg::index_of()) { + // Type or void (if not found) + using TA = typename cv::util::find_adapter_impl::type; + return deserialize_arg_with_adapter::exec(is); + } else { // not an adapter holding type runarg - use default deserialization GRunArg arg; getRunArgByIdx(is, arg, idx); return arg; @@ -350,9 +480,9 @@ cv::GCompileArgs getCompileArgs(const std::vector &sArgs) { return args; } -template -cv::GRunArgs getRunArgsWithRMats(const std::vector &p) { - std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(p); +template +cv::GRunArgs getRunArgsWithAdapters(const std::vector &bytes) { + std::unique_ptr pIs = cv::gapi::s11n::detail::getInStream(bytes); cv::gapi::s11n::IIStream& is = *pIs; cv::GRunArgs args; @@ -361,12 +491,14 @@ cv::GRunArgs getRunArgsWithRMats(const std::vector &p) { for (uint32_t i = 0; i < sz; ++i) { uint32_t idx = 0; is >> idx; - args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); + args.push_back(cv::gapi::detail::deserialize_runarg::exec(is, idx)); } return args; } } // namespace detail +/** @} */ + } // namespace gapi } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/s11n/base.hpp b/modules/gapi/include/opencv2/gapi/s11n/base.hpp index d9335ee9f7..11440b27e5 100644 --- a/modules/gapi/include/opencv2/gapi/s11n/base.hpp +++ b/modules/gapi/include/opencv2/gapi/s11n/base.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #ifndef OPENCV_GAPI_S11N_BASE_HPP #define OPENCV_GAPI_S11N_BASE_HPP @@ -12,31 +12,65 @@ namespace cv { namespace gapi { + +/** + * @brief This namespace contains G-API serialization and + * deserialization functions and data structures. + */ namespace s11n { struct IOStream; struct IIStream; namespace detail { +//! @addtogroup gapi_serialization +//! @{ + struct NotImplemented { }; -// The default S11N for custom types is NotImplemented -// Don't! sublass from NotImplemented if you actually implement S11N. +/** @brief This structure allows to implement serialization routines for custom types. + * + * The default S11N for custom types is not implemented. + * + * @note When providing an overloaded implementation for S11N with your type + * don't inherit it from NotImplemented structure. + * + * @note There are lots of overloaded >> and << operators for basic and OpenCV/G-API types + * which can be utilized when serializing a custom type. + * + * Example of usage: + * @snippet modules/gapi/samples/api_ref_snippets.cpp S11N usage + * + */ template struct S11N: public NotImplemented { + /** + * @brief This function allows user to serialize their custom type. + * + * @note The default overload throws an exception if called. User need to + * properly overload the function to use it. + */ static void serialize(IOStream &, const T &) { GAPI_Assert(false && "No serialization routine is provided!"); } + /** + * @brief This function allows user to deserialize their custom type. + * + * @note The default overload throws an exception if called. User need to + * properly overload the function to use it. + */ static T deserialize(IIStream &) { GAPI_Assert(false && "No deserialization routine is provided!"); } }; +/// @private -- Exclude this struct from OpenCV documentation template struct has_S11N_spec { static constexpr bool value = !std::is_base_of::type>>::value; }; +//! @} gapi_serialization } // namespace detail } // namespace s11n diff --git a/modules/gapi/include/opencv2/gapi/stereo.hpp b/modules/gapi/include/opencv2/gapi/stereo.hpp index 278ae34950..dcf8f4d260 100644 --- a/modules/gapi/include/opencv2/gapi/stereo.hpp +++ b/modules/gapi/include/opencv2/gapi/stereo.hpp @@ -38,6 +38,11 @@ enum class StereoOutputFormat { DISPARITY_16Q_11_4 = DISPARITY_FIXED16_12_4 ///< Same as DISPARITY_FIXED16_12_4 }; + +/** + * @brief This namespace contains G-API Operation Types for Stereo and + * related functionality. + */ namespace calib3d { G_TYPED_KERNEL(GStereo, , "org.opencv.stereo") { diff --git a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp index 86de279fe9..1ed6e24b49 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #ifndef OPENCV_GAPI_GSTREAMING_DESYNC_HPP @@ -73,9 +73,10 @@ G desync(const G &g) { * which produces an array of cv::util::optional<> objects. * * @note This feature is highly experimental now and is currently - * limited to a single GMat argument only. + * limited to a single GMat/GFrame argument only. */ GAPI_EXPORTS GMat desync(const GMat &g); +GAPI_EXPORTS GFrame desync(const GFrame &f); } // namespace streaming } // namespace gapi diff --git a/modules/gapi/include/opencv2/gapi/streaming/format.hpp b/modules/gapi/include/opencv2/gapi/streaming/format.hpp index c9d2fa3e0a..f7c3bd457d 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/format.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/format.hpp @@ -74,7 +74,7 @@ e.g when graph's input needs to be passed directly to output, like in Streaming @param in Input image @return Copy of the input */ -GAPI_EXPORTS GMat copy(const GMat& in); +GAPI_EXPORTS_W GMat copy(const GMat& in); /** @brief Makes a copy of the input frame. Note that this copy may be not real (no actual data copied). Use this function to maintain graph contracts, diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp new file mode 100644 index 0000000000..9dc5ead7d7 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/cfg_params.hpp @@ -0,0 +1,88 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP + +#include +#include +#include + +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +/** + * @brief Public class is using for creation of onevpl::GSource instances. + * + * Class members availaible through methods @ref CfgParam::get_name() and @ref CfgParam::get_value() are used by + * onevpl::GSource inner logic to create or find oneVPL particular implementation + * (software/hardware, specific API version and etc.). + * + * @note Because oneVPL may provide several implementations which are satisfying with multiple (or single one) @ref CfgParam + * criteria therefore it is possible to configure `preferred` parameters. This kind of CfgParams are created + * using `is_major = false` argument in @ref CfgParam::create method and are not used by creating oneVPL particular implementations. + * Instead they fill out a "score table" to select preferrable implementation from available list. Implementation are satisfying + * with most of these optional params would be chosen. + * If no one optional CfgParam params were present then first of available oneVPL implementation would be applied. + * Please get on https://spec.oneapi.io/versions/latest/elements/oneVPL/source/API_ref/VPL_disp_api_func.html?highlight=mfxcreateconfig#mfxsetconfigfilterproperty + * for using OneVPL configuration. In this schema `mfxU8 *name` represents @ref CfgParam::get_name() and + * `mfxVariant value` is @ref CfgParam::get_value() + */ +struct GAPI_EXPORTS CfgParam { + using name_t = std::string; + using value_t = cv::util::variant; + + /** + * Create onevp::GSource configuration parameter. + * + *@param name name of parameter. + *@param value value of parameter. + *@param is_major TRUE if parameter MUST be provided by OneVPL inner implementation, FALSE for optional (for resolve multiple available implementations). + * + */ + template + static CfgParam create(const std::string& name, ValueType&& value, bool is_major = true) { + CfgParam param(name, CfgParam::value_t(std::forward(value)), is_major); + return param; + } + + struct Priv; + + const name_t& get_name() const; + const value_t& get_value() const; + bool is_major() const; + bool operator==(const CfgParam& rhs) const; + bool operator< (const CfgParam& rhs) const; + bool operator!=(const CfgParam& rhs) const; + + CfgParam& operator=(const CfgParam& src); + CfgParam& operator=(CfgParam&& src); + CfgParam(const CfgParam& src); + CfgParam(CfgParam&& src); + ~CfgParam(); +private: + CfgParam(const std::string& param_name, value_t&& param_value, bool is_major_param); + std::shared_ptr m_priv; +}; + +} //namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_CFG_PARAMS_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp new file mode 100644 index 0000000000..2c299520f7 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/data_provider_interface.hpp @@ -0,0 +1,74 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP +#include +#include + +#include // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct GAPI_EXPORTS DataProviderException : public std::exception { + virtual ~DataProviderException() {}; +}; + +struct GAPI_EXPORTS DataProviderSystemErrorException : public DataProviderException { + DataProviderSystemErrorException(int error_code, const std::string& desription = std::string()); + virtual ~DataProviderSystemErrorException(); + virtual const char* what() const noexcept override; + +private: + std::string reason; +}; + +/** + * @brief Public interface allows to customize extraction of video stream data + * used by onevpl::GSource instead of reading stream from file (by default). + * + * Interface implementation constructor MUST provide consistency and creates fully operable object. + * If error happened implementation MUST throw `DataProviderException` kind exceptions + * + * @note Interface implementation MUST manage stream and other constructed resources by itself to avoid any kind of leak. + * For simple interface implementation example please see `StreamDataProvider` in `tests/streaming/gapi_streaming_tests.cpp` + */ +struct GAPI_EXPORTS IDataProvider { + using Ptr = std::shared_ptr; + + virtual ~IDataProvider() {}; + + /** + * The function is used by onevpl::GSource to extract binary data stream from @ref IDataProvider + * implementation. + * + * It MUST throw `DataProviderException` kind exceptions in fail cases. + * It MUST return 0 in EOF which considered as not-fail case. + * + * @param out_data_bytes_size the available capacity of out_data buffer. + * @param out_data the output consumer buffer with capacity out_data_bytes_size. + * @return fetched bytes count. + */ + virtual size_t fetch_data(size_t out_data_bytes_size, void* out_data) = 0; + + /** + * The function is used by onevpl::GSource to check more binary data availability. + * + * It MUST return TRUE in case of EOF and NO_THROW exceptions. + * + * @return boolean value which detects end of stream + */ + virtual bool empty() const = 0; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_DATA_PROVIDER_INTERFACE_HPP diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp new file mode 100644 index 0000000000..b7bf4f2ffb --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/source.hpp @@ -0,0 +1,64 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP + +#include +#include +#include +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +using CfgParams = std::vector; + +/** + * @brief G-API streaming source based on OneVPL implementation. + * + * This class implements IStreamSource interface. + * Its constructor takes source file path (in usual way) or @ref onevpl::IDataProvider + * interface implementation (for not file-based sources). It also allows to pass-through + * oneVPL configuration parameters by using several @ref onevpl::CfgParam. + * + * @note stream sources are passed to G-API via shared pointers, so + * please gapi::make_onevpl_src<> to create objects and ptr() to pass a + * GSource to cv::gin(). + */ +class GAPI_EXPORTS GSource : public IStreamSource +{ +public: + struct Priv; + + GSource(const std::string& filePath, + const CfgParams& cfg_params = CfgParams{}); + GSource(std::shared_ptr source, + const CfgParams& cfg_params = CfgParams{}); + ~GSource() override; + + bool pull(cv::gapi::wip::Data& data) override; + GMetaArg descr_of() const override; + +private: + explicit GSource(std::unique_ptr&& impl); + std::unique_ptr m_priv; +}; +} // namespace onevpl + +template +GAPI_EXPORTS_W cv::Ptr inline make_onevpl_src(Args&&... args) +{ + return make_src(std::forward(args)...); +} + +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_HPP diff --git a/modules/gapi/include/opencv2/gapi/util/util.hpp b/modules/gapi/include/opencv2/gapi/util/util.hpp index afcf5596fd..eb435a3eef 100644 --- a/modules/gapi/include/opencv2/gapi/util/util.hpp +++ b/modules/gapi/include/opencv2/gapi/util/util.hpp @@ -117,6 +117,65 @@ namespace detail static type get(std::tuple&& objs) { return std::forward>(objs); } }; } // namespace detail + +namespace util +{ +template +struct overload_lamba_set; + +template +struct overload_lamba_set : public L1 +{ + overload_lamba_set(L1&& lambda) : L1(std::move(lambda)) {} + overload_lamba_set(const L1& lambda) : L1(lambda) {} + + using L1::operator(); +}; + +template +struct overload_lamba_set : public L1, public overload_lamba_set +{ + using base_type = overload_lamba_set; + overload_lamba_set(L1 &&lambda1, L&& ...lambdas): + L1(std::move(lambda1)), + base_type(std::forward(lambdas)...) {} + + overload_lamba_set(const L1 &lambda1, L&& ...lambdas): + L1(lambda1), + base_type(std::forward(lambdas)...) {} + + using L1::operator(); + using base_type::operator(); +}; + +template +overload_lamba_set overload_lambdas(L&& ...lambdas) +{ + return overload_lamba_set(std::forward(lambdas)...); +} + +template +struct find_adapter_impl; + +template +struct find_adapter_impl +{ + using type = typename std::conditional::value, + T, + void>::type; + static constexpr bool found = std::is_base_of::value; +}; + +template +struct find_adapter_impl +{ + using type = typename std::conditional::value, + T, + typename find_adapter_impl::type>::type; + static constexpr bool found = std::is_base_of::value || + find_adapter_impl::found; +}; +} // namespace util } // namespace cv // \endcond diff --git a/modules/gapi/include/opencv2/gapi/util/variant.hpp b/modules/gapi/include/opencv2/gapi/util/variant.hpp index 71a06d2dcf..f412110deb 100644 --- a/modules/gapi/include/opencv2/gapi/util/variant.hpp +++ b/modules/gapi/include/opencv2/gapi/util/variant.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include // max_of_t #include @@ -44,6 +45,12 @@ namespace util static const constexpr std::size_t value = detail::type_list_index_helper<0, Target, Types...>::value; }; + template + struct type_list_element + { + using type = typename std::tuple_element >::type; + }; + class bad_variant_access: public std::exception { public: @@ -233,9 +240,87 @@ namespace util template const T& get(const util::variant &v); + template + typename util::type_list_element::type& get(util::variant &v); + + template + const typename util::type_list_element::type& get(const util::variant &v); + template bool holds_alternative(const util::variant &v) noexcept; + + // Visitor + namespace detail + { + struct visitor_interface {}; + + // Class `visitor_return_type_deduction_helper` + // introduces solution for deduction `return_type` in `visit` function in common way + // for both Lambda and class Visitor and keep one interface invocation point: `visit` only + // his helper class is required to unify return_type deduction mechanism because + // for Lambda it is possible to take type of `decltype(visitor(get<0>(var)))` + // but for class Visitor there is no operator() in base case, + // because it provides `operator() (std::size_t index, ...)` + // So `visitor_return_type_deduction_helper` expose `operator()` + // uses only for class Visitor only for deduction `return type` in visit() + template + struct visitor_return_type_deduction_helper + { + using return_type = R; + + // to be used in Lambda return type deduction context only + template + return_type operator() (T&&); + }; + } + + // Special purpose `static_visitor` can receive additional arguments + template + struct static_visitor : public detail::visitor_interface, + public detail::visitor_return_type_deduction_helper { + + // assign responsibility for return type deduction to helper class + using return_type = typename detail::visitor_return_type_deduction_helper::return_type; + using detail::visitor_return_type_deduction_helper::operator(); + friend Impl; + + template + return_type operator() (std::size_t index, VariantValue&& value, Args&& ...args) + { + suppress_unused_warning(index); + return static_cast(this)-> visit( + std::forward(value), + std::forward(args)...); + } + }; + + // Special purpose `static_indexed_visitor` can receive additional arguments + // And make forwarding current variant index as runtime function argument to its `Impl` + template + struct static_indexed_visitor : public detail::visitor_interface, + public detail::visitor_return_type_deduction_helper { + + // assign responsibility for return type deduction to helper class + using return_type = typename detail::visitor_return_type_deduction_helper::return_type; + using detail::visitor_return_type_deduction_helper::operator(); + friend Impl; + + template + return_type operator() (std::size_t Index, VariantValue&& value, Args&& ...args) + { + return static_cast(this)-> visit(Index, + std::forward(value), + std::forward(args)...); + } + }; + + template + struct variant_size; + + template + struct variant_size> + : std::integral_constant { }; // FIXME: T&&, const TT&& versions. // Implementation ////////////////////////////////////////////////////////// @@ -402,6 +487,22 @@ namespace util throw_error(bad_variant_access()); } + template + typename util::type_list_element::type& get(util::variant &v) + { + using ReturnType = typename util::type_list_element::type; + return const_cast(get(static_cast &>(v))); + } + + template + const typename util::type_list_element::type& get(const util::variant &v) + { + static_assert(Index < sizeof...(Types), + "`Index` it out of bound of `util::variant` type list"); + using ReturnType = typename util::type_list_element::type; + return get(v); + } + template bool holds_alternative(const util::variant &v) noexcept { @@ -428,7 +529,130 @@ namespace util { return !(lhs == rhs); } -} // namespace cv + +namespace detail +{ + // terminate recursion implementation for `non-void` ReturnType + template + ReturnType apply_visitor_impl(Visitor&&, Variant&, + std::true_type, std::false_type, + VisitorArgs&& ...) + { + return {}; + } + + // terminate recursion implementation for `void` ReturnType + template + void apply_visitor_impl(Visitor&&, Variant&, + std::true_type, std::true_type, + VisitorArgs&& ...) + { + } + + // Intermediate resursion processor for Lambda Visitors + template + typename std::enable_if::type>::value, ReturnType>::type + apply_visitor_impl(Visitor&& visitor, Variant&& v, std::false_type not_processed, + std::integral_constant should_no_return, + VisitorArgs&& ...args) + { + static_assert(std::is_same(v)))>::value, + "Different `ReturnType`s detected! All `Visitor::visit` or `overload_lamba_set`" + " must return the same type"); + suppress_unused_warning(not_processed); + if (v.index() == CurIndex) + { + return visitor.operator()(get(v), std::forward(args)... ); + } + + using is_variant_processed_t = std::integral_constant= ElemCount>; + return apply_visitor_impl( + std::forward(visitor), + std::forward(v), + is_variant_processed_t{}, + should_no_return, + std::forward(args)...); + } + + //Visual Studio 2014 compilation fix: cast visitor to base class before invoke operator() + template + typename std::enable_if::type>, + typename std::decay::type>::value, ReturnType>::type + invoke_class_visitor(Visitor& visitor, Value&& v, VisitorArgs&&...args) + { + return static_cast::type>&>(visitor).operator() (CurIndex, std::forward(v), std::forward(args)... ); + } + + //Visual Studio 2014 compilation fix: cast visitor to base class before invoke operator() + template + typename std::enable_if::type>, + typename std::decay::type>::value, ReturnType>::type + invoke_class_visitor(Visitor& visitor, Value&& v, VisitorArgs&&...args) + { + return static_cast::type>&>(visitor).operator() (CurIndex, std::forward(v), std::forward(args)... ); + } + + // Intermediate recursion processor for special case `visitor_interface` derived Visitors + template + typename std::enable_if::type>::value, ReturnType>::type + apply_visitor_impl(Visitor&& visitor, Variant&& v, std::false_type not_processed, + std::integral_constant should_no_return, + VisitorArgs&& ...args) + { + static_assert(std::is_same(v)))>::value, + "Different `ReturnType`s detected! All `Visitor::visit` or `overload_lamba_set`" + " must return the same type"); + suppress_unused_warning(not_processed); + if (v.index() == CurIndex) + { + return invoke_class_visitor(visitor, get(v), std::forward(args)... ); + } + + using is_variant_processed_t = std::integral_constant= ElemCount>; + return apply_visitor_impl( + std::forward(visitor), + std::forward(v), + is_variant_processed_t{}, + should_no_return, + std::forward(args)...); + } +} // namespace detail + + template + auto visit(Visitor &visitor, const Variant& var, VisitorArg &&...args) -> decltype(visitor(get<0>(var))) + { + constexpr std::size_t varsize = util::variant_size::value; + static_assert(varsize != 0, "utils::variant must contains one type at least "); + using is_variant_processed_t = std::false_type; + + using ReturnType = decltype(visitor(get<0>(var))); + using return_t = std::is_same; + return detail::apply_visitor_impl( + std::forward(visitor), + var, is_variant_processed_t{}, + return_t{}, + std::forward(args)...); + } + + template + auto visit(Visitor&& visitor, const Variant& var) -> decltype(visitor(get<0>(var))) + { + constexpr std::size_t varsize = util::variant_size::value; + static_assert(varsize != 0, "utils::variant must contains one type at least "); + using is_variant_processed_t = std::false_type; + + using ReturnType = decltype(visitor(get<0>(var))); + using return_t = std::is_same; + return detail::apply_visitor_impl( + std::forward(visitor), + var, is_variant_processed_t{}, + return_t{}); + } } // namespace util +} // namespace cv #endif // OPENCV_GAPI_UTIL_VARIANT_HPP diff --git a/modules/gapi/include/opencv2/gapi/video.hpp b/modules/gapi/include/opencv2/gapi/video.hpp index 10965b0aa6..4dcc1d4182 100644 --- a/modules/gapi/include/opencv2/gapi/video.hpp +++ b/modules/gapi/include/opencv2/gapi/video.hpp @@ -42,6 +42,10 @@ struct GAPI_EXPORTS KalmanParams Mat controlMatrix; }; +/** + * @brief This namespace contains G-API Operations and functions for + * video-oriented algorithms, like optical flow and background subtraction. + */ namespace video { using GBuildPyrOutput = std::tuple, GScalar>; diff --git a/modules/gapi/misc/python/package/gapi/__init__.py b/modules/gapi/misc/python/package/gapi/__init__.py index 733c980010..b1326712fc 100644 --- a/modules/gapi/misc/python/package/gapi/__init__.py +++ b/modules/gapi/misc/python/package/gapi/__init__.py @@ -11,6 +11,36 @@ def register(mname): return parameterized +@register('cv2.gapi') +def networks(*args): + return cv.gapi_GNetPackage(list(map(cv.detail.strip, args))) + + +@register('cv2.gapi') +def compile_args(*args): + return list(map(cv.GCompileArg, args)) + + +@register('cv2') +def GIn(*args): + return [*args] + + +@register('cv2') +def GOut(*args): + return [*args] + + +@register('cv2') +def gin(*args): + return [*args] + + +@register('cv2.gapi') +def descr_of(*args): + return [*args] + + @register('cv2') class GOpaque(): # NB: Inheritance from c++ class cause segfault. @@ -54,6 +84,10 @@ class GOpaque(): def __new__(self): return cv.GOpaqueT(cv.gapi.CV_RECT) + class Prim(): + def __new__(self): + return cv.GOpaqueT(cv.gapi.CV_DRAW_PRIM) + class Any(): def __new__(self): return cv.GOpaqueT(cv.gapi.CV_ANY) @@ -113,6 +147,10 @@ class GArray(): def __new__(self): return cv.GArrayT(cv.gapi.CV_GMAT) + class Prim(): + def __new__(self): + return cv.GArray(cv.gapi.CV_DRAW_PRIM) + class Any(): def __new__(self): return cv.GArray(cv.gapi.CV_ANY) @@ -134,6 +172,7 @@ def op(op_id, in_types, out_types): cv.GArray.Scalar: cv.gapi.CV_SCALAR, cv.GArray.Mat: cv.gapi.CV_MAT, cv.GArray.GMat: cv.gapi.CV_GMAT, + cv.GArray.Prim: cv.gapi.CV_DRAW_PRIM, cv.GArray.Any: cv.gapi.CV_ANY } @@ -149,22 +188,24 @@ def op(op_id, in_types, out_types): cv.GOpaque.Point2f: cv.gapi.CV_POINT2F, cv.GOpaque.Size: cv.gapi.CV_SIZE, cv.GOpaque.Rect: cv.gapi.CV_RECT, + cv.GOpaque.Prim: cv.gapi.CV_DRAW_PRIM, cv.GOpaque.Any: cv.gapi.CV_ANY } type2str = { - cv.gapi.CV_BOOL: 'cv.gapi.CV_BOOL' , - cv.gapi.CV_INT: 'cv.gapi.CV_INT' , - cv.gapi.CV_DOUBLE: 'cv.gapi.CV_DOUBLE' , - cv.gapi.CV_FLOAT: 'cv.gapi.CV_FLOAT' , - cv.gapi.CV_STRING: 'cv.gapi.CV_STRING' , - cv.gapi.CV_POINT: 'cv.gapi.CV_POINT' , - cv.gapi.CV_POINT2F: 'cv.gapi.CV_POINT2F' , - cv.gapi.CV_SIZE: 'cv.gapi.CV_SIZE', - cv.gapi.CV_RECT: 'cv.gapi.CV_RECT', - cv.gapi.CV_SCALAR: 'cv.gapi.CV_SCALAR', - cv.gapi.CV_MAT: 'cv.gapi.CV_MAT', - cv.gapi.CV_GMAT: 'cv.gapi.CV_GMAT' + cv.gapi.CV_BOOL: 'cv.gapi.CV_BOOL' , + cv.gapi.CV_INT: 'cv.gapi.CV_INT' , + cv.gapi.CV_DOUBLE: 'cv.gapi.CV_DOUBLE' , + cv.gapi.CV_FLOAT: 'cv.gapi.CV_FLOAT' , + cv.gapi.CV_STRING: 'cv.gapi.CV_STRING' , + cv.gapi.CV_POINT: 'cv.gapi.CV_POINT' , + cv.gapi.CV_POINT2F: 'cv.gapi.CV_POINT2F' , + cv.gapi.CV_SIZE: 'cv.gapi.CV_SIZE', + cv.gapi.CV_RECT: 'cv.gapi.CV_RECT', + cv.gapi.CV_SCALAR: 'cv.gapi.CV_SCALAR', + cv.gapi.CV_MAT: 'cv.gapi.CV_MAT', + cv.gapi.CV_GMAT: 'cv.gapi.CV_GMAT', + cv.gapi.CV_DRAW_PRIM: 'cv.gapi.CV_DRAW_PRIM' } # NB: Second lvl decorator takes class to decorate @@ -244,3 +285,15 @@ def kernel(op_cls): return cls return kernel_with_params + + +# FIXME: On the c++ side every class is placed in cv2 module. +cv.gapi.wip.draw.Rect = cv.gapi_wip_draw_Rect +cv.gapi.wip.draw.Text = cv.gapi_wip_draw_Text +cv.gapi.wip.draw.Circle = cv.gapi_wip_draw_Circle +cv.gapi.wip.draw.Line = cv.gapi_wip_draw_Line +cv.gapi.wip.draw.Mosaic = cv.gapi_wip_draw_Mosaic +cv.gapi.wip.draw.Image = cv.gapi_wip_draw_Image +cv.gapi.wip.draw.Poly = cv.gapi_wip_draw_Poly + +cv.gapi.streaming.queue_capacity = cv.gapi_streaming_queue_capacity diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 6b782cfc8d..64f8277740 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -11,12 +11,14 @@ #include // NB: Python wrapper replaces :: with _ for classes -using gapi_GKernelPackage = cv::gapi::GKernelPackage; -using gapi_GNetPackage = cv::gapi::GNetPackage; -using gapi_ie_PyParams = cv::gapi::ie::PyParams; -using gapi_wip_IStreamSource_Ptr = cv::Ptr; -using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; -using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; +using gapi_GKernelPackage = cv::gapi::GKernelPackage; +using gapi_GNetPackage = cv::gapi::GNetPackage; +using gapi_ie_PyParams = cv::gapi::ie::PyParams; +using gapi_wip_IStreamSource_Ptr = cv::Ptr; +using detail_ExtractArgsCallback = cv::detail::ExtractArgsCallback; +using detail_ExtractMetaCallback = cv::detail::ExtractMetaCallback; +using vector_GNetParam = std::vector; +using gapi_streaming_queue_capacity = cv::gapi::streaming::queue_capacity; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs @@ -42,6 +44,7 @@ using GArray_Rect = cv::GArray; using GArray_Scalar = cv::GArray; using GArray_Mat = cv::GArray; using GArray_GMat = cv::GArray; +using GArray_Prim = cv::GArray; // FIXME: Python wrapper generate code without namespace std, // so it cause error: "string wasn't declared" @@ -124,6 +127,95 @@ PyObject* pyopencv_from(const cv::detail::PyObjectHolder& v) return o; } +// #FIXME: Is it possible to implement pyopencv_from/pyopencv_to for generic +// cv::variant ? +template <> +PyObject* pyopencv_from(const cv::gapi::wip::draw::Prim& prim) +{ + switch (prim.index()) + { + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + case cv::gapi::wip::draw::Prim::index_of(): + return pyopencv_from(cv::util::get(prim)); + } + + util::throw_error(std::logic_error("Unsupported draw primitive type")); +} + +template <> +PyObject* pyopencv_from(const cv::gapi::wip::draw::Prims& value) +{ + return pyopencv_from_generic_vec(value); +} + +template<> +bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prim& value, const ArgInfo&) +{ +#define TRY_EXTRACT(Prim) \ + if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_gapi_wip_draw_##Prim##_TypePtr))) \ + { \ + value = reinterpret_cast(obj)->v; \ + return true; \ + } \ + + TRY_EXTRACT(Rect) + TRY_EXTRACT(Text) + TRY_EXTRACT(Circle) + TRY_EXTRACT(Line) + TRY_EXTRACT(Mosaic) + TRY_EXTRACT(Image) + TRY_EXTRACT(Poly) +#undef TRY_EXTRACT + + failmsg("Unsupported primitive type"); + return false; +} + +template <> +bool pyopencv_to(PyObject* obj, cv::gapi::wip::draw::Prims& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + +template <> +bool pyopencv_to(PyObject* obj, cv::GMetaArg& value, const ArgInfo&) +{ +#define TRY_EXTRACT(Meta) \ + if (PyObject_TypeCheck(obj, \ + reinterpret_cast(pyopencv_##Meta##_TypePtr))) \ + { \ + value = reinterpret_cast(obj)->v; \ + return true; \ + } \ + + TRY_EXTRACT(GMatDesc) + TRY_EXTRACT(GScalarDesc) + TRY_EXTRACT(GArrayDesc) + TRY_EXTRACT(GOpaqueDesc) +#undef TRY_EXTRACT + + failmsg("Unsupported cv::GMetaArg type"); + return false; +} + +template <> +bool pyopencv_to(PyObject* obj, cv::GMetaArgs& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + + template<> PyObject* pyopencv_from(const cv::GArg& value) { @@ -136,20 +228,21 @@ PyObject* pyopencv_from(const cv::GArg& value) #define UNSUPPORTED(T) case cv::detail::OpaqueKind::CV_##T: break switch (value.opaque_kind) { - HANDLE_CASE(BOOL, bool); - HANDLE_CASE(INT, int); - HANDLE_CASE(DOUBLE, double); - HANDLE_CASE(FLOAT, float); - HANDLE_CASE(STRING, std::string); - HANDLE_CASE(POINT, cv::Point); - HANDLE_CASE(POINT2F, cv::Point2f); - HANDLE_CASE(SIZE, cv::Size); - HANDLE_CASE(RECT, cv::Rect); - HANDLE_CASE(SCALAR, cv::Scalar); - HANDLE_CASE(MAT, cv::Mat); - HANDLE_CASE(UNKNOWN, cv::detail::PyObjectHolder); + HANDLE_CASE(BOOL, bool); + HANDLE_CASE(INT, int); + HANDLE_CASE(INT64, int64_t); + HANDLE_CASE(DOUBLE, double); + HANDLE_CASE(FLOAT, float); + HANDLE_CASE(STRING, std::string); + HANDLE_CASE(POINT, cv::Point); + HANDLE_CASE(POINT2F, cv::Point2f); + HANDLE_CASE(SIZE, cv::Size); + HANDLE_CASE(RECT, cv::Rect); + HANDLE_CASE(SCALAR, cv::Scalar); + HANDLE_CASE(MAT, cv::Mat); + HANDLE_CASE(UNKNOWN, cv::detail::PyObjectHolder); + HANDLE_CASE(DRAW_PRIM, cv::gapi::wip::draw::Prim); UNSUPPORTED(UINT64); - UNSUPPORTED(DRAW_PRIM); #undef HANDLE_CASE #undef UNSUPPORTED } @@ -163,6 +256,18 @@ bool pyopencv_to(PyObject* obj, cv::GArg& value, const ArgInfo& info) return true; } +template <> +bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + return pyopencv_to_generic_vec(obj, value, info); +} + +template <> +PyObject* pyopencv_from(const std::vector& value) +{ + return pyopencv_from_generic_vec(value); +} + template <> bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) { @@ -175,12 +280,6 @@ PyObject* pyopencv_from(const std::vector& value) return pyopencv_from_generic_vec(value); } -template <> -bool pyopencv_to(PyObject* obj, GRunArgs& value, const ArgInfo& info) -{ - return pyopencv_to_generic_vec(obj, value, info); -} - template<> PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) { @@ -188,6 +287,7 @@ PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) { case cv::detail::OpaqueKind::CV_BOOL : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_INT : return pyopencv_from(o.rref()); + case cv::detail::OpaqueKind::CV_INT64 : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_DOUBLE : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_FLOAT : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_STRING : return pyopencv_from(o.rref()); @@ -196,10 +296,10 @@ PyObject* pyopencv_from(const cv::detail::OpaqueRef& o) case cv::detail::OpaqueKind::CV_SIZE : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_RECT : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_UNKNOWN : return pyopencv_from(o.rref()); + case cv::detail::OpaqueKind::CV_DRAW_PRIM : return pyopencv_from(o.rref()); case cv::detail::OpaqueKind::CV_UINT64 : break; case cv::detail::OpaqueKind::CV_SCALAR : break; case cv::detail::OpaqueKind::CV_MAT : break; - case cv::detail::OpaqueKind::CV_DRAW_PRIM : break; } PyErr_SetString(PyExc_TypeError, "Unsupported GOpaque type"); @@ -213,6 +313,7 @@ PyObject* pyopencv_from(const cv::detail::VectorRef& v) { case cv::detail::OpaqueKind::CV_BOOL : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_INT : return pyopencv_from_generic_vec(v.rref()); + case cv::detail::OpaqueKind::CV_INT64 : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_DOUBLE : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_FLOAT : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_STRING : return pyopencv_from_generic_vec(v.rref()); @@ -223,8 +324,8 @@ PyObject* pyopencv_from(const cv::detail::VectorRef& v) case cv::detail::OpaqueKind::CV_SCALAR : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_MAT : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_UNKNOWN : return pyopencv_from_generic_vec(v.rref()); + case cv::detail::OpaqueKind::CV_DRAW_PRIM : return pyopencv_from_generic_vec(v.rref()); case cv::detail::OpaqueKind::CV_UINT64 : break; - case cv::detail::OpaqueKind::CV_DRAW_PRIM : break; } PyErr_SetString(PyExc_TypeError, "Unsupported GArray type"); @@ -249,52 +350,69 @@ PyObject* pyopencv_from(const GRunArg& v) return pyopencv_from(util::get(v)); } - PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs"); + PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs. Index of variant is unknown"); + return NULL; +} + +template +PyObject* pyopencv_from(const cv::optional& opt) +{ + if (!opt.has_value()) + { + Py_RETURN_NONE; + } + return pyopencv_from(*opt); +} + +template <> +PyObject* pyopencv_from(const GOptRunArg& v) +{ + switch (v.index()) + { + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + + case GOptRunArg::index_of>(): + return pyopencv_from(util::get>(v)); + } + + PyErr_SetString(PyExc_TypeError, "Failed to unpack GOptRunArg. Index of variant is unknown"); return NULL; } template<> PyObject* pyopencv_from(const GRunArgs& value) { - size_t i, n = value.size(); - - // NB: It doesn't make sense to return list with a single element - if (n == 1) - { - PyObject* item = pyopencv_from(value[0]); - if(!item) - { - return NULL; - } - return item; - } - - PyObject* list = PyList_New(n); - for(i = 0; i < n; ++i) - { - PyObject* item = pyopencv_from(value[i]); - if(!item) - { - Py_DECREF(list); - PyErr_SetString(PyExc_TypeError, "Failed to unpack GRunArgs"); - return NULL; - } - PyList_SetItem(list, i, item); - } - - return list; + return value.size() == 1 ? pyopencv_from(value[0]) : pyopencv_from_generic_vec(value); } template<> -bool pyopencv_to(PyObject* obj, GMetaArgs& value, const ArgInfo& info) +PyObject* pyopencv_from(const GOptRunArgs& value) { - return pyopencv_to_generic_vec(obj, value, info); + return value.size() == 1 ? pyopencv_from(value[0]) : pyopencv_from_generic_vec(value); } -template<> -PyObject* pyopencv_from(const GMetaArgs& value) +// FIXME: cv::variant should be wrapped once for all types. +template <> +PyObject* pyopencv_from(const cv::util::variant& v) { - return pyopencv_from_generic_vec(value); + using RunArgs = cv::util::variant; + switch (v.index()) + { + case RunArgs::index_of(): + return pyopencv_from(util::get(v)); + case RunArgs::index_of(): + return pyopencv_from(util::get(v)); + } + + PyErr_SetString(PyExc_TypeError, "Failed to recognize kind of RunArgs. Index of variant is unknown"); + return NULL; } template @@ -318,16 +436,16 @@ void pyopencv_to_generic_vec_with_check(PyObject* from, } template -static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw) +static T extract_proto_args(PyObject* py_args) { using namespace cv; GProtoArgs args; - Py_ssize_t size = PyTuple_Size(py_args); + Py_ssize_t size = PyList_Size(py_args); args.reserve(size); for (int i = 0; i < size; ++i) { - PyObject* item = PyTuple_GetItem(py_args, i); + PyObject* item = PyList_GetItem(py_args, i); if (PyObject_TypeCheck(item, reinterpret_cast(pyopencv_GScalar_TypePtr))) { args.emplace_back(reinterpret_cast(item)->v); @@ -346,22 +464,11 @@ static PyObject* extract_proto_args(PyObject* py_args, PyObject* kw) } else { - PyErr_SetString(PyExc_TypeError, "Unsupported type for cv.GIn()/cv.GOut()"); - return NULL; + util::throw_error(std::logic_error("Unsupported type for GProtoArgs")); } } - return pyopencv_from(T{std::move(args)}); -} - -static PyObject* pyopencv_cv_GIn(PyObject* , PyObject* py_args, PyObject* kw) -{ - return extract_proto_args(py_args, kw); -} - -static PyObject* pyopencv_cv_GOut(PyObject* , PyObject* py_args, PyObject* kw) -{ - return extract_proto_args(py_args, kw); + return T(std::move(args)); } static cv::detail::OpaqueRef extract_opaque_ref(PyObject* from, cv::detail::OpaqueKind kind) @@ -386,6 +493,7 @@ static cv::detail::OpaqueRef extract_opaque_ref(PyObject* from, cv::detail::Opaq HANDLE_CASE(RECT, cv::Rect); HANDLE_CASE(UNKNOWN, cv::GArg); UNSUPPORTED(UINT64); + UNSUPPORTED(INT64); UNSUPPORTED(SCALAR); UNSUPPORTED(MAT); UNSUPPORTED(DRAW_PRIM); @@ -406,20 +514,21 @@ static cv::detail::VectorRef extract_vector_ref(PyObject* from, cv::detail::Opaq #define UNSUPPORTED(T) case cv::detail::OpaqueKind::CV_##T: break switch (kind) { - HANDLE_CASE(BOOL, bool); - HANDLE_CASE(INT, int); - HANDLE_CASE(DOUBLE, double); - HANDLE_CASE(FLOAT, float); - HANDLE_CASE(STRING, std::string); - HANDLE_CASE(POINT, cv::Point); - HANDLE_CASE(POINT2F, cv::Point2f); - HANDLE_CASE(SIZE, cv::Size); - HANDLE_CASE(RECT, cv::Rect); - HANDLE_CASE(SCALAR, cv::Scalar); - HANDLE_CASE(MAT, cv::Mat); - HANDLE_CASE(UNKNOWN, cv::GArg); + HANDLE_CASE(BOOL, bool); + HANDLE_CASE(INT, int); + HANDLE_CASE(DOUBLE, double); + HANDLE_CASE(FLOAT, float); + HANDLE_CASE(STRING, std::string); + HANDLE_CASE(POINT, cv::Point); + HANDLE_CASE(POINT2F, cv::Point2f); + HANDLE_CASE(SIZE, cv::Size); + HANDLE_CASE(RECT, cv::Rect); + HANDLE_CASE(SCALAR, cv::Scalar); + HANDLE_CASE(MAT, cv::Mat); + HANDLE_CASE(UNKNOWN, cv::GArg); + HANDLE_CASE(DRAW_PRIM, cv::gapi::wip::draw::Prim); UNSUPPORTED(UINT64); - UNSUPPORTED(DRAW_PRIM); + UNSUPPORTED(INT64); #undef HANDLE_CASE #undef UNSUPPORTED } @@ -470,13 +579,15 @@ static cv::GRunArg extract_run_arg(const cv::GTypeInfo& info, PyObject* item) static cv::GRunArgs extract_run_args(const cv::GTypesInfo& info, PyObject* py_args) { - cv::GRunArgs args; - Py_ssize_t tuple_size = PyTuple_Size(py_args); - args.reserve(tuple_size); + GAPI_Assert(PyList_Check(py_args)); - for (int i = 0; i < tuple_size; ++i) + cv::GRunArgs args; + Py_ssize_t list_size = PyList_Size(py_args); + args.reserve(list_size); + + for (int i = 0; i < list_size; ++i) { - args.push_back(extract_run_arg(info[i], PyTuple_GetItem(py_args, i))); + args.push_back(extract_run_arg(info[i], PyList_GetItem(py_args, i))); } return args; @@ -517,13 +628,15 @@ static cv::GMetaArg extract_meta_arg(const cv::GTypeInfo& info, PyObject* item) static cv::GMetaArgs extract_meta_args(const cv::GTypesInfo& info, PyObject* py_args) { - cv::GMetaArgs metas; - Py_ssize_t tuple_size = PyTuple_Size(py_args); - metas.reserve(tuple_size); + GAPI_Assert(PyList_Check(py_args)); - for (int i = 0; i < tuple_size; ++i) + cv::GMetaArgs metas; + Py_ssize_t list_size = PyList_Size(py_args); + metas.reserve(list_size); + + for (int i = 0; i < list_size; ++i) { - metas.push_back(extract_meta_arg(info[i], PyTuple_GetItem(py_args, i))); + metas.push_back(extract_meta_arg(info[i], PyList_GetItem(py_args, i))); } return metas; @@ -581,7 +694,8 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, cv::detail::PyObjectHolder result( PyObject_CallObject(kernel.get(), args.get()), false); - if (PyErr_Occurred()) { + if (PyErr_Occurred()) + { PyErr_PrintEx(0); PyErr_Clear(); throw std::logic_error("Python kernel failed with error!"); @@ -589,8 +703,27 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, // NB: In fact it's impossible situation, becase errors were handled above. GAPI_Assert(result.get() && "Python kernel returned NULL!"); - outs = out_info.size() == 1 ? cv::GRunArgs{extract_run_arg(out_info[0], result.get())} - : extract_run_args(out_info, result.get()); + if (out_info.size() == 1) + { + outs = cv::GRunArgs{extract_run_arg(out_info[0], result.get())}; + } + else if (out_info.size() > 1) + { + GAPI_Assert(PyTuple_Check(result.get())); + + Py_ssize_t tuple_size = PyTuple_Size(result.get()); + outs.reserve(tuple_size); + + for (int i = 0; i < tuple_size; ++i) + { + outs.push_back(extract_run_arg(out_info[i], PyTuple_GetItem(result.get(), i))); + } + } + else + { + // Seems to be impossible case. + GAPI_Assert(false); + } } catch (...) { @@ -604,30 +737,12 @@ static cv::GRunArgs run_py_kernel(cv::detail::PyObjectHolder kernel, static GMetaArg get_meta_arg(PyObject* obj) { - if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GMatDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GScalarDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GArrayDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else if (PyObject_TypeCheck(obj, - reinterpret_cast(pyopencv_GOpaqueDesc_TypePtr))) - { - return cv::GMetaArg{reinterpret_cast(obj)->v}; - } - else + cv::GMetaArg arg; + if (!pyopencv_to(obj, arg, ArgInfo("arg", false))) { util::throw_error(std::logic_error("Unsupported output meta type")); } + return arg; } static cv::GMetaArgs get_meta_args(PyObject* tuple) @@ -645,8 +760,9 @@ static cv::GMetaArgs get_meta_args(PyObject* tuple) } static GMetaArgs run_py_meta(cv::detail::PyObjectHolder out_meta, - const cv::GMetaArgs &meta, - const cv::GArgs &gargs) { + const cv::GMetaArgs &meta, + const cv::GArgs &gargs) +{ PyGILState_STATE gstate; gstate = PyGILState_Ensure(); @@ -688,7 +804,8 @@ static GMetaArgs run_py_meta(cv::detail::PyObjectHolder out_meta, cv::detail::PyObjectHolder result( PyObject_CallObject(out_meta.get(), args.get()), false); - if (PyErr_Occurred()) { + if (PyErr_Occurred()) + { PyErr_PrintEx(0); PyErr_Clear(); throw std::logic_error("Python outMeta failed with error!"); @@ -720,21 +837,24 @@ static PyObject* pyopencv_cv_gapi_kernels(PyObject* , PyObject* py_args, PyObjec PyObject* user_kernel = PyTuple_GetItem(py_args, i); PyObject* id_obj = PyObject_GetAttrString(user_kernel, "id"); - if (!id_obj) { + if (!id_obj) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain id, please use cv.gapi.kernel to define kernel"); return NULL; } PyObject* out_meta = PyObject_GetAttrString(user_kernel, "outMeta"); - if (!out_meta) { + if (!out_meta) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain outMeta, please use cv.gapi.kernel to define kernel"); return NULL; } PyObject* run = PyObject_GetAttrString(user_kernel, "run"); - if (!run) { + if (!run) + { PyErr_SetString(PyExc_TypeError, "Python kernel should contain run, please use cv.gapi.kernel to define kernel"); return NULL; @@ -756,23 +876,6 @@ static PyObject* pyopencv_cv_gapi_kernels(PyObject* , PyObject* py_args, PyObjec return pyopencv_from(pkg); } -static PyObject* pyopencv_cv_gapi_networks(PyObject*, PyObject* py_args, PyObject*) -{ - using namespace cv; - gapi::GNetPackage pkg; - Py_ssize_t size = PyTuple_Size(py_args); - for (int i = 0; i < size; ++i) - { - gapi_ie_PyParams params; - PyObject* item = PyTuple_GetItem(py_args, i); - if (pyopencv_to(item, params, ArgInfo("PyParams", false))) - { - pkg += gapi::networks(params); - } - } - return pyopencv_from(pkg); -} - static PyObject* pyopencv_cv_gapi_op(PyObject* , PyObject* py_args, PyObject*) { using namespace cv; @@ -834,53 +937,54 @@ static PyObject* pyopencv_cv_gapi_op(PyObject* , PyObject* py_args, PyObject*) return pyopencv_from(cv::gapi::wip::op(id, outMetaWrapper, std::move(args))); } -static PyObject* pyopencv_cv_gin(PyObject*, PyObject* py_args, PyObject*) +template<> +bool pyopencv_to(PyObject* obj, cv::detail::ExtractArgsCallback& value, const ArgInfo&) { - cv::detail::PyObjectHolder holder{py_args}; - auto callback = cv::detail::ExtractArgsCallback{[=](const cv::GTypesInfo& info) + cv::detail::PyObjectHolder holder{obj}; + value = cv::detail::ExtractArgsCallback{[=](const cv::GTypesInfo& info) + { + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + cv::GRunArgs args; + try + { + args = extract_run_args(info, holder.get()); + } + catch (...) { - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - cv::GRunArgs args; - try - { - args = extract_run_args(info, holder.get()); - } - catch (...) - { - PyGILState_Release(gstate); - throw; - } PyGILState_Release(gstate); - return args; - }}; - - return pyopencv_from(callback); + throw; + } + PyGILState_Release(gstate); + return args; + }}; + return true; } -static PyObject* pyopencv_cv_descr_of(PyObject*, PyObject* py_args, PyObject*) +template<> +bool pyopencv_to(PyObject* obj, cv::detail::ExtractMetaCallback& value, const ArgInfo&) { - Py_INCREF(py_args); - auto callback = cv::detail::ExtractMetaCallback{[=](const cv::GTypesInfo& info) - { - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); + cv::detail::PyObjectHolder holder{obj}; + value = cv::detail::ExtractMetaCallback{[=](const cv::GTypesInfo& info) + { + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); - cv::GMetaArgs args; - try - { - args = extract_meta_args(info, py_args); - } - catch (...) - { - PyGILState_Release(gstate); - throw; - } + cv::GMetaArgs args; + try + { + args = extract_meta_args(info, holder.get()); + } + catch (...) + { PyGILState_Release(gstate); - return args; - }}; - return pyopencv_from(callback); + throw; + } + PyGILState_Release(gstate); + return args; + }}; + return true; } template @@ -895,9 +999,12 @@ struct PyOpenCV_Converter> if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_GArrayT_TypePtr))) { auto& array = reinterpret_cast(obj)->v; - try { + try + { value = cv::util::get>(array.arg()); - } catch (...) { + } + catch (...) + { return false; } return true; @@ -918,9 +1025,12 @@ struct PyOpenCV_Converter> if (PyObject_TypeCheck(obj, reinterpret_cast(pyopencv_GOpaqueT_TypePtr))) { auto& opaque = reinterpret_cast(obj)->v; - try { + try + { value = cv::util::get>(opaque.arg()); - } catch (...) { + } + catch (...) + { return false; } return true; @@ -929,11 +1039,39 @@ struct PyOpenCV_Converter> } }; +template<> +bool pyopencv_to(PyObject* obj, cv::GProtoInputArgs& value, const ArgInfo& info) +{ + try + { + value = extract_proto_args(obj); + return true; + } + catch (...) + { + failmsg("Can't parse cv::GProtoInputArgs"); + return false; + } +} + +template<> +bool pyopencv_to(PyObject* obj, cv::GProtoOutputArgs& value, const ArgInfo& info) +{ + try + { + value = extract_proto_args(obj); + return true; + } + catch (...) + { + failmsg("Can't parse cv::GProtoOutputArgs"); + return false; + } +} // extend cv.gapi methods #define PYOPENCV_EXTRA_METHODS_GAPI \ {"kernels", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_kernels), "kernels(...) -> GKernelPackage"}, \ - {"networks", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_networks), "networks(...) -> GNetPackage"}, \ {"__op", CV_PY_FN_WITH_KW(pyopencv_cv_gapi_op), "__op(...) -> retval\n"}, diff --git a/modules/gapi/misc/python/python_bridge.hpp b/modules/gapi/misc/python/python_bridge.hpp index 0d1c6d51c5..11d1728730 100644 --- a/modules/gapi/misc/python/python_bridge.hpp +++ b/modules/gapi/misc/python/python_bridge.hpp @@ -10,6 +10,7 @@ #include #include #include +#include // Prim #define ID(T, E) T #define ID_(T, E) ID(T, E), @@ -24,24 +25,29 @@ GAPI_Assert(false && "Unsupported type"); \ } +using cv::gapi::wip::draw::Prim; + #define GARRAY_TYPE_LIST_G(G, G2) \ -WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ -WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ -WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ -WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ -WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ -WRAP_ARGS(cv::Point , cv::gapi::ArgType::CV_POINT, G) \ -WRAP_ARGS(cv::Point2f , cv::gapi::ArgType::CV_POINT2F, G) \ -WRAP_ARGS(cv::Size , cv::gapi::ArgType::CV_SIZE, G) \ -WRAP_ARGS(cv::Rect , cv::gapi::ArgType::CV_RECT, G) \ -WRAP_ARGS(cv::Scalar , cv::gapi::ArgType::CV_SCALAR, G) \ -WRAP_ARGS(cv::Mat , cv::gapi::ArgType::CV_MAT, G) \ -WRAP_ARGS(cv::GArg , cv::gapi::ArgType::CV_ANY, G) \ -WRAP_ARGS(cv::GMat , cv::gapi::ArgType::CV_GMAT, G2) \ +WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ +WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ +WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ +WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ +WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ +WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ +WRAP_ARGS(cv::Point , cv::gapi::ArgType::CV_POINT, G) \ +WRAP_ARGS(cv::Point2f , cv::gapi::ArgType::CV_POINT2F, G) \ +WRAP_ARGS(cv::Size , cv::gapi::ArgType::CV_SIZE, G) \ +WRAP_ARGS(cv::Rect , cv::gapi::ArgType::CV_RECT, G) \ +WRAP_ARGS(cv::Scalar , cv::gapi::ArgType::CV_SCALAR, G) \ +WRAP_ARGS(cv::Mat , cv::gapi::ArgType::CV_MAT, G) \ +WRAP_ARGS(Prim , cv::gapi::ArgType::CV_DRAW_PRIM, G) \ +WRAP_ARGS(cv::GArg , cv::gapi::ArgType::CV_ANY, G) \ +WRAP_ARGS(cv::GMat , cv::gapi::ArgType::CV_GMAT, G2) \ #define GOPAQUE_TYPE_LIST_G(G, G2) \ WRAP_ARGS(bool , cv::gapi::ArgType::CV_BOOL, G) \ WRAP_ARGS(int , cv::gapi::ArgType::CV_INT, G) \ +WRAP_ARGS(int64_t , cv::gapi::ArgType::CV_INT64, G) \ WRAP_ARGS(double , cv::gapi::ArgType::CV_DOUBLE, G) \ WRAP_ARGS(float , cv::gapi::ArgType::CV_FLOAT, G) \ WRAP_ARGS(std::string , cv::gapi::ArgType::CV_STRING, G) \ @@ -58,6 +64,7 @@ namespace gapi { enum ArgType { CV_BOOL, CV_INT, + CV_INT64, CV_DOUBLE, CV_FLOAT, CV_STRING, @@ -68,6 +75,7 @@ enum ArgType { CV_SCALAR, CV_MAT, CV_GMAT, + CV_DRAW_PRIM, CV_ANY, }; diff --git a/modules/gapi/misc/python/samples/gaze_estimation.py b/modules/gapi/misc/python/samples/gaze_estimation.py new file mode 100644 index 0000000000..5536787e60 --- /dev/null +++ b/modules/gapi/misc/python/samples/gaze_estimation.py @@ -0,0 +1,458 @@ +import argparse +import time +import numpy as np +import cv2 as cv + + +# ------------------------Service operations------------------------ +def weight_path(model_path): + """ Get path of weights based on path to IR + + Params: + model_path: the string contains path to IR file + + Return: + Path to weights file + """ + assert model_path.endswith('.xml'), "Wrong topology path was provided" + return model_path[:-3] + 'bin' + + +def build_argparser(): + """ Parse arguments from command line + + Return: + Pack of arguments from command line + """ + parser = argparse.ArgumentParser(description='This is an OpenCV-based version of Gaze Estimation example') + + parser.add_argument('--input', + help='Path to the input video file') + parser.add_argument('--out', + help='Path to the output video file') + parser.add_argument('--facem', + default='face-detection-retail-0005.xml', + help='Path to OpenVINO face detection model (.xml)') + parser.add_argument('--faced', + default='CPU', + help='Target device for the face detection' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--headm', + default='head-pose-estimation-adas-0001.xml', + help='Path to OpenVINO head pose estimation model (.xml)') + parser.add_argument('--headd', + default='CPU', + help='Target device for the head pose estimation inference ' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--landm', + default='facial-landmarks-35-adas-0002.xml', + help='Path to OpenVINO landmarks detector model (.xml)') + parser.add_argument('--landd', + default='CPU', + help='Target device for the landmarks detector (e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--gazem', + default='gaze-estimation-adas-0002.xml', + help='Path to OpenVINO gaze vector estimaiton model (.xml)') + parser.add_argument('--gazed', + default='CPU', + help='Target device for the gaze vector estimation inference ' + + '(e.g. CPU, GPU, VPU, ...)') + parser.add_argument('--eyem', + default='open-closed-eye-0001.xml', + help='Path to OpenVINO open closed eye model (.xml)') + parser.add_argument('--eyed', + default='CPU', + help='Target device for the eyes state inference (e.g. CPU, GPU, VPU, ...)') + return parser + + +# ------------------------Support functions for custom kernels------------------------ +def intersection(surface, rect): + """ Remove zone of out of bound from ROI + + Params: + surface: image bounds is rect representation (top left coordinates and width and height) + rect: region of interest is also has rect representation + + Return: + Modified ROI with correct bounds + """ + l_x = max(surface[0], rect[0]) + l_y = max(surface[1], rect[1]) + width = min(surface[0] + surface[2], rect[0] + rect[2]) - l_x + height = min(surface[1] + surface[3], rect[1] + rect[3]) - l_y + if width < 0 or height < 0: + return (0, 0, 0, 0) + return (l_x, l_y, width, height) + + +def process_landmarks(r_x, r_y, r_w, r_h, landmarks): + """ Create points from result of inference of facial-landmarks network and size of input image + + Params: + r_x: x coordinate of top left corner of input image + r_y: y coordinate of top left corner of input image + r_w: width of input image + r_h: height of input image + landmarks: result of inference of facial-landmarks network + + Return: + Array of landmarks points for one face + """ + lmrks = landmarks[0] + raw_x = lmrks[::2] * r_w + r_x + raw_y = lmrks[1::2] * r_h + r_y + return np.array([[int(x), int(y)] for x, y in zip(raw_x, raw_y)]) + + +def eye_box(p_1, p_2, scale=1.8): + """ Get bounding box of eye + + Params: + p_1: point of left edge of eye + p_2: point of right edge of eye + scale: change size of box with this value + + Return: + Bounding box of eye and its midpoint + """ + + size = np.linalg.norm(p_1 - p_2) + midpoint = (p_1 + p_2) / 2 + width = scale * size + height = width + p_x = midpoint[0] - (width / 2) + p_y = midpoint[1] - (height / 2) + return (int(p_x), int(p_y), int(width), int(height)), list(map(int, midpoint)) + + +# ------------------------Custom graph operations------------------------ +@cv.gapi.op('custom.GProcessPoses', + in_types=[cv.GArray.GMat, cv.GArray.GMat, cv.GArray.GMat], + out_types=[cv.GArray.GMat]) +class GProcessPoses: + @staticmethod + def outMeta(arr_desc0, arr_desc1, arr_desc2): + return cv.empty_array_desc() + + +@cv.gapi.op('custom.GParseEyes', + in_types=[cv.GArray.GMat, cv.GArray.Rect, cv.GOpaque.Size], + out_types=[cv.GArray.Rect, cv.GArray.Rect, cv.GArray.Point, cv.GArray.Point]) +class GParseEyes: + @staticmethod + def outMeta(arr_desc0, arr_desc1, arr_desc2): + return cv.empty_array_desc(), cv.empty_array_desc(), \ + cv.empty_array_desc(), cv.empty_array_desc() + + +@cv.gapi.op('custom.GGetStates', + in_types=[cv.GArray.GMat, cv.GArray.GMat], + out_types=[cv.GArray.Int, cv.GArray.Int]) +class GGetStates: + @staticmethod + def outMeta(arr_desc0, arr_desc1): + return cv.empty_array_desc(), cv.empty_array_desc() + + +# ------------------------Custom kernels------------------------ +@cv.gapi.kernel(GProcessPoses) +class GProcessPosesImpl: + """ Custom kernel. Processed poses of heads + """ + @staticmethod + def run(in_ys, in_ps, in_rs): + """ Сustom kernel executable code + + Params: + in_ys: yaw angle of head + in_ps: pitch angle of head + in_rs: roll angle of head + + Return: + Arrays with heads poses + """ + return [np.array([ys[0], ps[0], rs[0]]).T for ys, ps, rs in zip(in_ys, in_ps, in_rs)] + + +@cv.gapi.kernel(GParseEyes) +class GParseEyesImpl: + """ Custom kernel. Get information about eyes + """ + @staticmethod + def run(in_landm_per_face, in_face_rcs, frame_size): + """ Сustom kernel executable code + + Params: + in_landm_per_face: landmarks from inference of facial-landmarks network for each face + in_face_rcs: bounding boxes for each face + frame_size: size of input image + + Return: + Arrays of ROI for left and right eyes, array of midpoints and + array of landmarks points + """ + left_eyes = [] + right_eyes = [] + midpoints = [] + lmarks = [] + surface = (0, 0, *frame_size) + for landm_face, rect in zip(in_landm_per_face, in_face_rcs): + points = process_landmarks(*rect, landm_face) + lmarks.extend(points) + + rect, midpoint_l = eye_box(points[0], points[1]) + left_eyes.append(intersection(surface, rect)) + + rect, midpoint_r = eye_box(points[2], points[3]) + right_eyes.append(intersection(surface, rect)) + + midpoints.append(midpoint_l) + midpoints.append(midpoint_r) + return left_eyes, right_eyes, midpoints, lmarks + + +@cv.gapi.kernel(GGetStates) +class GGetStatesImpl: + """ Custom kernel. Get state of eye - open or closed + """ + @staticmethod + def run(eyesl, eyesr): + """ Сustom kernel executable code + + Params: + eyesl: result of inference of open-closed-eye network for left eye + eyesr: result of inference of open-closed-eye network for right eye + + Return: + States of left eyes and states of right eyes + """ + out_l_st = [int(st) for eye_l in eyesl for st in (eye_l[:, 0] < eye_l[:, 1]).ravel()] + out_r_st = [int(st) for eye_r in eyesr for st in (eye_r[:, 0] < eye_r[:, 1]).ravel()] + return out_l_st, out_r_st + + +if __name__ == '__main__': + ARGUMENTS = build_argparser().parse_args() + + # ------------------------Demo's graph------------------------ + g_in = cv.GMat() + + # Detect faces + face_inputs = cv.GInferInputs() + face_inputs.setInput('data', g_in) + face_outputs = cv.gapi.infer('face-detection', face_inputs) + faces = face_outputs.at('detection_out') + + # Parse faces + sz = cv.gapi.streaming.size(g_in) + faces_rc = cv.gapi.parseSSD(faces, sz, 0.5, False, False) + + # Detect poses + head_inputs = cv.GInferInputs() + head_inputs.setInput('data', g_in) + face_outputs = cv.gapi.infer('head-pose', faces_rc, head_inputs) + angles_y = face_outputs.at('angle_y_fc') + angles_p = face_outputs.at('angle_p_fc') + angles_r = face_outputs.at('angle_r_fc') + + # Parse poses + heads_pos = GProcessPoses.on(angles_y, angles_p, angles_r) + + # Detect landmarks + landmark_inputs = cv.GInferInputs() + landmark_inputs.setInput('data', g_in) + landmark_outputs = cv.gapi.infer('facial-landmarks', faces_rc, + landmark_inputs) + landmark = landmark_outputs.at('align_fc3') + + # Parse landmarks + left_eyes, right_eyes, mids, lmarks = GParseEyes.on(landmark, faces_rc, sz) + + # Detect eyes + eyes_inputs = cv.GInferInputs() + eyes_inputs.setInput('input.1', g_in) + eyesl_outputs = cv.gapi.infer('open-closed-eye', left_eyes, eyes_inputs) + eyesr_outputs = cv.gapi.infer('open-closed-eye', right_eyes, eyes_inputs) + eyesl = eyesl_outputs.at('19') + eyesr = eyesr_outputs.at('19') + + # Process eyes states + l_eye_st, r_eye_st = GGetStates.on(eyesl, eyesr) + + # Gaze estimation + gaze_inputs = cv.GInferListInputs() + gaze_inputs.setInput('left_eye_image', left_eyes) + gaze_inputs.setInput('right_eye_image', right_eyes) + gaze_inputs.setInput('head_pose_angles', heads_pos) + gaze_outputs = cv.gapi.infer2('gaze-estimation', g_in, gaze_inputs) + gaze_vectors = gaze_outputs.at('gaze_vector') + + out = cv.gapi.copy(g_in) + # ------------------------End of graph------------------------ + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(out, + faces_rc, + left_eyes, + right_eyes, + gaze_vectors, + angles_y, + angles_p, + angles_r, + l_eye_st, + r_eye_st, + mids, + lmarks)) + + # Networks + face_net = cv.gapi.ie.params('face-detection', ARGUMENTS.facem, + weight_path(ARGUMENTS.facem), ARGUMENTS.faced) + head_pose_net = cv.gapi.ie.params('head-pose', ARGUMENTS.headm, + weight_path(ARGUMENTS.headm), ARGUMENTS.headd) + landmarks_net = cv.gapi.ie.params('facial-landmarks', ARGUMENTS.landm, + weight_path(ARGUMENTS.landm), ARGUMENTS.landd) + gaze_net = cv.gapi.ie.params('gaze-estimation', ARGUMENTS.gazem, + weight_path(ARGUMENTS.gazem), ARGUMENTS.gazed) + eye_net = cv.gapi.ie.params('open-closed-eye', ARGUMENTS.eyem, + weight_path(ARGUMENTS.eyem), ARGUMENTS.eyed) + + nets = cv.gapi.networks(face_net, head_pose_net, landmarks_net, gaze_net, eye_net) + + # Kernels pack + kernels = cv.gapi.kernels(GParseEyesImpl, GProcessPosesImpl, GGetStatesImpl) + + # ------------------------Execution part------------------------ + ccomp = comp.compileStreaming(args=cv.gapi.compile_args(kernels, nets)) + source = cv.gapi.wip.make_capture_src(ARGUMENTS.input) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + frames = 0 + fps = 0 + print('Processing') + START_TIME = time.time() + + while True: + start_time_cycle = time.time() + has_frame, (oimg, + outr, + l_eyes, + r_eyes, + outg, + out_y, + out_p, + out_r, + out_st_l, + out_st_r, + out_mids, + outl) = ccomp.pull() + + if not has_frame: + break + + # Draw + GREEN = (0, 255, 0) + RED = (0, 0, 255) + WHITE = (255, 255, 255) + BLUE = (255, 0, 0) + PINK = (255, 0, 255) + YELLOW = (0, 255, 255) + + M_PI_180 = np.pi / 180 + M_PI_2 = np.pi / 2 + M_PI = np.pi + + FACES_SIZE = len(outr) + + for i, out_rect in enumerate(outr): + # Face box + cv.rectangle(oimg, out_rect, WHITE, 1) + rx, ry, rwidth, rheight = out_rect + + # Landmarks + lm_radius = int(0.01 * rwidth + 1) + lmsize = int(len(outl) / FACES_SIZE) + for j in range(lmsize): + cv.circle(oimg, outl[j + i * lmsize], lm_radius, YELLOW, -1) + + # Headposes + yaw = out_y[i] + pitch = out_p[i] + roll = out_r[i] + sin_y = np.sin(yaw[:] * M_PI_180) + sin_p = np.sin(pitch[:] * M_PI_180) + sin_r = np.sin(roll[:] * M_PI_180) + + cos_y = np.cos(yaw[:] * M_PI_180) + cos_p = np.cos(pitch[:] * M_PI_180) + cos_r = np.cos(roll[:] * M_PI_180) + + axis_length = 0.4 * rwidth + x_center = int(rx + rwidth / 2) + y_center = int(ry + rheight / 2) + + # center to right + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * (cos_r * cos_y + sin_y * sin_p * sin_r)), + int(y_center + axis_length * cos_p * sin_r)], + RED, 2) + + # center to top + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * (cos_r * sin_y * sin_p + cos_y * sin_r)), + int(y_center - axis_length * cos_p * cos_r)], + GREEN, 2) + + # center to forward + cv.line(oimg, [x_center, y_center], + [int(x_center + axis_length * sin_y * cos_p), + int(y_center + axis_length * sin_p)], + PINK, 2) + + scale_box = 0.002 * rwidth + cv.putText(oimg, "head pose: (y=%0.0f, p=%0.0f, r=%0.0f)" % + (np.round(yaw), np.round(pitch), np.round(roll)), + [int(rx), int(ry + rheight + 5 * rwidth / 100)], + cv.FONT_HERSHEY_PLAIN, scale_box * 2, WHITE, 1) + + # Eyes boxes + color_l = GREEN if out_st_l[i] else RED + cv.rectangle(oimg, l_eyes[i], color_l, 1) + color_r = GREEN if out_st_r[i] else RED + cv.rectangle(oimg, r_eyes[i], color_r, 1) + + # Gaze vectors + norm_gazes = np.linalg.norm(outg[i][0]) + gaze_vector = outg[i][0] / norm_gazes + + arrow_length = 0.4 * rwidth + gaze_arrow = [arrow_length * gaze_vector[0], -arrow_length * gaze_vector[1]] + left_arrow = [int(a+b) for a, b in zip(out_mids[0 + i * 2], gaze_arrow)] + right_arrow = [int(a+b) for a, b in zip(out_mids[1 + i * 2], gaze_arrow)] + if out_st_l[i]: + cv.arrowedLine(oimg, out_mids[0 + i * 2], left_arrow, BLUE, 2) + if out_st_r[i]: + cv.arrowedLine(oimg, out_mids[1 + i * 2], right_arrow, BLUE, 2) + + v0, v1, v2 = outg[i][0] + + gaze_angles = [180 / M_PI * (M_PI_2 + np.arctan2(v2, v0)), + 180 / M_PI * (M_PI_2 - np.arccos(v1 / norm_gazes))] + cv.putText(oimg, "gaze angles: (h=%0.0f, v=%0.0f)" % + (np.round(gaze_angles[0]), np.round(gaze_angles[1])), + [int(rx), int(ry + rheight + 12 * rwidth / 100)], + cv.FONT_HERSHEY_PLAIN, scale_box * 2, WHITE, 1) + + # Add FPS value to frame + cv.putText(oimg, "FPS: %0i" % (fps), [int(20), int(40)], + cv.FONT_HERSHEY_PLAIN, 2, RED, 2) + + # Show result + cv.imshow('Gaze Estimation', oimg) + cv.waitKey(1) + + fps = int(1. / (time.time() - start_time_cycle)) + frames += 1 + EXECUTION_TIME = time.time() - START_TIME + print('Execution successful') + print('Mean FPS is ', int(frames / EXECUTION_TIME)) diff --git a/modules/gapi/misc/python/shadow_gapi.hpp b/modules/gapi/misc/python/shadow_gapi.hpp index 40dab41581..33a0e0fde1 100644 --- a/modules/gapi/misc/python/shadow_gapi.hpp +++ b/modules/gapi/misc/python/shadow_gapi.hpp @@ -3,64 +3,81 @@ namespace cv { - struct GAPI_EXPORTS_W_SIMPLE GCompileArg { }; +struct GAPI_EXPORTS_W_SIMPLE GCompileArg +{ + GAPI_WRAP GCompileArg(gapi::GKernelPackage arg); + GAPI_WRAP GCompileArg(gapi::GNetPackage arg); + GAPI_WRAP GCompileArg(gapi::streaming::queue_capacity arg); +}; - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage pkg); - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GNetPackage pkg); - GAPI_EXPORTS_W GCompileArgs compile_args(gapi::GKernelPackage kernels, gapi::GNetPackage nets); +class GAPI_EXPORTS_W_SIMPLE GInferInputs +{ +public: + GAPI_WRAP GInferInputs(); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GMat& value); + GAPI_WRAP GInferInputs& setInput(const std::string& name, const cv::GFrame& value); +}; - // NB: This classes doesn't exist in *.so - // HACK: Mark them as a class to force python wrapper generate code for this entities - class GAPI_EXPORTS_W_SIMPLE GProtoArg { }; - class GAPI_EXPORTS_W_SIMPLE GProtoInputArgs { }; - class GAPI_EXPORTS_W_SIMPLE GProtoOutputArgs { }; - class GAPI_EXPORTS_W_SIMPLE GRunArg { }; - class GAPI_EXPORTS_W_SIMPLE GMetaArg { GAPI_WRAP GMetaArg(); }; +class GAPI_EXPORTS_W_SIMPLE GInferListInputs +{ +public: + GAPI_WRAP GInferListInputs(); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); + GAPI_WRAP GInferListInputs setInput(const std::string& name, const cv::GArray& value); +}; - using GProtoInputArgs = GIOProtoArgs; - using GProtoOutputArgs = GIOProtoArgs; +class GAPI_EXPORTS_W_SIMPLE GInferOutputs +{ +public: + GAPI_WRAP GInferOutputs(); + GAPI_WRAP cv::GMat at(const std::string& name); +}; - class GAPI_EXPORTS_W_SIMPLE GInferInputs - { - public: - GAPI_WRAP GInferInputs(); - GAPI_WRAP void setInput(const std::string& name, const cv::GMat& value); - GAPI_WRAP void setInput(const std::string& name, const cv::GFrame& value); - }; +class GAPI_EXPORTS_W_SIMPLE GInferListOutputs +{ +public: + GAPI_WRAP GInferListOutputs(); + GAPI_WRAP cv::GArray at(const std::string& name); +}; - class GAPI_EXPORTS_W_SIMPLE GInferListInputs - { - public: - GAPI_WRAP GInferListInputs(); - GAPI_WRAP void setInput(const std::string& name, const cv::GArray& value); - GAPI_WRAP void setInput(const std::string& name, const cv::GArray& value); - }; +namespace gapi +{ +namespace wip +{ +class GAPI_EXPORTS_W IStreamSource { }; +namespace draw +{ + // NB: These render primitives are partially wrapped in shadow file + // because cv::Rect conflicts with cv::gapi::wip::draw::Rect in python generator + // and cv::Rect2i breaks standalone mode. + struct Rect + { + GAPI_WRAP Rect(const cv::Rect2i& rect_, + const cv::Scalar& color_, + int thick_ = 1, + int lt_ = 8, + int shift_ = 0); + }; - class GAPI_EXPORTS_W_SIMPLE GInferOutputs - { - public: - GAPI_WRAP GInferOutputs(); - GAPI_WRAP cv::GMat at(const std::string& name); - }; + struct Mosaic + { + GAPI_WRAP Mosaic(const cv::Rect2i& mos_, int cellSz_, int decim_); + }; +} // namespace draw +} // namespace wip +namespace streaming +{ + // FIXME: Extend to work with an arbitrary G-type. + cv::GOpaque GAPI_EXPORTS_W timestamp(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seqNo(cv::GMat); + cv::GOpaque GAPI_EXPORTS_W seq_id(cv::GMat); - class GAPI_EXPORTS_W_SIMPLE GInferListOutputs - { - public: - GAPI_WRAP GInferListOutputs(); - GAPI_WRAP cv::GArray at(const std::string& name); - }; + GAPI_EXPORTS_W cv::GMat desync(const cv::GMat &g); +} // namespace streaming +} // namespace gapi - namespace detail - { - struct GAPI_EXPORTS_W_SIMPLE ExtractArgsCallback { }; - struct GAPI_EXPORTS_W_SIMPLE ExtractMetaCallback { }; - } // namespace detail - - namespace gapi - { - namespace wip - { - class GAPI_EXPORTS_W IStreamSource { }; - } // namespace wip - } // namespace gapi +namespace detail +{ + gapi::GNetParam GAPI_EXPORTS_W strip(gapi::ie::PyParams params); +} // namespace detail } // namespace cv diff --git a/modules/gapi/misc/python/test/test_gapi_core.py b/modules/gapi/misc/python/test/test_gapi_core.py index 814d05d7cd..780558d98b 100644 --- a/modules/gapi/misc/python/test/test_gapi_core.py +++ b/modules/gapi/misc/python/test/test_gapi_core.py @@ -3,187 +3,209 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -# Plaidml is an optional backend -pkgs = [ - ('ocl' , cv.gapi.core.ocl.kernels()), - ('cpu' , cv.gapi.core.cpu.kernels()), - ('fluid' , cv.gapi.core.fluid.kernels()) - # ('plaidml', cv.gapi.core.plaidml.kernels()) - ] +try: + + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + # Plaidml is an optional backend + pkgs = [ + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] -class gapi_core_test(NewOpenCVTests): + class gapi_core_test(NewOpenCVTests): - def test_add(self): - # TODO: Extend to use any type and size here - sz = (720, 1280) - in1 = np.full(sz, 100) - in2 = np.full(sz, 50) + def test_add(self): + # TODO: Extend to use any type and size here + sz = (720, 1280) + in1 = np.full(sz, 100) + in2 = np.full(sz, 50) - # OpenCV - expected = cv.add(in1, in2) + # OpenCV + expected = cv.add(in1, in2) - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - g_out = cv.gapi.add(g_in1, g_in2) - comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + g_out = cv.gapi.add(g_in1, g_in2) + comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1, in2), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - - - def test_add_uint8(self): - sz = (720, 1280) - in1 = np.full(sz, 100, dtype=np.uint8) - in2 = np.full(sz, 50 , dtype=np.uint8) - - # OpenCV - expected = cv.add(in1, in2) - - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - g_out = cv.gapi.add(g_in1, g_in2) - comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) - - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1, in2), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - - - def test_mean(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.imread(img_path) - - # OpenCV - expected = cv.mean(in_mat) - - # G-API - g_in = cv.GMat() - g_out = cv.gapi.mean(g_in) - comp = cv.GComputation(g_in, g_out) - - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - - - def test_split3(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.imread(img_path) - - # OpenCV - expected = cv.split(in_mat) - - # G-API - g_in = cv.GMat() - b, g, r = cv.gapi.split3(g_in) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) - # Comparison - for e, a in zip(expected, actual): - self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF), + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1, in2), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), 'Failed on ' + pkg_name + ' backend') - self.assertEqual(e.dtype, a.dtype, 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - def test_threshold(self): - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in_mat = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) - maxv = (30, 30) + def test_add_uint8(self): + sz = (720, 1280) + in1 = np.full(sz, 100, dtype=np.uint8) + in2 = np.full(sz, 50 , dtype=np.uint8) - # OpenCV - expected_thresh, expected_mat = cv.threshold(in_mat, maxv[0], maxv[0], cv.THRESH_TRIANGLE) + # OpenCV + expected = cv.add(in1, in2) - # G-API - g_in = cv.GMat() - g_sc = cv.GScalar() - mat, threshold = cv.gapi.threshold(g_in, g_sc, cv.THRESH_TRIANGLE) - comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(mat, threshold)) + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + g_out = cv.gapi.add(g_in1, g_in2) + comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual_mat, actual_thresh = comp.apply(cv.gin(in_mat, maxv), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected_mat, actual_mat, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected_mat.dtype, actual_mat.dtype, - 'Failed on ' + pkg_name + ' backend') - self.assertEqual(expected_thresh, actual_thresh[0], - 'Failed on ' + pkg_name + ' backend') - - def test_kmeans(self): - # K-means params - count = 100 - sz = (count, 2) - in_mat = np.random.random(sz).astype(np.float32) - K = 5 - flags = cv.KMEANS_RANDOM_CENTERS - attempts = 1; - criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) - - # G-API - g_in = cv.GMat() - compactness, out_labels, centers = cv.gapi.kmeans(g_in, K, criteria, attempts, flags) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(compactness, out_labels, centers)) - - compact, labels, centers = comp.apply(cv.gin(in_mat)) - - # Assert - self.assertTrue(compact >= 0) - self.assertEqual(sz[0], labels.shape[0]) - self.assertEqual(1, labels.shape[1]) - self.assertTrue(labels.size != 0) - self.assertEqual(centers.shape[1], sz[1]); - self.assertEqual(centers.shape[0], K); - self.assertTrue(centers.size != 0); + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1, in2), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected.dtype, actual.dtype, 'Failed on ' + pkg_name + ' backend') - def generate_random_points(self, sz): - arr = np.random.random(sz).astype(np.float32).T - return list(zip(arr[0], arr[1])) + def test_mean(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) + + # OpenCV + expected = cv.mean(in_mat) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.mean(g_in) + comp = cv.GComputation(g_in, g_out) + + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - def test_kmeans_2d(self): - # K-means 2D params - count = 100 - sz = (count, 2) - amount = sz[0] - K = 5 - flags = cv.KMEANS_RANDOM_CENTERS - attempts = 1; - criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0); - in_vector = self.generate_random_points(sz) - in_labels = [] + def test_split3(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.imread(img_path) - # G-API - data = cv.GArrayT(cv.gapi.CV_POINT2F) - best_labels = cv.GArrayT(cv.gapi.CV_INT) + # OpenCV + expected = cv.split(in_mat) - compactness, out_labels, centers = cv.gapi.kmeans(data, K, best_labels, criteria, attempts, flags); - comp = cv.GComputation(cv.GIn(data, best_labels), cv.GOut(compactness, out_labels, centers)); + # G-API + g_in = cv.GMat() + b, g, r = cv.gapi.split3(g_in) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - compact, labels, centers = comp.apply(cv.gin(in_vector, in_labels)); + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) + # Comparison + for e, a in zip(expected, actual): + self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(e.dtype, a.dtype, 'Failed on ' + pkg_name + ' backend') - # Assert - self.assertTrue(compact >= 0) - self.assertEqual(amount, len(labels)) - self.assertEqual(K, len(centers)) + + def test_threshold(self): + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in_mat = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) + maxv = (30, 30) + + # OpenCV + expected_thresh, expected_mat = cv.threshold(in_mat, maxv[0], maxv[0], cv.THRESH_TRIANGLE) + + # G-API + g_in = cv.GMat() + g_sc = cv.GScalar() + mat, threshold = cv.gapi.threshold(g_in, g_sc, cv.THRESH_TRIANGLE) + comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(mat, threshold)) + + for pkg_name, pkg in pkgs: + actual_mat, actual_thresh = comp.apply(cv.gin(in_mat, maxv), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected_mat, actual_mat, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected_mat.dtype, actual_mat.dtype, + 'Failed on ' + pkg_name + ' backend') + self.assertEqual(expected_thresh, actual_thresh[0], + 'Failed on ' + pkg_name + ' backend') + + + def test_kmeans(self): + # K-means params + count = 100 + sz = (count, 2) + in_mat = np.random.random(sz).astype(np.float32) + K = 5 + flags = cv.KMEANS_RANDOM_CENTERS + attempts = 1 + criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) + + # G-API + g_in = cv.GMat() + compactness, out_labels, centers = cv.gapi.kmeans(g_in, K, criteria, attempts, flags) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(compactness, out_labels, centers)) + + compact, labels, centers = comp.apply(cv.gin(in_mat)) + + # Assert + self.assertTrue(compact >= 0) + self.assertEqual(sz[0], labels.shape[0]) + self.assertEqual(1, labels.shape[1]) + self.assertTrue(labels.size != 0) + self.assertEqual(centers.shape[1], sz[1]) + self.assertEqual(centers.shape[0], K) + self.assertTrue(centers.size != 0) + + + def generate_random_points(self, sz): + arr = np.random.random(sz).astype(np.float32).T + return list(zip(arr[0], arr[1])) + + + def test_kmeans_2d(self): + # K-means 2D params + count = 100 + sz = (count, 2) + amount = sz[0] + K = 5 + flags = cv.KMEANS_RANDOM_CENTERS + attempts = 1 + criteria = (cv.TERM_CRITERIA_MAX_ITER + cv.TERM_CRITERIA_EPS, 30, 0) + in_vector = self.generate_random_points(sz) + in_labels = [] + + # G-API + data = cv.GArrayT(cv.gapi.CV_POINT2F) + best_labels = cv.GArrayT(cv.gapi.CV_INT) + + compactness, out_labels, centers = cv.gapi.kmeans(data, K, best_labels, criteria, attempts, flags) + comp = cv.GComputation(cv.GIn(data, best_labels), cv.GOut(compactness, out_labels, centers)) + + compact, labels, centers = comp.apply(cv.gin(in_vector, in_labels)) + + # Assert + self.assertTrue(compact >= 0) + self.assertEqual(amount, len(labels)) + self.assertEqual(K, len(centers)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_imgproc.py b/modules/gapi/misc/python/test/test_gapi_imgproc.py index ed6f883fe5..365a5a8cca 100644 --- a/modules/gapi/misc/python/test/test_gapi_imgproc.py +++ b/modules/gapi/misc/python/test/test_gapi_imgproc.py @@ -3,103 +3,124 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -# Plaidml is an optional backend -pkgs = [ - ('ocl' , cv.gapi.core.ocl.kernels()), - ('cpu' , cv.gapi.core.cpu.kernels()), - ('fluid' , cv.gapi.core.fluid.kernels()) - # ('plaidml', cv.gapi.core.plaidml.kernels()) - ] +try: + + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + # Plaidml is an optional backend + pkgs = [ + ('ocl' , cv.gapi.core.ocl.kernels()), + ('cpu' , cv.gapi.core.cpu.kernels()), + ('fluid' , cv.gapi.core.fluid.kernels()) + # ('plaidml', cv.gapi.core.plaidml.kernels()) + ] -class gapi_imgproc_test(NewOpenCVTests): + class gapi_imgproc_test(NewOpenCVTests): - def test_good_features_to_track(self): - # TODO: Extend to use any type and size here - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in1 = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) + def test_good_features_to_track(self): + # TODO: Extend to use any type and size here + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in1 = cv.cvtColor(cv.imread(img_path), cv.COLOR_RGB2GRAY) - # NB: goodFeaturesToTrack configuration - max_corners = 50 - quality_lvl = 0.01 - min_distance = 10 - block_sz = 3 - use_harris_detector = True - k = 0.04 - mask = None + # NB: goodFeaturesToTrack configuration + max_corners = 50 + quality_lvl = 0.01 + min_distance = 10 + block_sz = 3 + use_harris_detector = True + k = 0.04 + mask = None - # OpenCV - expected = cv.goodFeaturesToTrack(in1, max_corners, quality_lvl, - min_distance, mask=mask, - blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) + # OpenCV + expected = cv.goodFeaturesToTrack(in1, max_corners, quality_lvl, + min_distance, mask=mask, + blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.goodFeaturesToTrack(g_in, max_corners, quality_lvl, - min_distance, mask, block_sz, use_harris_detector, k) + # G-API + g_in = cv.GMat() + g_out = cv.gapi.goodFeaturesToTrack(g_in, max_corners, quality_lvl, + min_distance, mask, block_sz, use_harris_detector, k) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1), args=cv.compile_args(pkg)) - # NB: OpenCV & G-API have different output shapes: - # OpenCV - (num_points, 1, 2) - # G-API - (num_points, 2) - # Comparison - self.assertEqual(0.0, cv.norm(expected.flatten(), - np.array(actual, dtype=np.float32).flatten(), - cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1), args=cv.gapi.compile_args(pkg)) + # NB: OpenCV & G-API have different output shapes: + # OpenCV - (num_points, 1, 2) + # G-API - (num_points, 2) + # Comparison + self.assertEqual(0.0, cv.norm(expected.flatten(), + np.array(actual, dtype=np.float32).flatten(), + cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - def test_rgb2gray(self): - # TODO: Extend to use any type and size here - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - in1 = cv.imread(img_path) + def test_rgb2gray(self): + # TODO: Extend to use any type and size here + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + in1 = cv.imread(img_path) - # OpenCV - expected = cv.cvtColor(in1, cv.COLOR_RGB2GRAY) + # OpenCV + expected = cv.cvtColor(in1, cv.COLOR_RGB2GRAY) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.RGB2Gray(g_in) + # G-API + g_in = cv.GMat() + g_out = cv.gapi.RGB2Gray(g_in) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(in1), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(in1), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') - def test_bounding_rect(self): - sz = 1280 - fscale = 256 + def test_bounding_rect(self): + sz = 1280 + fscale = 256 - def sample_value(fscale): - return np.random.uniform(0, 255 * fscale) / fscale + def sample_value(fscale): + return np.random.uniform(0, 255 * fscale) / fscale - points = np.array([(sample_value(fscale), sample_value(fscale)) for _ in range(1280)], np.float32) + points = np.array([(sample_value(fscale), sample_value(fscale)) for _ in range(1280)], np.float32) - # OpenCV - expected = cv.boundingRect(points) + # OpenCV + expected = cv.boundingRect(points) - # G-API - g_in = cv.GMat() - g_out = cv.gapi.boundingRect(g_in) + # G-API + g_in = cv.GMat() + g_out = cv.gapi.boundingRect(g_in) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - for pkg_name, pkg in pkgs: - actual = comp.apply(cv.gin(points), args=cv.compile_args(pkg)) - # Comparison - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), - 'Failed on ' + pkg_name + ' backend') + for pkg_name, pkg in pkgs: + actual = comp.apply(cv.gin(points), args=cv.gapi.compile_args(pkg)) + # Comparison + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF), + 'Failed on ' + pkg_name + ' backend') + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_infer.py b/modules/gapi/misc/python/test/test_gapi_infer.py index db048f5786..8ecc957e41 100644 --- a/modules/gapi/misc/python/test/test_gapi_infer.py +++ b/modules/gapi/misc/python/test/test_gapi_infer.py @@ -3,318 +3,338 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -class test_gapi_infer(NewOpenCVTests): +try: - def infer_reference_network(self, model_path, weights_path, img): - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - - blob = cv.dnn.blobFromImage(img) - - net.setInput(blob) - return net.forward(net.getUnconnectedOutLayersNames()) + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') - def make_roi(self, img, roi): - return img[roi[1]:roi[1] + roi[3], roi[0]:roi[0] + roi[2], ...] + class test_gapi_infer(NewOpenCVTests): + + def infer_reference_network(self, model_path, weights_path, img): + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + + blob = cv.dnn.blobFromImage(img) + + net.setInput(blob) + return net.forward(net.getUnconnectedOutLayersNames()) - def test_age_gender_infer(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' - - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.resize(cv.imread(img_path), (62,62)) - - # OpenCV DNN - dnn_age, dnn_gender = self.infer_reference_network(model_path, weights_path, img) - - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) - - outputs = cv.gapi.infer("net", inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") - - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp))) - - # Check - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + def make_roi(self, img, roi): + return img[roi[1]:roi[1] + roi[3], roi[0]:roi[0] + roi[2], ...] - def test_age_gender_infer_roi(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + def test_age_gender_infer(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) - roi = (10, 10, 62, 62) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.resize(cv.imread(img_path), (62,62)) - # OpenCV DNN - dnn_age, dnn_gender = self.infer_reference_network(model_path, + # OpenCV DNN + dnn_age, dnn_gender = self.infer_reference_network(model_path, weights_path, img) + + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + outputs = cv.gapi.infer("net", inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.gapi.compile_args(cv.gapi.networks(pp))) + + # Check + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + + + def test_age_gender_infer_roi(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' + + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) + roi = (10, 10, 62, 62) + + # OpenCV DNN + dnn_age, dnn_gender = self.infer_reference_network(model_path, + weights_path, + self.make_roi(img, roi)) + + # OpenCV G-API + g_in = cv.GMat() + g_roi = cv.GOpaqueT(cv.gapi.CV_RECT) + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) + + outputs = cv.gapi.infer("net", g_roi, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") + + comp = cv.GComputation(cv.GIn(g_in, g_roi), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + + gapi_age, gapi_gender = comp.apply(cv.gin(img, roi), args=cv.gapi.compile_args(cv.gapi.networks(pp))) + + # Check + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + + + def test_age_gender_infer_roi_list(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return + + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' + + rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) + + # OpenCV DNN + dnn_age_list = [] + dnn_gender_list = [] + for roi in rois: + age, gender = self.infer_reference_network(model_path, weights_path, self.make_roi(img, roi)) + dnn_age_list.append(age) + dnn_gender_list.append(gender) - # OpenCV G-API - g_in = cv.GMat() - g_roi = cv.GOpaqueT(cv.gapi.CV_RECT) - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + # OpenCV G-API + g_in = cv.GMat() + g_rois = cv.GArrayT(cv.gapi.CV_RECT) + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) - outputs = cv.gapi.infer("net", g_roi, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") + outputs = cv.gapi.infer("net", g_rois, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") - comp = cv.GComputation(cv.GIn(g_in, g_roi), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - gapi_age, gapi_gender = comp.apply(cv.gin(img, roi), args=cv.compile_args(cv.gapi.networks(pp))) + gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) - # Check - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + # Check + for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, + gapi_gender_list, + dnn_age_list, + dnn_gender_list): + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) - def test_age_gender_infer_roi_list(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + def test_age_gender_infer2_roi(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' + root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + device_id = 'CPU' - rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) + rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + img = cv.imread(img_path) - # OpenCV DNN - dnn_age_list = [] - dnn_gender_list = [] - for roi in rois: - age, gender = self.infer_reference_network(model_path, - weights_path, - self.make_roi(img, roi)) - dnn_age_list.append(age) - dnn_gender_list.append(gender) + # OpenCV DNN + dnn_age_list = [] + dnn_gender_list = [] + for roi in rois: + age, gender = self.infer_reference_network(model_path, + weights_path, + self.make_roi(img, roi)) + dnn_age_list.append(age) + dnn_gender_list.append(gender) - # OpenCV G-API - g_in = cv.GMat() - g_rois = cv.GArrayT(cv.gapi.CV_RECT) - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + # OpenCV G-API + g_in = cv.GMat() + g_rois = cv.GArrayT(cv.gapi.CV_RECT) + inputs = cv.GInferListInputs() + inputs.setInput('data', g_rois) - outputs = cv.gapi.infer("net", g_rois, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") + outputs = cv.gapi.infer2("net", g_in, inputs) + age_g = outputs.at("age_conv3") + gender_g = outputs.at("prob") - comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), - args=cv.compile_args(cv.gapi.networks(pp))) + gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) - # Check - for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, - gapi_gender_list, - dnn_age_list, - dnn_gender_list): - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) - - - def test_age_gender_infer2_roi(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return - - root_path = '/omz_intel_models/intel/age-gender-recognition-retail-0013/FP32/age-gender-recognition-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - device_id = 'CPU' - - rois = [(10, 15, 62, 62), (23, 50, 62, 62), (14, 100, 62, 62), (80, 50, 62, 62)] - img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - img = cv.imread(img_path) - - # OpenCV DNN - dnn_age_list = [] - dnn_gender_list = [] - for roi in rois: - age, gender = self.infer_reference_network(model_path, - weights_path, - self.make_roi(img, roi)) - dnn_age_list.append(age) - dnn_gender_list.append(gender) - - # OpenCV G-API - g_in = cv.GMat() - g_rois = cv.GArrayT(cv.gapi.CV_RECT) - inputs = cv.GInferListInputs() - inputs.setInput('data', g_rois) - - outputs = cv.gapi.infer2("net", g_in, inputs) - age_g = outputs.at("age_conv3") - gender_g = outputs.at("prob") - - comp = cv.GComputation(cv.GIn(g_in, g_rois), cv.GOut(age_g, gender_g)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - - gapi_age_list, gapi_gender_list = comp.apply(cv.gin(img, rois), - args=cv.compile_args(cv.gapi.networks(pp))) - - # Check - for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, - gapi_gender_list, - dnn_age_list, - dnn_gender_list): - self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) - self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) + # Check + for gapi_age, gapi_gender, dnn_age, dnn_gender in zip(gapi_age_list, + gapi_gender_list, + dnn_age_list, + dnn_gender_list): + self.assertEqual(0.0, cv.norm(dnn_gender, gapi_gender, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(dnn_age, gapi_age, cv.NORM_INF)) - def test_person_detection_retail_0013(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + def test_person_detection_retail_0013(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - device_id = 'CPU' - img = cv.resize(cv.imread(img_path), (544, 320)) + root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + device_id = 'CPU' + img = cv.resize(cv.imread(img_path), (544, 320)) - # OpenCV DNN - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + # OpenCV DNN + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - blob = cv.dnn.blobFromImage(img) + blob = cv.dnn.blobFromImage(img) - def parseSSD(detections, size): - h, w = size - bboxes = [] - detections = detections.reshape(-1, 7) - for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: - if confidence >= 0.5: - x = int(xmin * w) - y = int(ymin * h) - width = int(xmax * w - x) - height = int(ymax * h - y) - bboxes.append((x, y, width, height)) + def parseSSD(detections, size): + h, w = size + bboxes = [] + detections = detections.reshape(-1, 7) + for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: + if confidence >= 0.5: + x = int(xmin * w) + y = int(ymin * h) + width = int(xmax * w - x) + height = int(ymax * h - y) + bboxes.append((x, y, width, height)) - return bboxes + return bboxes - net.setInput(blob) - dnn_detections = net.forward() - dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) + net.setInput(blob) + dnn_detections = net.forward() + dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) - g_sz = cv.gapi.streaming.size(g_in) - outputs = cv.gapi.infer("net", inputs) - detections = outputs.at("detection_out") - bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) + g_sz = cv.gapi.streaming.size(g_in) + outputs = cv.gapi.infer("net", inputs) + detections = outputs.at("detection_out") + bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - gapi_age, gapi_gender = comp.apply(cv.gin(img), args=cv.compile_args(cv.gapi.networks(pp))) + gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) - gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), - args=cv.compile_args(cv.gapi.networks(pp))) - - # Comparison - self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), - np.array(gapi_boxes).flatten(), - cv.NORM_INF)) + # Comparison + self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), + np.array(gapi_boxes).flatten(), + cv.NORM_INF)) - def test_person_detection_retail_0013(self): - # NB: Check IE - if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): - return + def test_person_detection_retail_0013(self): + # NB: Check IE + if not cv.dnn.DNN_TARGET_CPU in cv.dnn.getAvailableTargets(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE): + return - root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' - model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) - img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) - device_id = 'CPU' - img = cv.resize(cv.imread(img_path), (544, 320)) + root_path = '/omz_intel_models/intel/person-detection-retail-0013/FP32/person-detection-retail-0013' + model_path = self.find_file(root_path + '.xml', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + weights_path = self.find_file(root_path + '.bin', [os.environ.get('OPENCV_DNN_TEST_DATA_PATH')]) + img_path = self.find_file('gpu/lbpcascade/er.png', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + device_id = 'CPU' + img = cv.resize(cv.imread(img_path), (544, 320)) - # OpenCV DNN - net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) - net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) - net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) + # OpenCV DNN + net = cv.dnn.readNetFromModelOptimizer(model_path, weights_path) + net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE) + net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU) - blob = cv.dnn.blobFromImage(img) + blob = cv.dnn.blobFromImage(img) - def parseSSD(detections, size): - h, w = size - bboxes = [] - detections = detections.reshape(-1, 7) - for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: - if confidence >= 0.5: - x = int(xmin * w) - y = int(ymin * h) - width = int(xmax * w - x) - height = int(ymax * h - y) - bboxes.append((x, y, width, height)) + def parseSSD(detections, size): + h, w = size + bboxes = [] + detections = detections.reshape(-1, 7) + for sample_id, class_id, confidence, xmin, ymin, xmax, ymax in detections: + if confidence >= 0.5: + x = int(xmin * w) + y = int(ymin * h) + width = int(xmax * w - x) + height = int(ymax * h - y) + bboxes.append((x, y, width, height)) - return bboxes + return bboxes - net.setInput(blob) - dnn_detections = net.forward() - dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) + net.setInput(blob) + dnn_detections = net.forward() + dnn_boxes = parseSSD(np.array(dnn_detections), img.shape[:2]) - # OpenCV G-API - g_in = cv.GMat() - inputs = cv.GInferInputs() - inputs.setInput('data', g_in) + # OpenCV G-API + g_in = cv.GMat() + inputs = cv.GInferInputs() + inputs.setInput('data', g_in) - g_sz = cv.gapi.streaming.size(g_in) - outputs = cv.gapi.infer("net", inputs) - detections = outputs.at("detection_out") - bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) + g_sz = cv.gapi.streaming.size(g_in) + outputs = cv.gapi.infer("net", inputs) + detections = outputs.at("detection_out") + bboxes = cv.gapi.parseSSD(detections, g_sz, 0.5, False, False) - comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) - pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(bboxes)) + pp = cv.gapi.ie.params("net", model_path, weights_path, device_id) - gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), - args=cv.compile_args(cv.gapi.networks(pp))) + gapi_boxes = comp.apply(cv.gin(img.astype(np.float32)), + args=cv.gapi.compile_args(cv.gapi.networks(pp))) - # Comparison - self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), - np.array(gapi_boxes).flatten(), - cv.NORM_INF)) + # Comparison + self.assertEqual(0.0, cv.norm(np.array(dnn_boxes).flatten(), + np.array(gapi_boxes).flatten(), + cv.NORM_INF)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/misc/python/test/test_gapi_render.py b/modules/gapi/misc/python/test/test_gapi_render.py new file mode 100644 index 0000000000..70601a72e5 --- /dev/null +++ b/modules/gapi/misc/python/test/test_gapi_render.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python + +import numpy as np +import cv2 as cv +import os +import sys +import unittest + +from tests_common import NewOpenCVTests + +try: + + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + # FIXME: FText isn't supported yet. + class gapi_render_test(NewOpenCVTests): + def __init__(self, *args): + super().__init__(*args) + + self.size = (300, 300, 3) + + # Rect + self.rect = (30, 30, 50, 50) + self.rcolor = (0, 255, 0) + self.rlt = cv.LINE_4 + self.rthick = 2 + self.rshift = 3 + + # Text + self.text = 'Hello, world!' + self.org = (100, 100) + self.ff = cv.FONT_HERSHEY_SIMPLEX + self.fs = 1.0 + self.tthick = 2 + self.tlt = cv.LINE_8 + self.tcolor = (255, 255, 255) + self.blo = False + + # Circle + self.center = (200, 200) + self.radius = 200 + self.ccolor = (255, 255, 0) + self.cthick = 2 + self.clt = cv.LINE_4 + self.cshift = 1 + + # Line + self.pt1 = (50, 50) + self.pt2 = (200, 200) + self.lcolor = (0, 255, 128) + self.lthick = 5 + self.llt = cv.LINE_8 + self.lshift = 2 + + # Poly + self.pts = [(50, 100), (100, 200), (25, 250)] + self.pcolor = (0, 0, 255) + self.pthick = 3 + self.plt = cv.LINE_4 + self.pshift = 1 + + # Image + self.iorg = (150, 150) + img_path = self.find_file('cv/face/david2.jpg', [os.environ.get('OPENCV_TEST_DATA_PATH')]) + self.img = cv.resize(cv.imread(img_path), (50, 50)) + self.alpha = np.full(self.img.shape[:2], 0.8, dtype=np.float32) + + # Mosaic + self.mos = (100, 100, 100, 100) + self.cell_sz = 25 + self.decim = 0 + + # Render primitives + self.prims = [cv.gapi.wip.draw.Rect(self.rect, self.rcolor, self.rthick, self.rlt, self.rshift), + cv.gapi.wip.draw.Text(self.text, self.org, self.ff, self.fs, self.tcolor, self.tthick, self.tlt, self.blo), + cv.gapi.wip.draw.Circle(self.center, self.radius, self.ccolor, self.cthick, self.clt, self.cshift), + cv.gapi.wip.draw.Line(self.pt1, self.pt2, self.lcolor, self.lthick, self.llt, self.lshift), + cv.gapi.wip.draw.Mosaic(self.mos, self.cell_sz, self.decim), + cv.gapi.wip.draw.Image(self.iorg, self.img, self.alpha), + cv.gapi.wip.draw.Poly(self.pts, self.pcolor, self.pthick, self.plt, self.pshift)] + + def cvt_nv12_to_yuv(self, y, uv): + h,w,_ = uv.shape + upsample_uv = cv.resize(uv, (h * 2, w * 2)) + return cv.merge([y, upsample_uv]) + + def cvt_yuv_to_nv12(self, yuv, y_out, uv_out): + chs = cv.split(yuv, [y_out, None, None]) + uv = cv.merge([chs[1], chs[2]]) + uv_out = cv.resize(uv, (uv.shape[0] // 2, uv.shape[1] // 2), dst=uv_out) + return y_out, uv_out + + def cvt_bgr_to_yuv_color(self, bgr): + y = bgr[2] * 0.299000 + bgr[1] * 0.587000 + bgr[0] * 0.114000; + u = bgr[2] * -0.168736 + bgr[1] * -0.331264 + bgr[0] * 0.500000 + 128; + v = bgr[2] * 0.500000 + bgr[1] * -0.418688 + bgr[0] * -0.081312 + 128; + return (y, u, v) + + def blend_img(self, background, org, img, alpha): + x, y = org + h, w, _ = img.shape + roi_img = background[x:x+w, y:y+h, :] + img32f_w = cv.merge([alpha] * 3).astype(np.float32) + roi32f_w = np.full(roi_img.shape, 1.0, dtype=np.float32) + roi32f_w -= img32f_w + img32f = (img / 255).astype(np.float32) + roi32f = (roi_img / 255).astype(np.float32) + cv.multiply(img32f, img32f_w, dst=img32f) + cv.multiply(roi32f, roi32f_w, dst=roi32f) + roi32f += img32f + roi_img[...] = np.round(roi32f * 255) + + # This is quite naive implementations used as a simple reference + # doesn't consider corner cases. + def draw_mosaic(self, img, mos, cell_sz, decim): + x,y,w,h = mos + mosaic_area = img[x:x+w, y:y+h, :] + for i in range(0, mosaic_area.shape[0], cell_sz): + for j in range(0, mosaic_area.shape[1], cell_sz): + cell_roi = mosaic_area[j:j+cell_sz, i:i+cell_sz, :] + s0, s1, s2 = cv.mean(cell_roi)[:3] + mosaic_area[j:j+cell_sz, i:i+cell_sz] = (round(s0), round(s1), round(s2)) + + def render_primitives_bgr_ref(self, img): + cv.rectangle(img, self.rect, self.rcolor, self.rthick, self.rlt, self.rshift) + cv.putText(img, self.text, self.org, self.ff, self.fs, self.tcolor, self.tthick, self.tlt, self.blo) + cv.circle(img, self.center, self.radius, self.ccolor, self.cthick, self.clt, self.cshift) + cv.line(img, self.pt1, self.pt2, self.lcolor, self.lthick, self.llt, self.lshift) + cv.fillPoly(img, np.expand_dims(np.array([self.pts]), axis=0), self.pcolor, self.plt, self.pshift) + self.draw_mosaic(img, self.mos, self.cell_sz, self.decim) + self.blend_img(img, self.iorg, self.img, self.alpha) + + def render_primitives_nv12_ref(self, y_plane, uv_plane): + yuv = self.cvt_nv12_to_yuv(y_plane, uv_plane) + cv.rectangle(yuv, self.rect, self.cvt_bgr_to_yuv_color(self.rcolor), self.rthick, self.rlt, self.rshift) + cv.putText(yuv, self.text, self.org, self.ff, self.fs, self.cvt_bgr_to_yuv_color(self.tcolor), self.tthick, self.tlt, self.blo) + cv.circle(yuv, self.center, self.radius, self.cvt_bgr_to_yuv_color(self.ccolor), self.cthick, self.clt, self.cshift) + cv.line(yuv, self.pt1, self.pt2, self.cvt_bgr_to_yuv_color(self.lcolor), self.lthick, self.llt, self.lshift) + cv.fillPoly(yuv, np.expand_dims(np.array([self.pts]), axis=0), self.cvt_bgr_to_yuv_color(self.pcolor), self.plt, self.pshift) + self.draw_mosaic(yuv, self.mos, self.cell_sz, self.decim) + self.blend_img(yuv, self.iorg, cv.cvtColor(self.img, cv.COLOR_BGR2YUV), self.alpha) + self.cvt_yuv_to_nv12(yuv, y_plane, uv_plane) + + def test_render_primitives_on_bgr_graph(self): + expected = np.zeros(self.size, dtype=np.uint8) + actual = np.array(expected, copy=True) + + # OpenCV + self.render_primitives_bgr_ref(expected) + + # G-API + g_in = cv.GMat() + g_prims = cv.GArray.Prim() + g_out = cv.gapi.wip.draw.render3ch(g_in, g_prims) + + + comp = cv.GComputation(cv.GIn(g_in, g_prims), cv.GOut(g_out)) + actual = comp.apply(cv.gin(actual, self.prims)) + + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + def test_render_primitives_on_bgr_function(self): + expected = np.zeros(self.size, dtype=np.uint8) + actual = np.array(expected, copy=True) + + # OpenCV + self.render_primitives_bgr_ref(expected) + + # G-API + cv.gapi.wip.draw.render(actual, self.prims) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + def test_render_primitives_on_nv12_graph(self): + y_expected = np.zeros((self.size[0], self.size[1], 1), dtype=np.uint8) + uv_expected = np.zeros((self.size[0] // 2, self.size[1] // 2, 2), dtype=np.uint8) + + y_actual = np.array(y_expected, copy=True) + uv_actual = np.array(uv_expected, copy=True) + + # OpenCV + self.render_primitives_nv12_ref(y_expected, uv_expected) + + # G-API + g_y = cv.GMat() + g_uv = cv.GMat() + g_prims = cv.GArray.Prim() + g_out_y, g_out_uv = cv.gapi.wip.draw.renderNV12(g_y, g_uv, g_prims) + + comp = cv.GComputation(cv.GIn(g_y, g_uv, g_prims), cv.GOut(g_out_y, g_out_uv)) + y_actual, uv_actual = comp.apply(cv.gin(y_actual, uv_actual, self.prims)) + + self.assertEqual(0.0, cv.norm(y_expected, y_actual, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(uv_expected, uv_actual, cv.NORM_INF)) + + def test_render_primitives_on_nv12_function(self): + y_expected = np.zeros((self.size[0], self.size[1], 1), dtype=np.uint8) + uv_expected = np.zeros((self.size[0] // 2, self.size[1] // 2, 2), dtype=np.uint8) + + y_actual = np.array(y_expected, copy=True) + uv_actual = np.array(uv_expected, copy=True) + + # OpenCV + self.render_primitives_nv12_ref(y_expected, uv_expected) + + # G-API + cv.gapi.wip.draw.render(y_actual, uv_actual, self.prims) + + self.assertEqual(0.0, cv.norm(y_expected, y_actual, cv.NORM_INF)) + self.assertEqual(0.0, cv.norm(uv_expected, uv_actual, cv.NORM_INF)) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py index 2f921901db..34a07b535b 100644 --- a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py +++ b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py @@ -225,7 +225,7 @@ try: comp = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(g_out)) pkg = cv.gapi.kernels(GAddImpl) - actual = comp.apply(cv.gin(in_mat1, in_mat2), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat1, in_mat2), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -245,7 +245,7 @@ try: comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_ch1, g_ch2, g_ch3)) pkg = cv.gapi.kernels(GSplit3Impl) - ch1, ch2, ch3 = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + ch1, ch2, ch3 = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(in_ch1, ch1, cv.NORM_INF)) self.assertEqual(0.0, cv.norm(in_ch2, ch2, cv.NORM_INF)) @@ -266,7 +266,7 @@ try: comp = cv.GComputation(g_in, g_out) pkg = cv.gapi.kernels(GMeanImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) # Comparison self.assertEqual(expected, actual) @@ -287,7 +287,7 @@ try: comp = cv.GComputation(cv.GIn(g_in, g_sc), cv.GOut(g_out)) pkg = cv.gapi.kernels(GAddCImpl) - actual = comp.apply(cv.gin(in_mat, sc), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat, sc), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -305,7 +305,7 @@ try: comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_sz)) pkg = cv.gapi.kernels(GSizeImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -322,7 +322,7 @@ try: comp = cv.GComputation(cv.GIn(g_r), cv.GOut(g_sz)) pkg = cv.gapi.kernels(GSizeRImpl) - actual = comp.apply(cv.gin(roi), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(roi), args=cv.gapi.compile_args(pkg)) # cv.norm works with tuples ? self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -340,7 +340,7 @@ try: comp = cv.GComputation(cv.GIn(g_pts), cv.GOut(g_br)) pkg = cv.gapi.kernels(GBoundingRectImpl) - actual = comp.apply(cv.gin(points), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(points), args=cv.gapi.compile_args(pkg)) # cv.norm works with tuples ? self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) @@ -371,7 +371,7 @@ try: comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) pkg = cv.gapi.kernels(GGoodFeaturesImpl) - actual = comp.apply(cv.gin(in_mat), args=cv.compile_args(pkg)) + actual = comp.apply(cv.gin(in_mat), args=cv.gapi.compile_args(pkg)) # NB: OpenCV & G-API have different output types. # OpenCV - numpy array with shape (num_points, 1, 2) @@ -453,10 +453,10 @@ try: g_in = cv.GArray.Int() comp = cv.GComputation(cv.GIn(g_in), cv.GOut(GSum.on(g_in))) - s = comp.apply(cv.gin([1, 2, 3, 4]), args=cv.compile_args(cv.gapi.kernels(GSumImpl))) + s = comp.apply(cv.gin([1, 2, 3, 4]), args=cv.gapi.compile_args(cv.gapi.kernels(GSumImpl))) self.assertEqual(10, s) - s = comp.apply(cv.gin([1, 2, 8, 7]), args=cv.compile_args(cv.gapi.kernels(GSumImpl))) + s = comp.apply(cv.gin([1, 2, 8, 7]), args=cv.gapi.compile_args(cv.gapi.kernels(GSumImpl))) self.assertEqual(18, s) self.assertEqual(18, GSumImpl.last_result) @@ -488,13 +488,13 @@ try: 'tuple': (42, 42) } - out = comp.apply(cv.gin(table, 'int'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'int'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual(42, out) - out = comp.apply(cv.gin(table, 'str'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'str'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual('hello, world!', out) - out = comp.apply(cv.gin(table, 'tuple'), args=cv.compile_args(cv.gapi.kernels(GLookUpImpl))) + out = comp.apply(cv.gin(table, 'tuple'), args=cv.gapi.compile_args(cv.gapi.kernels(GLookUpImpl))) self.assertEqual((42, 42), out) @@ -517,11 +517,11 @@ try: comp = cv.GComputation(cv.GIn(g_arr0, g_arr1), cv.GOut(g_out)) - arr0 = [(2, 2), 2.0] - arr1 = [3, 'str'] + arr0 = ((2, 2), 2.0) + arr1 = (3, 'str') out = comp.apply(cv.gin(arr0, arr1), - args=cv.compile_args(cv.gapi.kernels(GConcatImpl))) + args=cv.gapi.compile_args(cv.gapi.kernels(GConcatImpl))) self.assertEqual(arr0 + arr1, out) @@ -550,7 +550,7 @@ try: img1 = np.array([1, 2, 3]) with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) @@ -577,7 +577,7 @@ try: img1 = np.array([1, 2, 3]) with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) @@ -607,7 +607,7 @@ try: # FIXME: Cause Bad variant access. # Need to provide more descriptive error messsage. with self.assertRaises(Exception): comp.apply(cv.gin(img0, img1), - args=cv.compile_args( + args=cv.gapi.compile_args( cv.gapi.kernels(GAddImpl))) def test_pipeline_with_custom_kernels(self): @@ -657,7 +657,7 @@ try: g_mean = cv.gapi.mean(g_transposed) comp = cv.GComputation(cv.GIn(g_bgr), cv.GOut(g_mean)) - actual = comp.apply(cv.gin(img), args=cv.compile_args( + actual = comp.apply(cv.gin(img), args=cv.gapi.compile_args( cv.gapi.kernels(GResizeImpl, GTransposeImpl))) self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) diff --git a/modules/gapi/misc/python/test/test_gapi_streaming.py b/modules/gapi/misc/python/test/test_gapi_streaming.py index 5356abc76a..d7914c5157 100644 --- a/modules/gapi/misc/python/test/test_gapi_streaming.py +++ b/modules/gapi/misc/python/test/test_gapi_streaming.py @@ -3,201 +3,367 @@ import numpy as np import cv2 as cv import os +import sys +import unittest +import time from tests_common import NewOpenCVTests -class test_gapi_streaming(NewOpenCVTests): - def test_image_input(self): - sz = (1280, 720) - in_mat = np.random.randint(0, 100, sz).astype(np.uint8) - - # OpenCV - expected = cv.medianBlur(in_mat, 3) - - # G-API - g_in = cv.GMat() - g_out = cv.gapi.medianBlur(g_in, 3) - c = cv.GComputation(g_in, g_out) - ccomp = c.compileStreaming(cv.descr_of(in_mat)) - ccomp.setSource(cv.gin(in_mat)) - ccomp.start() - - _, actual = ccomp.pull() - - # Assert - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) +try: + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') - def test_video_input(self): - ksize = 3 - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + @cv.gapi.op('custom.delay', in_types=[cv.GMat], out_types=[cv.GMat]) + class GDelay: + """Delay for 10 ms.""" - # OpenCV - cap = cv.VideoCapture(path) - - # G-API - g_in = cv.GMat() - g_out = cv.gapi.medianBlur(g_in, ksize) - c = cv.GComputation(g_in, g_out) - - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() - - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, expected = cap.read() - has_actual, actual = ccomp.pull() - - self.assertEqual(has_expected, has_actual) - - if not has_actual: - break - - self.assertEqual(0.0, cv.norm(cv.medianBlur(expected, ksize), actual, cv.NORM_INF)) - - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; + @staticmethod + def outMeta(desc): + return desc - def test_video_split3(self): - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + @cv.gapi.kernel(GDelay) + class GDelayImpl: + """Implementation for GDelay operation.""" - # OpenCV - cap = cv.VideoCapture(path) - - # G-API - g_in = cv.GMat() - b, g, r = cv.gapi.split3(g_in) - c = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) - - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() - - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() - - self.assertEqual(has_expected, has_actual) - - if not has_actual: - break - - expected = cv.split(frame) - for e, a in zip(expected, actual): - self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF)) - - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; + @staticmethod + def run(img): + time.sleep(0.01) + return img - def test_video_add(self): - sz = (576, 768, 3) - in_mat = np.random.randint(0, 100, sz).astype(np.uint8) + class test_gapi_streaming(NewOpenCVTests): - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - - # OpenCV - cap = cv.VideoCapture(path) - - # G-API - g_in1 = cv.GMat() - g_in2 = cv.GMat() - out = cv.gapi.add(g_in1, g_in2) - c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(out)) - - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(cv.gin(source, in_mat)) - ccomp.start() - - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() - - self.assertEqual(has_expected, has_actual) - - if not has_actual: - break - - expected = cv.add(frame, in_mat) - self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) - - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; - - - def test_video_good_features_to_track(self): - path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) - - # NB: goodFeaturesToTrack configuration - max_corners = 50 - quality_lvl = 0.01 - min_distance = 10 - block_sz = 3 - use_harris_detector = True - k = 0.04 - mask = None - - # OpenCV - cap = cv.VideoCapture(path) - - # G-API - g_in = cv.GMat() - g_gray = cv.gapi.RGB2Gray(g_in) - g_out = cv.gapi.goodFeaturesToTrack(g_gray, max_corners, quality_lvl, - min_distance, mask, block_sz, use_harris_detector, k) - - c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) - - ccomp = c.compileStreaming() - source = cv.gapi.wip.make_capture_src(path) - ccomp.setSource(source) - ccomp.start() - - # Assert - max_num_frames = 10 - proc_num_frames = 0 - while cap.isOpened(): - has_expected, frame = cap.read() - has_actual, actual = ccomp.pull() - - self.assertEqual(has_expected, has_actual) - - if not has_actual: - break + def test_image_input(self): + sz = (1280, 720) + in_mat = np.random.randint(0, 100, sz).astype(np.uint8) # OpenCV - frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) - expected = cv.goodFeaturesToTrack(frame, max_corners, quality_lvl, - min_distance, mask=mask, - blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) - for e, a in zip(expected, actual): - # NB: OpenCV & G-API have different output shapes: - # OpenCV - (num_points, 1, 2) - # G-API - (num_points, 2) - self.assertEqual(0.0, cv.norm(e.flatten(), - np.array(a, np.float32).flatten(), - cv.NORM_INF)) + expected = cv.medianBlur(in_mat, 3) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.medianBlur(g_in, 3) + c = cv.GComputation(g_in, g_out) + ccomp = c.compileStreaming(cv.gapi.descr_of(in_mat)) + ccomp.setSource(cv.gin(in_mat)) + ccomp.start() + + _, actual = ccomp.pull() + + # Assert + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + + def test_video_input(self): + ksize = 3 + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + g_out = cv.gapi.medianBlur(g_in, ksize) + c = cv.GComputation(g_in, g_out) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, expected = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + self.assertEqual(0.0, cv.norm(cv.medianBlur(expected, ksize), actual, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break + + + def test_video_split3(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + b, g, r = cv.gapi.split3(g_in) + c = cv.GComputation(cv.GIn(g_in), cv.GOut(b, g, r)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + expected = cv.split(frame) + for e, a in zip(expected, actual): + self.assertEqual(0.0, cv.norm(e, a, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break + + + def test_video_add(self): + sz = (576, 768, 3) + in_mat = np.random.randint(0, 100, sz).astype(np.uint8) + + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in1 = cv.GMat() + g_in2 = cv.GMat() + out = cv.gapi.add(g_in1, g_in2) + c = cv.GComputation(cv.GIn(g_in1, g_in2), cv.GOut(out)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source, in_mat)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + expected = cv.add(frame, in_mat) + self.assertEqual(0.0, cv.norm(expected, actual, cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break + + + def test_video_good_features_to_track(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # NB: goodFeaturesToTrack configuration + max_corners = 50 + quality_lvl = 0.01 + min_distance = 10 + block_sz = 3 + use_harris_detector = True + k = 0.04 + mask = None + + # OpenCV + cap = cv.VideoCapture(path) + + # G-API + g_in = cv.GMat() + g_gray = cv.gapi.RGB2Gray(g_in) + g_out = cv.gapi.goodFeaturesToTrack(g_gray, max_corners, quality_lvl, + min_distance, mask, block_sz, use_harris_detector, k) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + while cap.isOpened(): + has_expected, frame = cap.read() + has_actual, actual = ccomp.pull() + + self.assertEqual(has_expected, has_actual) + + if not has_actual: + break + + # OpenCV + frame = cv.cvtColor(frame, cv.COLOR_RGB2GRAY) + expected = cv.goodFeaturesToTrack(frame, max_corners, quality_lvl, + min_distance, mask=mask, + blockSize=block_sz, useHarrisDetector=use_harris_detector, k=k) + for e, a in zip(expected, actual): + # NB: OpenCV & G-API have different output shapes: + # OpenCV - (num_points, 1, 2) + # G-API - (num_points, 2) + self.assertEqual(0.0, cv.norm(e.flatten(), + np.array(a, np.float32).flatten(), + cv.NORM_INF)) + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + break + + + def test_gapi_streaming_meta(self): + ksize = 3 + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # G-API + g_in = cv.GMat() + g_ts = cv.gapi.streaming.timestamp(g_in) + g_seqno = cv.gapi.streaming.seqNo(g_in) + g_seqid = cv.gapi.streaming.seq_id(g_in) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_ts, g_seqno, g_seqid)) + + ccomp = c.compileStreaming() + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + curr_frame_number = 0 + while True: + has_frame, (ts, seqno, seqid) = ccomp.pull() + + if not has_frame: + break + + self.assertEqual(curr_frame_number, seqno) + self.assertEqual(curr_frame_number, seqid) + + curr_frame_number += 1 + if curr_frame_number == max_num_frames: + break + + + def test_desync(self): + path = self.find_file('cv/video/768x576.avi', [os.environ['OPENCV_TEST_DATA_PATH']]) + + # G-API + g_in = cv.GMat() + g_out1 = cv.gapi.copy(g_in) + des = cv.gapi.streaming.desync(g_in) + g_out2 = GDelay.on(des) + + c = cv.GComputation(cv.GIn(g_in), cv.GOut(g_out1, g_out2)) + + kernels = cv.gapi.kernels(GDelayImpl) + ccomp = c.compileStreaming(args=cv.gapi.compile_args(kernels)) + source = cv.gapi.wip.make_capture_src(path) + ccomp.setSource(cv.gin(source)) + ccomp.start() + + # Assert + max_num_frames = 10 + proc_num_frames = 0 + + out_counter = 0 + desync_out_counter = 0 + none_counter = 0 + while True: + has_frame, (out1, out2) = ccomp.pull() + if not has_frame: + break + + if not out1 is None: + out_counter += 1 + if not out2 is None: + desync_out_counter += 1 + else: + none_counter += 1 + + proc_num_frames += 1 + if proc_num_frames == max_num_frames: + ccomp.stop() + break + + self.assertLess(0, proc_num_frames) + self.assertLess(desync_out_counter, out_counter) + self.assertLess(0, none_counter) + + + def test_compile_streaming_empty(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + comp.compileStreaming() + + + def test_compile_streaming_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + comp.compileStreaming(cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + def test_compile_streaming_descr_of(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming(cv.gapi.descr_of(img)) + + + def test_compile_streaming_descr_of_and_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming(cv.gapi.descr_of(img), + cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + def test_compile_streaming_meta(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming([cv.GMatDesc(cv.CV_8U, 3, (300, 300))]) + + + def test_compile_streaming_meta_and_args(self): + g_in = cv.GMat() + comp = cv.GComputation(g_in, cv.gapi.medianBlur(g_in, 3)) + img = np.zeros((3,300,300), dtype=np.float32) + comp.compileStreaming([cv.GMatDesc(cv.CV_8U, 3, (300, 300))], + cv.gapi.compile_args(cv.gapi.streaming.queue_capacity(1))) + + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass - proc_num_frames += 1 - if proc_num_frames == max_num_frames: - break; if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/gapi/misc/python/test/test_gapi_types.py b/modules/gapi/misc/python/test/test_gapi_types.py index 0f3b194a2f..dde554f5e1 100644 --- a/modules/gapi/misc/python/test/test_gapi_types.py +++ b/modules/gapi/misc/python/test/test_gapi_types.py @@ -3,29 +3,51 @@ import numpy as np import cv2 as cv import os +import sys +import unittest from tests_common import NewOpenCVTests -class gapi_types_test(NewOpenCVTests): - def test_garray_type(self): - types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, - cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , - cv.gapi.CV_RECT , cv.gapi.CV_SCALAR, cv.gapi.CV_MAT , cv.gapi.CV_GMAT] +try: - for t in types: - g_array = cv.GArrayT(t) - self.assertEqual(t, g_array.type()) + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + class gapi_types_test(NewOpenCVTests): + + def test_garray_type(self): + types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, + cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , + cv.gapi.CV_RECT , cv.gapi.CV_SCALAR, cv.gapi.CV_MAT , cv.gapi.CV_GMAT] + + for t in types: + g_array = cv.GArrayT(t) + self.assertEqual(t, g_array.type()) - def test_gopaque_type(self): - types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, - cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , - cv.gapi.CV_RECT] + def test_gopaque_type(self): + types = [cv.gapi.CV_BOOL , cv.gapi.CV_INT , cv.gapi.CV_DOUBLE , cv.gapi.CV_FLOAT, + cv.gapi.CV_STRING, cv.gapi.CV_POINT , cv.gapi.CV_POINT2F, cv.gapi.CV_SIZE , + cv.gapi.CV_RECT] - for t in types: - g_opaque = cv.GOpaqueT(t) - self.assertEqual(t, g_opaque.type()) + for t in types: + g_opaque = cv.GOpaqueT(t) + self.assertEqual(t, g_opaque.type()) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass if __name__ == '__main__': diff --git a/modules/gapi/samples/api_ref_snippets.cpp b/modules/gapi/samples/api_ref_snippets.cpp index 9c3e73a17c..0abcab89b3 100644 --- a/modules/gapi/samples/api_ref_snippets.cpp +++ b/modules/gapi/samples/api_ref_snippets.cpp @@ -4,11 +4,23 @@ #include #include +#include +#include +#include + #include #include #include +static void gscalar_example() +{ + //! [gscalar_implicit] + cv::GMat a; + cv::GMat b = a + 1; + //! [gscalar_implicit] +} + static void typed_example() { const cv::Size sz(32, 32); @@ -47,6 +59,120 @@ static void typed_example() //! [Typed_Example] } +static void bind_serialization_example() +{ + // ! [bind after deserialization] + cv::GCompiled compd; + std::vector bytes; + auto graph = cv::gapi::deserialize(bytes); + auto meta = cv::gapi::deserialize(bytes); + + compd = graph.compile(std::move(meta), cv::compile_args()); + auto in_args = cv::gapi::deserialize(bytes); + auto out_args = cv::gapi::deserialize(bytes); + compd(std::move(in_args), cv::gapi::bind(out_args)); + // ! [bind after deserialization] +} + +static void bind_deserialization_example() +{ + // ! [bind before serialization] + std::vector graph_outs; + cv::GRunArgs out_args; + + for (auto &&out : graph_outs) { + out_args.emplace_back(cv::gapi::bind(out)); + } + const auto sargsout = cv::gapi::serialize(out_args); + // ! [bind before serialization] +} + +struct SimpleCustomType { + bool val; + bool operator==(const SimpleCustomType& other) const { + return val == other.val; + } +}; + +struct SimpleCustomType2 { + int val; + std::string name; + std::vector vec; + std::map mmap; + bool operator==(const SimpleCustomType2& other) const { + return val == other.val && name == other.name && + vec == other.vec && mmap == other.mmap; + } +}; + +// ! [S11N usage] +namespace cv { +namespace gapi { +namespace s11n { +namespace detail { +template<> struct S11N { + static void serialize(IOStream &os, const SimpleCustomType &p) { + os << p.val; + } + static SimpleCustomType deserialize(IIStream &is) { + SimpleCustomType p; + is >> p.val; + return p; + } +}; + +template<> struct S11N { + static void serialize(IOStream &os, const SimpleCustomType2 &p) { + os << p.val << p.name << p.vec << p.mmap; + } + static SimpleCustomType2 deserialize(IIStream &is) { + SimpleCustomType2 p; + is >> p.val >> p.name >> p.vec >> p.mmap; + return p; + } +}; +} // namespace detail +} // namespace s11n +} // namespace gapi +} // namespace cv +// ! [S11N usage] + +namespace cv { +namespace detail { +template<> struct CompileArgTag { + static const char* tag() { + return "org.opencv.test.simple_custom_type"; + } +}; + +template<> struct CompileArgTag { + static const char* tag() { + return "org.opencv.test.simple_custom_type_2"; + } +}; +} // namespace detail +} // namespace cv + +static void s11n_example() +{ + SimpleCustomType customVar1 { false }; + SimpleCustomType2 customVar2 { 1248, "World", {1280, 720, 640, 480}, + { {5, 32434142342}, {7, 34242432} } }; + + std::vector sArgs = cv::gapi::serialize( + cv::compile_args(customVar1, customVar2)); + + cv::GCompileArgs dArgs = cv::gapi::deserialize(sArgs); + + SimpleCustomType dCustomVar1 = cv::gapi::getCompileArg(dArgs).value(); + SimpleCustomType2 dCustomVar2 = cv::gapi::getCompileArg(dArgs).value(); + + (void) dCustomVar1; + (void) dCustomVar2; +} + G_TYPED_KERNEL(IAdd, , "test.custom.add") { static cv::GMatDesc outMeta(const cv::GMatDesc &in) { return in; } }; @@ -116,7 +242,12 @@ int main(int argc, char *argv[]) >(); //! [kernels_snippet] - // Just call typed example with no input/output + // Just call typed example with no input/output - avoid warnings about + // unused functions typed_example(); + gscalar_example(); + bind_serialization_example(); + bind_deserialization_example(); + s11n_example(); return 0; } diff --git a/modules/gapi/samples/face_detection_mtcnn.cpp b/modules/gapi/samples/face_detection_mtcnn.cpp index c437bdbba4..50cb666a90 100644 --- a/modules/gapi/samples/face_detection_mtcnn.cpp +++ b/modules/gapi/samples/face_detection_mtcnn.cpp @@ -589,30 +589,23 @@ int main(int argc, char* argv[]) { //Preprocessing BGR2RGB + transpose (NCWH is expected instead of NCHW) cv::GMat in_original; cv::GMat in_originalRGB = cv::gapi::BGR2RGB(in_original); + cv::GMat in_transposedRGB = cv::gapi::transpose(in_originalRGB); cv::GOpaque in_sz = cv::gapi::streaming::size(in_original); - cv::GMat in_resized[MAX_PYRAMID_LEVELS]; - cv::GMat in_transposed[MAX_PYRAMID_LEVELS]; cv::GMat regressions[MAX_PYRAMID_LEVELS]; cv::GMat scores[MAX_PYRAMID_LEVELS]; cv::GArray nms_p_faces[MAX_PYRAMID_LEVELS]; cv::GArray total_faces[MAX_PYRAMID_LEVELS]; - cv::GArray faces_init(std::vector{}); //The very first PNet pyramid layer to init total_faces[0] - in_resized[0] = cv::gapi::resize(in_originalRGB, level_size[0]); - in_transposed[0] = cv::gapi::transpose(in_resized[0]); - std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposed[0], get_pnet_level_name(level_size[0])); + std::tie(regressions[0], scores[0]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[0])); cv::GArray faces0 = custom::BuildFaces::on(scores[0], regressions[0], static_cast(scales[0]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares = custom::ApplyRegression::on(faces0, true); cv::GArray final_faces_pnet0 = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares); - nms_p_faces[0] = custom::RunNMS::on(final_faces_pnet0, 0.5f, false); - total_faces[0] = custom::AccumulatePyramidOutputs::on(faces_init, nms_p_faces[0]); + total_faces[0] = custom::RunNMS::on(final_faces_pnet0, 0.5f, false); //The rest PNet pyramid layers to accumlate all layers result in total_faces[PYRAMID_LEVELS - 1]] for (int i = 1; i < pyramid_levels; ++i) { - in_resized[i] = cv::gapi::resize(in_originalRGB, level_size[i]); - in_transposed[i] = cv::gapi::transpose(in_resized[i]); - std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposed[i], get_pnet_level_name(level_size[i])); + std::tie(regressions[i], scores[i]) = run_mtcnn_p(in_transposedRGB, get_pnet_level_name(level_size[i])); cv::GArray faces = custom::BuildFaces::on(scores[i], regressions[i], static_cast(scales[i]), conf_thresh_p); cv::GArray final_p_faces_for_bb2squares_i = custom::ApplyRegression::on(faces, true); cv::GArray final_faces_pnet_i = custom::BBoxesToSquares::on(final_p_faces_for_bb2squares_i); @@ -626,8 +619,7 @@ int main(int argc, char* argv[]) { //Refinement part of MTCNN graph cv::GArray faces_roi_pnet = custom::R_O_NetPreProcGetROIs::on(final_faces_pnet, in_sz); cv::GArray regressionsRNet, scoresRNet; - cv::GMat in_originalRGB_transposed = cv::gapi::transpose(in_originalRGB); - std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer(faces_roi_pnet, in_originalRGB_transposed); + std::tie(regressionsRNet, scoresRNet) = cv::gapi::infer(faces_roi_pnet, in_transposedRGB); //Refinement post-processing cv::GArray rnet_post_proc_faces = custom::RNetPostProc::on(final_faces_pnet, scoresRNet, regressionsRNet, conf_thresh_r); @@ -638,7 +630,7 @@ int main(int argc, char* argv[]) { //Output part of MTCNN graph cv::GArray faces_roi_rnet = custom::R_O_NetPreProcGetROIs::on(final_faces_rnet, in_sz); cv::GArray regressionsONet, scoresONet, landmarksONet; - std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer(faces_roi_rnet, in_originalRGB_transposed); + std::tie(regressionsONet, landmarksONet, scoresONet) = cv::gapi::infer(faces_roi_rnet, in_transposedRGB); //Output post-processing cv::GArray onet_post_proc_faces = custom::ONetPostProc::on(final_faces_rnet, scoresONet, regressionsONet, landmarksONet, conf_thresh_o); diff --git a/modules/gapi/samples/gaze_estimation.cpp b/modules/gapi/samples/gaze_estimation.cpp index 6396d500c1..96b2ebf163 100644 --- a/modules/gapi/samples/gaze_estimation.cpp +++ b/modules/gapi/samples/gaze_estimation.cpp @@ -16,13 +16,13 @@ const std::string keys = "{ h help | | Print this help message }" "{ input | | Path to the input video file }" "{ facem | face-detection-retail-0005.xml | Path to OpenVINO face detection model (.xml) }" - "{ faced | CPU | Target device for the face detection (e.g. CPU, GPU, VPU, ...) }" + "{ faced | CPU | Target device for the face detection (e.g. CPU, GPU, ...) }" "{ landm | facial-landmarks-35-adas-0002.xml | Path to OpenVINO landmarks detector model (.xml) }" - "{ landd | CPU | Target device for the landmarks detector (e.g. CPU, GPU, VPU, ...) }" + "{ landd | CPU | Target device for the landmarks detector (e.g. CPU, GPU, ...) }" "{ headm | head-pose-estimation-adas-0001.xml | Path to OpenVINO head pose estimation model (.xml) }" - "{ headd | CPU | Target device for the head pose estimation inference (e.g. CPU, GPU, VPU, ...) }" + "{ headd | CPU | Target device for the head pose estimation inference (e.g. CPU, GPU, ...) }" "{ gazem | gaze-estimation-adas-0002.xml | Path to OpenVINO gaze vector estimaiton model (.xml) }" - "{ gazed | CPU | Target device for the gaze vector estimation inference (e.g. CPU, GPU, VPU, ...) }" + "{ gazed | CPU | Target device for the gaze vector estimation inference (e.g. CPU, GPU, ...) }" ; namespace { @@ -338,7 +338,7 @@ int main(int argc, char *argv[]) cv::GMat in; cv::GMat faces = cv::gapi::infer(in); - cv::GOpaque sz = custom::Size::on(in); // FIXME + cv::GOpaque sz = cv::gapi::streaming::size(in); cv::GArray faces_rc = custom::ParseSSD::on(faces, sz, true); cv::GArray angles_y, angles_p, angles_r; std::tie(angles_y, angles_p, angles_r) = cv::gapi::infer(faces_rc, in); diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp new file mode 100644 index 0000000000..9a3abdf392 --- /dev/null +++ b/modules/gapi/samples/onevpl_infer_single_roi.cpp @@ -0,0 +1,293 @@ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include // CommandLineParser + +const std::string about = + "This is an OpenCV-based version of oneVPLSource decoder example"; +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input demultiplexed video file }" + "{ output | | Path to the output RAW video file. Use .avi extension }" + "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ cfg_params | :;: | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }"; + + +namespace { +std::string get_weights_path(const std::string &model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + CV_Assert(sz > EXT_LEN); + + auto ext = model_path.substr(sz - EXT_LEN); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ + return static_cast(std::tolower(c)); + }); + CV_Assert(ext == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} +} // anonymous namespace + +namespace custom { +G_API_NET(FaceDetector, , "face-detector"); + +using GDetections = cv::GArray; +using GRect = cv::GOpaque; +using GSize = cv::GOpaque; +using GPrims = cv::GArray; + +G_API_OP(LocateROI, , "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { + return cv::empty_gopaque_desc(); + } +}; + +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +G_API_OP(BBoxes, , "sample.custom.b-boxes") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { + // This is the place where we can run extra analytics + // on the input image frame and select the ROI (region + // of interest) where we want to detect our objects (or + // run any other inference). + // + // Currently it doesn't do anything intelligent, + // but only crops the input image to square (this is + // the most convenient aspect ratio for detectors to use) + + static void run(const cv::Size& in_size, cv::Rect &out_rect) { + + // Identify the central point & square size (- some padding) + const auto center = cv::Point{in_size.width/2, in_size.height/2}; + auto sqside = std::min(in_size.width, in_size.height); + + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; + } +}; + +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Rect &in_roi, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + CV_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + CV_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Size up_roi = in_roi.size(); + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + // taking the ROI into account + cv::Rect rc; + rc.x = static_cast(rc_left * up_roi.width); + rc.y = static_cast(rc_top * up_roi.height); + rc.width = static_cast(rc_right * up_roi.width) - rc.x; + rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; + rc.x += in_roi.x; + rc.y += in_roi.y; + out_objects.emplace_back(rc & surface); + } + } +}; + +GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { + // This kernel converts the rectangles into G-API's + // rendering primitives + static void run(const std::vector &in_face_rcs, + const cv::Rect &in_roi, + std::vector &out_prims) { + out_prims.clear(); + const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Rect(rc, clr, 2); + }; + out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan + for (auto &&rc : in_face_rcs) { + out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green + } + } +}; + +} // namespace custom + +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line); +} + +int main(int argc, char *argv[]) { + + cv::CommandLineParser cmd(argc, argv, keys); + cmd.about(about); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + // get file name + std::string file_path = cmd.get("input"); + const std::string output = cmd.get("output"); + const auto face_model_path = cmd.get("facem"); + + // check ouput file extension + if (!output.empty()) { + auto ext = output.find_last_of("."); + if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) { + std::cerr << "Output file should have *.avi extension for output video" << std::endl; + return -1; + } + } + + // get oneVPL cfg params from cmd + std::stringstream params_list(cmd.get("cfg_params")); + std::vector source_cfgs; + try { + std::string line; + while (std::getline(params_list, line, ';')) { + source_cfgs.push_back(cfg::create_from_string(line)); + } + } catch (const std::exception& ex) { + std::cerr << "Invalid cfg parameter: " << ex.what() << std::endl; + return -1; + } + + auto face_net = cv::gapi::ie::Params { + face_model_path, // path to topology IR + get_weights_path(face_model_path) // path to weights + }; + auto kernels = cv::gapi::kernels + < custom::OCVLocateROI + , custom::OCVParseSSD + , custom::OCVBBoxes>(); + auto networks = cv::gapi::networks(face_net); + + // Create source + cv::Ptr cap; + try { + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); + std::cout << "oneVPL source desription: " << cap->descr_of() << std::endl; + } catch (const std::exception& ex) { + std::cerr << "Cannot create source: " << ex.what() << std::endl; + return -1; + } + + cv::GMetaArg descr = cap->descr_of(); + auto frame_descr = cv::util::get(descr); + + // Now build the graph + cv::GFrame in; + auto size = cv::gapi::streaming::size(in); + auto roi = custom::LocateROI::on(size); + auto blob = cv::gapi::infer(roi, in); + auto rcs = custom::ParseSSD::on(blob, roi, size); + auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, roi)); + auto out = cv::gapi::streaming::BGR(out_frame); + + cv::GStreamingCompiled pipeline; + try { + pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(cv::compile_args(kernels, networks)); + } catch (const std::exception& ex) { + std::cerr << "Exception occured during pipeline construction: " << ex.what() << std::endl; + return -1; + } + // The execution part + + // TODO USE may set pool size from outside and set queue_capacity size, + // compile arg: cv::gapi::streaming::queue_capacity + pipeline.setSource(std::move(cap)); + pipeline.start(); + + int framesCount = 0; + cv::TickMeter t; + cv::VideoWriter writer; + if (!output.empty() && !writer.isOpened()) { + const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; + writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); + CV_Assert(writer.isOpened()); + } + + cv::Mat outMat; + t.start(); + while (pipeline.pull(cv::gout(outMat))) { + cv::imshow("Out", outMat); + cv::waitKey(1); + if (!output.empty()) { + writer << outMat; + } + framesCount++; + } + t.stop(); + std::cout << "Elapsed time: " << t.getTimeSec() << std::endl; + std::cout << "FPS: " << framesCount / t.getTimeSec() << std::endl; + std::cout << "framesCount: " << framesCount << std::endl; + + return 0; +} + + +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) { + using namespace cv::gapi::wip; + + if (line.empty()) { + throw std::runtime_error("Cannot parse CfgParam from emply line"); + } + + std::string::size_type name_endline_pos = line.find(':'); + if (name_endline_pos == std::string::npos) { + throw std::runtime_error("Cannot parse CfgParam from: " + line + + "\nExpected separator \":\""); + } + + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 1); + + return cv::gapi::wip::onevpl::CfgParam::create(name, value); +} +} diff --git a/modules/gapi/samples/semantic_segmentation.cpp b/modules/gapi/samples/semantic_segmentation.cpp index 0a6e7231c4..4cdb14cc5c 100644 --- a/modules/gapi/samples/semantic_segmentation.cpp +++ b/modules/gapi/samples/semantic_segmentation.cpp @@ -47,6 +47,53 @@ std::string get_weights_path(const std::string &model_path) { CV_Assert(ext == ".xml"); return model_path.substr(0u, sz - EXT_LEN) + ".bin"; } + +void classesToColors(const cv::Mat &out_blob, + cv::Mat &mask_img) { + const int H = out_blob.size[0]; + const int W = out_blob.size[1]; + + mask_img.create(H, W, CV_8UC3); + GAPI_Assert(out_blob.type() == CV_8UC1); + const uint8_t* const classes = out_blob.ptr(); + + for (int rowId = 0; rowId < H; ++rowId) { + for (int colId = 0; colId < W; ++colId) { + uint8_t class_id = classes[rowId * W + colId]; + mask_img.at(rowId, colId) = + class_id < colors.size() + ? colors[class_id] + : cv::Vec3b{0, 0, 0}; // NB: sample supports 20 classes + } + } +} + +void probsToClasses(const cv::Mat& probs, cv::Mat& classes) { + const int C = probs.size[1]; + const int H = probs.size[2]; + const int W = probs.size[3]; + + classes.create(H, W, CV_8UC1); + GAPI_Assert(probs.depth() == CV_32F); + float* out_p = reinterpret_cast(probs.data); + uint8_t* classes_p = reinterpret_cast(classes.data); + + for (int h = 0; h < H; ++h) { + for (int w = 0; w < W; ++w) { + double max = 0; + int class_id = 0; + for (int c = 0; c < C; ++c) { + int idx = c * H * W + h * W + w; + if (out_p[idx] > max) { + max = out_p[idx]; + class_id = c; + } + } + classes_p[h * W + w] = static_cast(class_id); + } + } +} + } // anonymous namespace namespace custom { @@ -57,25 +104,21 @@ G_API_OP(PostProcessing, , "sample.custom.post_pro }; GAPI_OCV_KERNEL(OCVPostProcessing, PostProcessing) { - static void run(const cv::Mat &in, const cv::Mat &detected_classes, cv::Mat &out) { - // This kernel constructs output image by class table and colors vector - - // The semantic-segmentation-adas-0001 output a blob with the shape - // [B, C=1, H=1024, W=2048] - const int outHeight = 1024; - const int outWidth = 2048; - cv::Mat maskImg(outHeight, outWidth, CV_8UC3); - const int* const classes = detected_classes.ptr(); - for (int rowId = 0; rowId < outHeight; ++rowId) { - for (int colId = 0; colId < outWidth; ++colId) { - size_t classId = static_cast(classes[rowId * outWidth + colId]); - maskImg.at(rowId, colId) = - classId < colors.size() - ? colors[classId] - : cv::Vec3b{0, 0, 0}; // sample detects 20 classes - } + static void run(const cv::Mat &in, const cv::Mat &out_blob, cv::Mat &out) { + cv::Mat classes; + // NB: If output has more than single plane, it contains probabilities + // otherwise class id. + if (out_blob.size[1] > 1) { + probsToClasses(out_blob, classes); + } else { + out_blob.convertTo(classes, CV_8UC1); + classes = classes.reshape(1, out_blob.size[2]); } - cv::resize(maskImg, out, in.size()); + + cv::Mat mask_img; + classesToColors(classes, mask_img); + + cv::resize(mask_img, out, in.size()); const float blending = 0.3f; out = in * blending + out * (1 - blending); } @@ -104,8 +147,8 @@ int main(int argc, char *argv[]) { // Now build the graph cv::GMat in; - cv::GMat detected_classes = cv::gapi::infer(in); - cv::GMat out = custom::PostProcessing::on(in, detected_classes); + cv::GMat out_blob = cv::gapi::infer(in); + cv::GMat out = custom::PostProcessing::on(in, out_blob); cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) .compileStreaming(cv::compile_args(kernels, networks)); diff --git a/modules/gapi/src/api/ginfer.cpp b/modules/gapi/src/api/ginfer.cpp index e3cc94041c..9db05a43c3 100644 --- a/modules/gapi/src/api/ginfer.cpp +++ b/modules/gapi/src/api/ginfer.cpp @@ -15,6 +15,10 @@ cv::gapi::GNetPackage::GNetPackage(std::initializer_list ii) : networks(ii) { } +cv::gapi::GNetPackage::GNetPackage(std::vector nets) + : networks(nets) { +} + std::vector cv::gapi::GNetPackage::backends() const { std::unordered_set unique_set; for (const auto &nn : networks) unique_set.insert(nn.backend); diff --git a/modules/gapi/src/api/gproto.cpp b/modules/gapi/src/api/gproto.cpp index 94234c9b4d..9b012770ca 100644 --- a/modules/gapi/src/api/gproto.cpp +++ b/modules/gapi/src/api/gproto.cpp @@ -14,7 +14,6 @@ #include "api/gorigin.hpp" #include "api/gproto_priv.hpp" -#include "logger.hpp" // FIXME: it should be a visitor! // FIXME: Reimplement with traits? @@ -277,13 +276,9 @@ void cv::validate_input_arg(const GRunArg& arg) void cv::validate_input_args(const GRunArgs& args) { - GAPI_LOG_DEBUG(nullptr, "Total count: " << args.size()); - size_t index = 0; for (const auto& arg : args) { - GAPI_LOG_DEBUG(nullptr, "Process index: " << index); validate_input_arg(arg); - index ++; } } diff --git a/modules/gapi/src/api/kernels_streaming.cpp b/modules/gapi/src/api/kernels_streaming.cpp index c88fbede5b..2c50551f4e 100644 --- a/modules/gapi/src/api/kernels_streaming.cpp +++ b/modules/gapi/src/api/kernels_streaming.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2020 Intel Corporation +// Copyright (C) 2020-2021 Intel Corporation #include "precomp.hpp" @@ -75,6 +75,11 @@ cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) { // object will feed both branches of the streaming executable. } +// All notes from the above desync(GMat) are also applicable here +cv::GFrame cv::gapi::streaming::desync(const cv::GFrame &f) { + return cv::gapi::copy(detail::desync(f)); +} + cv::GMat cv::gapi::streaming::BGR(const cv::GFrame& in) { return cv::gapi::streaming::GBGR::on(in); } diff --git a/modules/gapi/src/api/media.cpp b/modules/gapi/src/api/media.cpp index 884fc9e83d..b1c455d40a 100644 --- a/modules/gapi/src/api/media.cpp +++ b/modules/gapi/src/api/media.cpp @@ -35,6 +35,10 @@ cv::MediaFrame::IAdapter* cv::MediaFrame::getAdapter() const { return m->adapter.get(); } +void cv::MediaFrame::serialize(cv::gapi::s11n::IOStream& os) const { + return m->adapter->serialize(os); +} + cv::MediaFrame::View::View(Ptrs&& ptrs, Strides&& strs, Callback &&cb) : ptr (std::move(ptrs)) , stride(std::move(strs)) diff --git a/modules/gapi/src/api/render_ocv.cpp b/modules/gapi/src/api/render_ocv.cpp index 5ab2e1dd07..f1e9be4b48 100644 --- a/modules/gapi/src/api/render_ocv.cpp +++ b/modules/gapi/src/api/render_ocv.cpp @@ -159,7 +159,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& rp = cv::util::get(p); const auto color = converter.cvtColor(rp.color); - cv::rectangle(in, rp.rect, color , rp.thick); + cv::rectangle(in, rp.rect, color, rp.thick, rp.lt, rp.shift); break; } @@ -198,7 +198,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& cp = cv::util::get(p); const auto color = converter.cvtColor(cp.color); - cv::circle(in, cp.center, cp.radius, color, cp.thick); + cv::circle(in, cp.center, cp.radius, color, cp.thick, cp.lt, cp.shift); break; } @@ -206,7 +206,7 @@ void drawPrimitivesOCV(cv::Mat& in, { const auto& lp = cv::util::get(p); const auto color = converter.cvtColor(lp.color); - cv::line(in, lp.pt1, lp.pt2, color, lp.thick); + cv::line(in, lp.pt1, lp.pt2, color, lp.thick, lp.lt, lp.shift); break; } diff --git a/modules/gapi/src/api/s11n.cpp b/modules/gapi/src/api/s11n.cpp index d08f47fd26..bd7f46c88a 100644 --- a/modules/gapi/src/api/s11n.cpp +++ b/modules/gapi/src/api/s11n.cpp @@ -65,25 +65,25 @@ std::vector cv::gapi::serialize(const std::vector& vs) // FIXME: This function should move from S11N to GRunArg-related entities. // it has nothing to do with the S11N as it is -cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &results) +cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &out_args) { cv::GRunArgsP outputs; - outputs.reserve(results.size()); - for (cv::GRunArg &res_obj : results) + outputs.reserve(out_args.size()); + for (cv::GRunArg &res_obj : out_args) { using T = cv::GRunArg; switch (res_obj.index()) { #if !defined(GAPI_STANDALONE) case T::index_of() : - outputs.emplace_back((cv::UMat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; #endif case cv::GRunArg::index_of() : - outputs.emplace_back((cv::Mat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; case cv::GRunArg::index_of() : - outputs.emplace_back((cv::Scalar*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); break; case T::index_of() : outputs.emplace_back(cv::util::get(res_obj)); @@ -92,7 +92,10 @@ cv::GRunArgsP cv::gapi::bind(cv::GRunArgs &results) outputs.emplace_back(cv::util::get(res_obj)); break; case cv::GRunArg::index_of() : - outputs.emplace_back((cv::RMat*)(&(cv::util::get(res_obj)))); + outputs.emplace_back(&(cv::util::get(res_obj))); + break; + case cv::GRunArg::index_of() : + outputs.emplace_back(&(cv::util::get(res_obj))); break; default: GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode. @@ -130,6 +133,9 @@ cv::GRunArg cv::gapi::bind(cv::GRunArgP &out) case T::index_of() : return cv::GRunArg(*cv::util::get(out)); + case T::index_of() : + return cv::GRunArg(*cv::util::get(out)); + default: // ...maybe our types were extended GAPI_Assert(false && "This value type is UNKNOWN!"); diff --git a/modules/gapi/src/backends/common/gmetabackend.cpp b/modules/gapi/src/backends/common/gmetabackend.cpp index c535569b0c..40e87c3ea0 100644 --- a/modules/gapi/src/backends/common/gmetabackend.cpp +++ b/modules/gapi/src/backends/common/gmetabackend.cpp @@ -85,6 +85,19 @@ class GGraphMetaBackendImpl final: public cv::gapi::GBackend::Priv { const std::vector&) const override { return EPtr{new GraphMetaExecutable(graph, nodes)}; } + + virtual bool controlsMerge() const override + { + return true; + } + + virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &, + const ade::NodeHandle &, + const ade::NodeHandle &, + const ade::NodeHandle &) const override + { + return false; + } }; cv::gapi::GBackend graph_meta_backend() { diff --git a/modules/gapi/src/backends/common/serialization.cpp b/modules/gapi/src/backends/common/serialization.cpp index 7389bacb02..619b2feb74 100644 --- a/modules/gapi/src/backends/common/serialization.cpp +++ b/modules/gapi/src/backends/common/serialization.cpp @@ -32,6 +32,14 @@ void putData(GSerialized& s, const cv::gimpl::GModel::ConstGraph& cg, const ade: }); if (s.m_datas.end() == it) { s.m_datas.push_back(gdata); + + if (cg.metadata(nh).contains()) { + size_t datas_num = s.m_datas.size() - 1; + GAPI_DbgAssert(datas_num <= static_cast(std::numeric_limits::max())); + GSerialized::data_tag_t tag = static_cast(datas_num); + s.m_const_datas.emplace(tag, + cg.metadata(nh).get()); + } } } @@ -42,11 +50,20 @@ void putOp(GSerialized& s, const cv::gimpl::GModel::ConstGraph& cg, const ade::N s.m_ops.push_back(op); } -void mkDataNode(ade::Graph& g, const cv::gimpl::Data& data) { +ade::NodeHandle mkDataNode(ade::Graph& g, const cv::gimpl::Data& data) { cv::gimpl::GModel::Graph gm(g); auto nh = gm.createNode(); gm.metadata(nh).set(cv::gimpl::NodeType{cv::gimpl::NodeType::DATA}); gm.metadata(nh).set(data); + return nh; +} + +ade::NodeHandle mkConstDataNode(ade::Graph& g, const cv::gimpl::Data& data, const cv::gimpl::ConstValue& const_data) { + auto nh = mkDataNode(g, data); + + cv::gimpl::GModel::Graph gm(g); + gm.metadata(nh).set(const_data); + return nh; } void mkOpNode(ade::Graph& g, const cv::gimpl::Op& op) { @@ -184,18 +201,20 @@ IOStream& operator<< (IOStream& os, const cv::RMat& mat) { return os; } IIStream& operator>> (IIStream& is, cv::RMat&) { - util::throw_error(std::logic_error("operator>> for RMat should never be called")); + util::throw_error(std::logic_error("operator>> for RMat should never be called. " + "Instead, cv::gapi::deserialize() " + "should be used")); return is; } -IOStream& operator<< (IOStream& os, const cv::MediaFrame &) { - // Stub - GAPI_Assert(false && "cv::MediaFrame serialization is not supported!"); +IOStream& operator<< (IOStream& os, const cv::MediaFrame &frame) { + frame.serialize(os); return os; } IIStream& operator>> (IIStream& is, cv::MediaFrame &) { - // Stub - GAPI_Assert(false && "cv::MediaFrame serialization is not supported!"); + util::throw_error(std::logic_error("operator>> for MediaFrame should never be called. " + "Instead, cv::gapi::deserialize() " + "should be used")); return is; } @@ -624,6 +643,10 @@ IOStream& operator<< (IOStream& os, const cv::gimpl::Data &d) { return os << d.shape << d.rc << d.meta << d.storage << d.kind; } +IOStream& operator<< (IOStream& os, const cv::gimpl::ConstValue &cd) { + return os << cd.arg; +} + namespace { template @@ -667,6 +690,9 @@ IIStream& operator>> (IIStream& is, cv::gimpl::Data &d) { return is; } +IIStream& operator>> (IIStream& is, cv::gimpl::ConstValue &cd) { + return is >> cd.arg; +} IOStream& operator<< (IOStream& os, const cv::gimpl::DataObjectCounter &c) { return os << c.m_next_data_id; @@ -709,18 +735,34 @@ void serialize( IOStream& os } s.m_counter = cg.metadata().get(); s.m_proto = p; - os << s.m_ops << s.m_datas << s.m_counter << s.m_proto; + os << s.m_ops << s.m_datas << s.m_counter << s.m_proto << s.m_const_datas; } GSerialized deserialize(IIStream &is) { GSerialized s; - is >> s.m_ops >> s.m_datas >> s.m_counter >> s.m_proto; + is >> s.m_ops >> s.m_datas >> s.m_counter >> s.m_proto >> s.m_const_datas; return s; } void reconstruct(const GSerialized &s, ade::Graph &g) { GAPI_Assert(g.nodes().empty()); - for (const auto& d : s.m_datas) cv::gapi::s11n::mkDataNode(g, d); + + GSerialized::data_tag_t tag = 0; + for (const auto& d : s.m_datas) { + if (d.storage == gimpl::Data::Storage::CONST_VAL) { + auto cit = s.m_const_datas.find(tag); + if (cit == s.m_const_datas.end()) { + util::throw_error(std::logic_error("Cannot reconstruct graph: Data::Storage::CONST_VAL by tag: " + + std::to_string(tag) + " requires ConstValue")); + } + + mkConstDataNode(g, d, cit->second); + } else { + cv::gapi::s11n::mkDataNode(g, d); + } + + tag ++; + } for (const auto& op : s.m_ops) cv::gapi::s11n::mkOpNode(g, op); cv::gapi::s11n::linkNodes(g); diff --git a/modules/gapi/src/backends/common/serialization.hpp b/modules/gapi/src/backends/common/serialization.hpp index b4204ca64e..529fdc635d 100644 --- a/modules/gapi/src/backends/common/serialization.hpp +++ b/modules/gapi/src/backends/common/serialization.hpp @@ -31,6 +31,9 @@ struct GSerialized { std::vector m_datas; cv::gimpl::DataObjectCounter m_counter; cv::gimpl::Protocol m_proto; + + using data_tag_t = uint64_t; + std::map m_const_datas; }; //////////////////////////////////////////////////////////////////////////////// @@ -97,6 +100,9 @@ GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gimpl::Op &op); GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::Data &op); GAPI_EXPORTS IIStream& operator>> (IIStream& is, cv::gimpl::Data &op); +GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gimpl::ConstValue &cd); +GAPI_EXPORTS IIStream& operator>> (IIStream& os, cv::gimpl::ConstValue &cd); + // Render types //////////////////////////////////////////////////////////////// GAPI_EXPORTS IOStream& operator<< (IOStream& os, const cv::gapi::wip::draw::Text &t); diff --git a/modules/gapi/src/backends/cpu/gcpucore.cpp b/modules/gapi/src/backends/cpu/gcpucore.cpp index 168a2f9833..1f7dfb234f 100644 --- a/modules/gapi/src/backends/cpu/gcpucore.cpp +++ b/modules/gapi/src/backends/cpu/gcpucore.cpp @@ -652,7 +652,12 @@ GAPI_OCV_KERNEL(GCPUParseSSDBL, cv::gapi::nn::parsers::GParseSSDBL) std::vector& out_boxes, std::vector& out_labels) { - cv::parseSSDBL(in_ssd_result, in_size, confidence_threshold, filter_label, out_boxes, out_labels); + cv::ParseSSD(in_ssd_result, in_size, + confidence_threshold, + filter_label, + false, + false, + out_boxes, out_labels); } }; @@ -665,7 +670,13 @@ GAPI_OCV_KERNEL(GOCVParseSSD, cv::gapi::nn::parsers::GParseSSD) const bool filter_out_of_bounds, std::vector& out_boxes) { - cv::parseSSD(in_ssd_result, in_size, confidence_threshold, alignment_to_square, filter_out_of_bounds, out_boxes); + std::vector unused_labels; + cv::ParseSSD(in_ssd_result, in_size, + confidence_threshold, + -1, + alignment_to_square, + filter_out_of_bounds, + out_boxes, unused_labels); } }; diff --git a/modules/gapi/src/backends/cpu/gnnparsers.cpp b/modules/gapi/src/backends/cpu/gnnparsers.cpp index a5e4bf5f85..26de960b7f 100644 --- a/modules/gapi/src/backends/cpu/gnnparsers.cpp +++ b/modules/gapi/src/backends/cpu/gnnparsers.cpp @@ -170,12 +170,14 @@ private: } // namespace nn } // namespace gapi -void parseSSDBL(const cv::Mat& in_ssd_result, - const cv::Size& in_size, - const float confidence_threshold, - const int filter_label, - std::vector& out_boxes, - std::vector& out_labels) +void ParseSSD(const cv::Mat& in_ssd_result, + const cv::Size& in_size, + const float confidence_threshold, + const int filter_label, + const bool alignment_to_square, + const bool filter_out_of_bounds, + std::vector& out_boxes, + std::vector& out_labels) { cv::gapi::nn::SSDParser parser(in_ssd_result.size, in_size, in_ssd_result.ptr()); out_boxes.clear(); @@ -188,38 +190,6 @@ void parseSSDBL(const cv::Mat& in_ssd_result, { std::tie(rc, image_id, confidence, label) = parser.extract(i); - if (image_id < 0.f) - { - break; // marks end-of-detections - } - - if (confidence < confidence_threshold || - (filter_label != -1 && label != filter_label)) - { - continue; // filter out object classes if filter is specified - } // and skip objects with low confidence - out_boxes.emplace_back(rc & parser.getSurface()); - out_labels.emplace_back(label); - } -} - -void parseSSD(const cv::Mat& in_ssd_result, - const cv::Size& in_size, - const float confidence_threshold, - const bool alignment_to_square, - const bool filter_out_of_bounds, - std::vector& out_boxes) -{ - cv::gapi::nn::SSDParser parser(in_ssd_result.size, in_size, in_ssd_result.ptr()); - out_boxes.clear(); - cv::Rect rc; - float image_id, confidence; - int label; - const size_t range = parser.getMaxProposals(); - for (size_t i = 0; i < range; ++i) - { - std::tie(rc, image_id, confidence, label) = parser.extract(i); - if (image_id < 0.f) { break; // marks end-of-detections @@ -228,12 +198,14 @@ void parseSSD(const cv::Mat& in_ssd_result, { continue; // skip objects with low confidence } - + if((filter_label != -1) && (label != filter_label)) + { + continue; // filter out object classes if filter is specified + } if (alignment_to_square) { parser.adjustBoundingBox(rc); } - const auto clipped_rc = rc & parser.getSurface(); if (filter_out_of_bounds) { @@ -243,6 +215,7 @@ void parseSSD(const cv::Mat& in_ssd_result, } } out_boxes.emplace_back(clipped_rc); + out_labels.emplace_back(label); } } diff --git a/modules/gapi/src/backends/cpu/gnnparsers.hpp b/modules/gapi/src/backends/cpu/gnnparsers.hpp index 2ae6693705..28b6128f0c 100644 --- a/modules/gapi/src/backends/cpu/gnnparsers.hpp +++ b/modules/gapi/src/backends/cpu/gnnparsers.hpp @@ -11,19 +11,14 @@ namespace cv { -void parseSSDBL(const cv::Mat& in_ssd_result, - const cv::Size& in_size, - const float confidence_threshold, - const int filter_label, - std::vector& out_boxes, - std::vector& out_labels); - -void parseSSD(const cv::Mat& in_ssd_result, +void ParseSSD(const cv::Mat& in_ssd_result, const cv::Size& in_size, const float confidence_threshold, + const int filter_label, const bool alignment_to_square, const bool filter_out_of_bounds, - std::vector& out_boxes); + std::vector& out_boxes, + std::vector& out_labels); void parseYolo(const cv::Mat& in_yolo_result, const cv::Size& in_size, diff --git a/modules/gapi/src/backends/ie/bindings_ie.cpp b/modules/gapi/src/backends/ie/bindings_ie.cpp index 35191d7bcb..5874fe1378 100644 --- a/modules/gapi/src/backends/ie/bindings_ie.cpp +++ b/modules/gapi/src/backends/ie/bindings_ie.cpp @@ -37,3 +37,15 @@ cv::gapi::ie::PyParams cv::gapi::ie::params(const std::string &tag, const std::string &device) { return {tag, model, device}; } + +cv::gapi::ie::PyParams& cv::gapi::ie::PyParams::constInput(const std::string &layer_name, + const cv::Mat &data, + TraitAs hint) { + m_priv->constInput(layer_name, data, hint); + return *this; +} + +cv::gapi::ie::PyParams& cv::gapi::ie::PyParams::cfgNumRequests(size_t nireq) { + m_priv->cfgNumRequests(nireq); + return *this; +} diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 46b6bdbb97..03584c9561 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -108,6 +108,7 @@ inline IE::Precision toIE(int depth) { case CV_8U: return IE::Precision::U8; case CV_32S: return IE::Precision::I32; case CV_32F: return IE::Precision::FP32; + case CV_16F: return IE::Precision::FP16; default: GAPI_Assert(false && "IE. Unsupported data type"); } return IE::Precision::UNSPECIFIED; @@ -161,6 +162,7 @@ inline IE::Blob::Ptr wrapIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) { HANDLE(8U, uint8_t); HANDLE(32F, float); HANDLE(32S, int); + HANDLE(16F, int16_t); #undef HANDLE default: GAPI_Assert(false && "IE. Unsupported data type"); } @@ -222,8 +224,17 @@ struct IEUnit { IE::ExecutableNetwork this_network; cv::gimpl::ie::wrap::Plugin this_plugin; + InferenceEngine::RemoteContext::Ptr rctx = nullptr; + explicit IEUnit(const cv::gapi::ie::detail::ParamDesc &pp) : params(pp) { + InferenceEngine::ParamMap* ctx_params = + cv::util::any_cast(¶ms.context_config); + if (ctx_params != nullptr) { + auto ie_core = cv::gimpl::ie::wrap::getCore(); + rctx = ie_core.CreateContext(params.device_id, *ctx_params); + } + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { net = cv::gimpl::ie::wrap::readNetwork(params); inputs = net.getInputsInfo(); @@ -231,11 +242,7 @@ struct IEUnit { } else if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Import) { this_plugin = cv::gimpl::ie::wrap::getPlugin(params); this_plugin.SetConfig(params.config); - this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params); - // FIXME: ICNNetwork returns InputsDataMap/OutputsDataMap, - // but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap - inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo()); - outputs = cv::gimpl::ie::wrap::toOutputsDataMap(this_network.GetOutputsInfo()); + this_network = cv::gimpl::ie::wrap::importNetwork(this_plugin, params, rctx); if (!params.reshape_table.empty() || !params.layer_names_to_reshape.empty()) { GAPI_LOG_WARNING(NULL, "Reshape isn't supported for imported network"); } @@ -259,10 +266,18 @@ struct IEUnit { + params.model_path)); } if (params.num_in == 1u && params.input_names.empty()) { - params.input_names = { inputs.begin()->first }; + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + params.input_names = { inputs.begin()->first }; + } else { + params.input_names = { this_network.GetInputsInfo().begin()->first }; + } } if (params.num_out == 1u && params.output_names.empty()) { - params.output_names = { outputs.begin()->first }; + if (params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + params.output_names = { outputs.begin()->first }; + } else { + params.output_names = { this_network.GetOutputsInfo().begin()->first }; + } } if (!params.reshape_table.empty()) { GAPI_Assert((params.reshape_table.size() + params.layer_names_to_reshape.size()) <= @@ -279,7 +294,8 @@ struct IEUnit { // for loadNetwork they can be obtained by using readNetwork non_const_this->this_plugin = cv::gimpl::ie::wrap::getPlugin(params); non_const_this->this_plugin.SetConfig(params.config); - non_const_this->this_network = cv::gimpl::ie::wrap::loadNetwork(non_const_this->this_plugin, net, params); + non_const_this->this_network = cv::gimpl::ie::wrap::loadNetwork(non_const_this->this_plugin, + net, params, rctx); } return {params, this_plugin, this_network}; @@ -481,7 +497,32 @@ using GConstGIEModel = ade::ConstTypedGraph , IECallable >; +inline IE::Blob::Ptr extractRemoteBlob(IECallContext& ctx, std::size_t i) { + GAPI_Assert(ctx.inShape(i) == cv::GShape::GFRAME && + "Remote blob is supported for MediaFrame only"); + + cv::util::any any_blob_params = ctx.inFrame(i).blobParams(); + auto ie_core = cv::gimpl::ie::wrap::getCore(); + + using ParamType = std::pair; + + ParamType* blob_params = cv::util::any_cast(&any_blob_params); + if (blob_params == nullptr) { + GAPI_Assert(false && "Incorrect type of blobParams: " + "expected std::pair"); + } + + return ctx.uu.rctx->CreateBlob(blob_params->first, + blob_params->second); +} + inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) { + if (ctx.uu.rctx != nullptr) { + return extractRemoteBlob(ctx, i); + } + switch (ctx.inShape(i)) { case cv::GShape::GFRAME: { const auto& frame = ctx.inFrame(i); @@ -496,6 +537,24 @@ inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i) { } GAPI_Assert(false); } + + +static void setBlob(InferenceEngine::InferRequest& req, + cv::gapi::ie::detail::ParamDesc::Kind kind, + const std::string& layer_name, + IE::Blob::Ptr blob) { + // NB: In case importNetwork preprocessing must be + // passed as SetBlob argument. + if (kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + req.SetBlob(layer_name, blob); + } else { + GAPI_Assert(kind == cv::gapi::ie::detail::ParamDesc::Kind::Import); + IE::PreProcessInfo info; + info.setResizeAlgorithm(IE::RESIZE_BILINEAR); + req.SetBlob(layer_name, blob, info); + } +} + } // anonymous namespace std::vector cv::gimpl::ie::IECompiled::createInferRequests() { @@ -571,6 +630,11 @@ void cv::gimpl::ie::RequestPool::callback(cv::gimpl::ie::RequestPool::Task task, InferenceEngine::InferRequest& request, size_t id) { task.callback(request); + // NB: IE::InferRequest keeps the callback until the new one is set. + // Since user's callback might keep resources that should be released, + // need to destroy its after execution. + // Let's set the empty one to cause the destruction of a callback. + request.SetCompletionCallback([](){}); m_idle_ids.push(id); } @@ -772,7 +836,6 @@ static void PostOutputs(InferenceEngine::InferRequest &request, auto output = ctx->output(i); ctx->out.meta(output, ctx->input(0).meta); ctx->out.post(std::move(output)); - } } @@ -854,25 +917,30 @@ struct Infer: public cv::detail::KernelTag { // meta order. GAPI_Assert(uu.params.input_names.size() == in_metas.size() && "Known input layers count doesn't match input meta count"); - for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names), - ade::util::toRange(in_metas))) { - const auto &input_name = std::get<0>(it); - auto &&ii = uu.inputs.at(input_name); - const auto & mm = std::get<1>(it); - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names), + ade::util::toRange(in_metas))) { + const auto &input_name = std::get<0>(it); + auto &&ii = uu.inputs.at(input_name); + const auto & mm = std::get<1>(it); + + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - } - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // FIXME: It would be nice here to have an exact number of network's @@ -904,7 +972,10 @@ struct Infer: public cv::detail::KernelTag { // and redirect our data producers to this memory // (A memory dialog comes to the picture again) IE::Blob::Ptr this_blob = extractBlob(*ctx, i); - req.SetBlob(ctx->uu.params.input_names[i], this_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[i], + this_blob); } // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? @@ -936,22 +1007,26 @@ struct InferROI: public cv::detail::KernelTag { GAPI_Assert(1u == uu.params.input_names.size()); GAPI_Assert(2u == in_metas.size()); - // 0th is ROI, 1st is input image - const auto &input_name = uu.params.input_names.at(0); - auto &&ii = uu.inputs.at(input_name); - auto &&mm = in_metas.at(1u); - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); - } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + // 0th is ROI, 1st is input image + const auto &input_name = uu.params.input_names.at(0); + auto &&ii = uu.inputs.at(input_name); + auto &&mm = in_metas.at(1u); + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // FIXME: It would be nice here to have an exact number of network's @@ -980,10 +1055,11 @@ struct InferROI: public cv::detail::KernelTag { auto&& this_roi = ctx->inArg(0).rref(); IE::Blob::Ptr this_blob = extractBlob(*ctx, 1); - - req.SetBlob(*(ctx->uu.params.input_names.begin()), - IE::make_shared_blob(this_blob, toIE(this_roi))); - + setBlob(req, + ctx->uu.params.kind, + *(ctx->uu.params.input_names.begin()), + IE::make_shared_blob(this_blob, + toIE(this_roi))); // FIXME: Should it be done by kernel ? // What about to do that in RequestPool ? req.StartAsync(); @@ -1018,23 +1094,27 @@ struct InferList: public cv::detail::KernelTag { GAPI_Assert(uu.params.input_names.size() == (in_metas.size() - 1u) && "Known input layers count doesn't match input meta count"); - std::size_t idx = 1u; - for (auto &&input_name : uu.params.input_names) { - auto &&ii = uu.inputs.at(input_name); - const auto & mm = in_metas[idx++]; - configureInputInfo(ii, mm); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm, input_reshape_table); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + std::size_t idx = 1u; + for (auto &&input_name : uu.params.input_names) { + auto &&ii = uu.inputs.at(input_name); + const auto & mm = in_metas[idx++]; + configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - } - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } // roi-list version is much easier at the moment. @@ -1060,6 +1140,7 @@ struct InferList: public cv::detail::KernelTag { } IE::Blob::Ptr this_blob = extractBlob(*ctx, 1); + std::vector> cached_dims(ctx->uu.params.num_out); for (auto i : ade::util::iota(ctx->uu.params.num_out)) { const IE::DataPtr& ie_out = ctx->uu.outputs.at(ctx->uu.params.output_names[i]); @@ -1079,7 +1160,10 @@ struct InferList: public cv::detail::KernelTag { cv::gimpl::ie::RequestPool::Task { [ctx, rc, this_blob](InferenceEngine::InferRequest &req) { IE::Blob::Ptr roi_blob = IE::make_shared_blob(this_blob, toIE(rc)); - req.SetBlob(ctx->uu.params.input_names[0u], roi_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[0u], + roi_blob); req.StartAsync(); }, std::bind(callback, std::placeholders::_1, pos) @@ -1153,19 +1237,23 @@ struct InferList2: public cv::detail::KernelTag { && "Non-array inputs are not supported"); if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) { - // This is a cv::Rect -- configure the IE preprocessing - configureInputInfo(ii, mm_0); - if (uu.params.layer_names_to_reshape.find(input_name) != - uu.params.layer_names_to_reshape.end()) { - configureInputReshapeByImage(ii, mm_0, input_reshape_table); - } - ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + // NB: Configuring input precision and network reshape must be done + // only in the loadNetwork case. + if (uu.params.kind == cv::gapi::ie::detail::ParamDesc::Kind::Load) { + // This is a cv::Rect -- configure the IE preprocessing + configureInputInfo(ii, mm_0); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm_0, input_reshape_table); + } + ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); - // FIXME: This isn't the best place to call reshape function. - // Сorrect solution would be to do this in compile() method of network, - // but now input meta isn't passed to compile() method. - if (!input_reshape_table.empty()) { - const_cast(&uu.net)->reshape(input_reshape_table); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } } else { // This is a cv::GMat (equals to: cv::Mat) @@ -1230,8 +1318,10 @@ struct InferList2: public cv::detail::KernelTag { GAPI_Assert(false && "Only Rect and Mat types are supported for infer list 2!"); } - - req.SetBlob(ctx->uu.params.input_names[in_idx], this_blob); + setBlob(req, + ctx->uu.params.kind, + ctx->uu.params.input_names[in_idx], + this_blob); } req.StartAsync(); }, diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp index ba0632d4f0..1f9721dbf4 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.cpp @@ -22,24 +22,6 @@ namespace IE = InferenceEngine; namespace giewrap = cv::gimpl::ie::wrap; using GIEParam = cv::gapi::ie::detail::ParamDesc; -IE::InputsDataMap giewrap::toInputsDataMap (const IE::ConstInputsDataMap& inputs) { - IE::InputsDataMap transformed; - auto convert = [](const std::pair& p) { - return std::make_pair(p.first, std::const_pointer_cast(p.second)); - }; - std::transform(inputs.begin(), inputs.end(), std::inserter(transformed, transformed.end()), convert); - return transformed; -} - -IE::OutputsDataMap giewrap::toOutputsDataMap (const IE::ConstOutputsDataMap& outputs) { - IE::OutputsDataMap transformed; - auto convert = [](const std::pair& p) { - return std::make_pair(p.first, std::const_pointer_cast(p.second)); - }; - std::transform(outputs.begin(), outputs.end(), std::inserter(transformed, transformed.end()), convert); - return transformed; -} - #if INF_ENGINE_RELEASE < 2020000000 // < 2020.1 // Load extensions (taken from DNN module) std::vector giewrap::getExtensions(const GIEParam& params) { @@ -124,7 +106,11 @@ IE::Core giewrap::getPlugin(const GIEParam& params) { { try { +#if INF_ENGINE_RELEASE >= 2021040000 + plugin.AddExtension(std::make_shared(extlib), params.device_id); +#else plugin.AddExtension(IE::make_so_pointer(extlib), params.device_id); +#endif CV_LOG_INFO(NULL, "DNN-IE: Loaded extension plugin: " << extlib); break; } diff --git a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp index 3927c802b7..2e4bac1270 100644 --- a/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp +++ b/modules/gapi/src/backends/ie/giebackend/giewrapper.hpp @@ -13,6 +13,7 @@ #include #include +#include #include "opencv2/gapi/infer/ie.hpp" @@ -28,9 +29,6 @@ namespace wrap { GAPI_EXPORTS std::vector getExtensions(const GIEParam& params); GAPI_EXPORTS IE::CNNNetwork readNetwork(const GIEParam& params); -IE::InputsDataMap toInputsDataMap (const IE::ConstInputsDataMap& inputs); -IE::OutputsDataMap toOutputsDataMap(const IE::ConstOutputsDataMap& outputs); - #if INF_ENGINE_RELEASE < 2019020000 // < 2019.R2 using Plugin = IE::InferencePlugin; GAPI_EXPORTS IE::InferencePlugin getPlugin(const GIEParam& params); @@ -50,12 +48,29 @@ GAPI_EXPORTS IE::Core getCore(); GAPI_EXPORTS IE::Core getPlugin(const GIEParam& params); GAPI_EXPORTS inline IE::ExecutableNetwork loadNetwork( IE::Core& core, const IE::CNNNetwork& net, - const GIEParam& params) { - return core.LoadNetwork(net, params.device_id); + const GIEParam& params, + IE::RemoteContext::Ptr rctx = nullptr) { + if (rctx != nullptr) { + return core.LoadNetwork(net, rctx); + } else { + return core.LoadNetwork(net, params.device_id); + } } GAPI_EXPORTS inline IE::ExecutableNetwork importNetwork( IE::Core& core, - const GIEParam& param) { - return core.ImportNetwork(param.model_path, param.device_id, {}); + const GIEParam& params, + IE::RemoteContext::Ptr rctx = nullptr) { + if (rctx != nullptr) { + std::filebuf blobFile; + if (!blobFile.open(params.model_path, std::ios::in | std::ios::binary)) + { + blobFile.close(); + throw std::runtime_error("Could not open file"); + } + std::istream graphBlob(&blobFile); + return core.ImportNetwork(graphBlob, rctx); + } else { + return core.ImportNetwork(params.model_path, params.device_id, {}); + } } #endif // INF_ENGINE_RELEASE < 2019020000 }}}} diff --git a/modules/gapi/src/compiler/gstreaming.cpp b/modules/gapi/src/compiler/gstreaming.cpp index 3bdc0323b5..e45e770427 100644 --- a/modules/gapi/src/compiler/gstreaming.cpp +++ b/modules/gapi/src/compiler/gstreaming.cpp @@ -75,6 +75,11 @@ bool cv::GStreamingCompiled::Priv::pull(cv::GOptRunArgsP &&outs) return m_exec->pull(std::move(outs)); } +std::tuple> cv::GStreamingCompiled::Priv::pull() +{ + return m_exec->pull(); +} + bool cv::GStreamingCompiled::Priv::try_pull(cv::GRunArgsP &&outs) { return m_exec->try_pull(std::move(outs)); @@ -123,18 +128,9 @@ bool cv::GStreamingCompiled::pull(cv::GRunArgsP &&outs) return m_priv->pull(std::move(outs)); } -std::tuple cv::GStreamingCompiled::pull() +std::tuple> cv::GStreamingCompiled::pull() { - GRunArgs run_args; - GRunArgsP outs; - const auto& out_info = m_priv->outInfo(); - run_args.reserve(out_info.size()); - outs.reserve(out_info.size()); - - cv::detail::constructGraphOutputs(m_priv->outInfo(), run_args, outs); - - bool is_over = m_priv->pull(std::move(outs)); - return std::make_tuple(is_over, run_args); + return m_priv->pull(); } bool cv::GStreamingCompiled::pull(cv::GOptRunArgsP &&outs) diff --git a/modules/gapi/src/compiler/gstreaming_priv.hpp b/modules/gapi/src/compiler/gstreaming_priv.hpp index 59b19d4252..1b559ba310 100644 --- a/modules/gapi/src/compiler/gstreaming_priv.hpp +++ b/modules/gapi/src/compiler/gstreaming_priv.hpp @@ -46,6 +46,7 @@ public: void start(); bool pull(cv::GRunArgsP &&outs); bool pull(cv::GOptRunArgsP &&outs); + std::tuple> pull(); bool try_pull(cv::GRunArgsP &&outs); void stop(); diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 6f313197ba..ad9d380b4e 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -7,8 +7,6 @@ #include "precomp.hpp" -#include - #include #include @@ -157,10 +155,14 @@ void writeBackExec(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg) // FIXME: // Rework, find a better way to check if there should be // a real copy (add a pass to StreamingBackend?) + // NB: In case RMat adapter not equal to "RMatAdapter" need to + // copy data back to the host as well. + // FIXME: Rename "RMatAdapter" to "OpenCVAdapter". auto& out_mat = *util::get(g_arg); const auto& rmat = mag.template slot().at(rc.id); - auto mag_data = rmat.get()->data(); - if (out_mat.data != mag_data) { + auto* adapter = rmat.get(); + if ((adapter != nullptr && out_mat.data != adapter->data()) || + (adapter == nullptr)) { auto view = rmat.access(RMat::Access::R); asMat(view).copyTo(out_mat); } @@ -407,7 +409,8 @@ bool cv::gimpl::GExecutor::canReshape() const { // FIXME: Introduce proper reshaping support on GExecutor level // for all cases! - return (m_ops.size() == 1) && m_ops[0].isl_exec->canReshape(); + return std::all_of(m_ops.begin(), m_ops.end(), + [](const OpDesc& op) { return op.isl_exec->canReshape(); }); } void cv::gimpl::GExecutor::reshape(const GMetaArgs& inMetas, const GCompileArgs& args) @@ -417,7 +420,17 @@ void cv::gimpl::GExecutor::reshape(const GMetaArgs& inMetas, const GCompileArgs& ade::passes::PassContext ctx{g}; passes::initMeta(ctx, inMetas); passes::inferMeta(ctx, true); - m_ops[0].isl_exec->reshape(g, args); + + // NB: Before reshape islands need to re-init resources for every slot. + for (auto slot : m_slots) + { + initResource(slot.slot_nh, slot.data_nh); + } + + for (auto& op : m_ops) + { + op.isl_exec->reshape(g, args); + } } void cv::gimpl::GExecutor::prepareForNewStream() diff --git a/modules/gapi/src/executor/gstreamingexecutor.cpp b/modules/gapi/src/executor/gstreamingexecutor.cpp index 74c96bdf3e..bc43db8a37 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.cpp +++ b/modules/gapi/src/executor/gstreamingexecutor.cpp @@ -186,8 +186,9 @@ void sync_data(cv::gimpl::stream::Result &r, cv::GOptRunArgsP &outputs) // FIXME: this conversion should be unified switch (out_obj.index()) { - HANDLE_CASE(cv::Scalar); break; - HANDLE_CASE(cv::RMat); break; + HANDLE_CASE(cv::Scalar); break; + HANDLE_CASE(cv::RMat); break; + HANDLE_CASE(cv::MediaFrame); break; case T::index_of*>(): { // Mat: special handling. @@ -346,6 +347,10 @@ bool QueueReader::getInputVector(std::vector &in_queues, cv::GRunArgs &in_constants, cv::GRunArgs &isl_inputs) { + // NB: Need to release resources from the previous step, to fetch new ones. + // On some systems it might be impossible to allocate new memory + // until the old one is released. + m_cmd.clear(); // NOTE: in order to maintain the GRunArg's underlying object // lifetime, keep the whole cmd vector (of size == # of inputs) // in memory. @@ -1017,6 +1022,49 @@ void check_DesyncObjectConsumedByMultipleIslands(const cv::gimpl::GIslandModel:: } // for(nodes) } +// NB: Construct GRunArgsP based on passed info and store the memory in passed cv::GRunArgs. +// Needed for python bridge, because in case python user doesn't pass output arguments to apply. +void constructOptGraphOutputs(const cv::GTypesInfo &out_info, + cv::GOptRunArgs &args, + cv::GOptRunArgsP &outs) +{ + for (auto&& info : out_info) + { + switch (info.shape) + { + case cv::GShape::GMAT: + { + args.emplace_back(cv::optional{}); + outs.emplace_back(&cv::util::get>(args.back())); + break; + } + case cv::GShape::GSCALAR: + { + args.emplace_back(cv::optional{}); + outs.emplace_back(&cv::util::get>(args.back())); + break; + } + case cv::GShape::GARRAY: + { + cv::detail::VectorRef ref; + cv::util::get(info.ctor)(ref); + args.emplace_back(cv::util::make_optional(std::move(ref))); + outs.emplace_back(wrap_opt_arg(cv::util::get>(args.back()))); + break; + } + case cv::GShape::GOPAQUE: + { + cv::detail::OpaqueRef ref; + cv::util::get(info.ctor)(ref); + args.emplace_back(cv::util::make_optional(std::move(ref))); + outs.emplace_back(wrap_opt_arg(cv::util::get>(args.back()))); + break; + } + default: + cv::util::throw_error(std::logic_error("Unsupported optional output shape for Python")); + } + } +} } // anonymous namespace class cv::gimpl::GStreamingExecutor::Synchronizer final { @@ -1320,6 +1368,16 @@ cv::gimpl::GStreamingExecutor::GStreamingExecutor(std::unique_ptr && // per the same input frame, so the output traffic multiplies) GAPI_Assert(m_collector_map.size() > 0u); m_out_queue.set_capacity(queue_capacity * m_collector_map.size()); + + // FIXME: The code duplicates logic of collectGraphInfo() + cv::gimpl::GModel::ConstGraph cgr(*m_orig_graph); + auto meta = cgr.metadata().get().out_nhs; + out_info.reserve(meta.size()); + + ade::util::transform(meta, std::back_inserter(out_info), [&cgr](const ade::NodeHandle& nh) { + const auto& data = cgr.metadata(nh).get(); + return cv::GTypeInfo{data.shape, data.kind, data.ctor}; + }); } cv::gimpl::GStreamingExecutor::~GStreamingExecutor() @@ -1653,6 +1711,31 @@ bool cv::gimpl::GStreamingExecutor::pull(cv::GOptRunArgsP &&outs) return true; } +std::tuple> cv::gimpl::GStreamingExecutor::pull() +{ + using RunArgs = cv::util::variant; + bool is_over = false; + + if (m_desync) { + GOptRunArgs opt_run_args; + GOptRunArgsP opt_outs; + opt_outs.reserve(out_info.size()); + opt_run_args.reserve(out_info.size()); + + constructOptGraphOutputs(out_info, opt_run_args, opt_outs); + is_over = pull(std::move(opt_outs)); + return std::make_tuple(is_over, RunArgs(opt_run_args)); + } + + GRunArgs run_args; + GRunArgsP outs; + run_args.reserve(out_info.size()); + outs.reserve(out_info.size()); + + constructGraphOutputs(out_info, run_args, outs); + is_over = pull(std::move(outs)); + return std::make_tuple(is_over, RunArgs(run_args)); +} bool cv::gimpl::GStreamingExecutor::try_pull(cv::GRunArgsP &&outs) { diff --git a/modules/gapi/src/executor/gstreamingexecutor.hpp b/modules/gapi/src/executor/gstreamingexecutor.hpp index 40b7872682..b4aadcbbaf 100644 --- a/modules/gapi/src/executor/gstreamingexecutor.hpp +++ b/modules/gapi/src/executor/gstreamingexecutor.hpp @@ -195,6 +195,8 @@ protected: void wait_shutdown(); + cv::GTypesInfo out_info; + public: explicit GStreamingExecutor(std::unique_ptr &&g_model, const cv::GCompileArgs &comp_args); @@ -203,6 +205,7 @@ public: void start(); bool pull(cv::GRunArgsP &&outs); bool pull(cv::GOptRunArgsP &&outs); + std::tuple> pull(); bool try_pull(cv::GRunArgsP &&outs); void stop(); bool running() const; diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp new file mode 100644 index 0000000000..75cb2fdb38 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.cpp @@ -0,0 +1,227 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include +#include + +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef _WIN32 + #include + #include +#endif +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +VPLCPUAccelerationPolicy::VPLCPUAccelerationPolicy() { + GAPI_LOG_INFO(nullptr, "created"); +} + +VPLCPUAccelerationPolicy::~VPLCPUAccelerationPolicy() { + for (auto& pair : pool_table) { + pair.second.clear(); + // do not free key here: last surface will release it + } + GAPI_LOG_INFO(nullptr, "destroyed"); +} + +void VPLCPUAccelerationPolicy::init(session_t session) { + GAPI_LOG_INFO(nullptr, "initialize session: " << session); +} + +void VPLCPUAccelerationPolicy::deinit(session_t session) { + GAPI_LOG_INFO(nullptr, "deinitialize session: " << session); +} + +VPLCPUAccelerationPolicy::pool_key_t +VPLCPUAccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_size_bytes, + surface_ptr_ctr_t creator) { + GAPI_LOG_DEBUG(nullptr, "pool size: " << pool_size << ", surface size bytes: " << surface_size_bytes); + + // create empty pool + pool_t pool; + pool.reserve(pool_size); + + // allocate workplace memory area + size_t preallocated_raw_bytes = pool_size * surface_size_bytes; + size_t page_size_bytes = 4 * 1024; + void *preallocated_pool_memory_ptr = nullptr; + +#ifdef _WIN32 + SYSTEM_INFO sysInfo; + GetSystemInfo(&sysInfo); + page_size_bytes = sysInfo.dwPageSize; + + GAPI_LOG_DEBUG(nullptr, "page size: " << page_size_bytes << ", preallocated_raw_bytes: " << preallocated_raw_bytes); + preallocated_pool_memory_ptr = _aligned_malloc(preallocated_raw_bytes, page_size_bytes); +#else + GAPI_Assert(false && "Compatibility is not tested for systems differ than \"_WIN32\". " + "Please feel free to set it up under OPENCV contribution policy"); +#endif + + if (!preallocated_pool_memory_ptr) { + throw std::runtime_error("VPLCPUAccelerationPolicy::create_surface_pool - failed: not enough memory." + "Requested surface count: " + std::to_string(pool_size) + + ", surface bytes: " + std::to_string(surface_size_bytes)); + } + + // fill pool with surfaces + std::shared_ptr workspace_mem_owner (preallocated_pool_memory_ptr, [] (void *ptr){ + GAPI_LOG_INFO(nullptr, "Free workspace memory: " << ptr); +#ifdef _WIN32 + _aligned_free(ptr); + GAPI_LOG_INFO(nullptr, "Released workspace memory: " << ptr); + ptr = nullptr; +#else + GAPI_Assert(false && "Not implemented for systems differ than \"_WIN32\". " + "Please feel free to set it up under OPENCV contribution policy"); +#endif + + }); + size_t i = 0; + try { + for (; i < pool_size; i++) { + size_t preallocated_mem_offset = static_cast(i) * surface_size_bytes; + + surface_ptr_t surf_ptr = creator(workspace_mem_owner, + preallocated_mem_offset, + preallocated_raw_bytes); + pool.push_back(std::move(surf_ptr)); + } + } catch (const std::exception& ex) { + throw std::runtime_error(std::string("VPLCPUAccelerationPolicy::create_surface_pool - ") + + "cannot construct surface index: " + std::to_string(i) + ", error: " + + ex.what() + + "Requested surface count: " + std::to_string(pool_size) + + ", surface bytes: " + std::to_string(surface_size_bytes)); + } + + // remember pool by key + GAPI_LOG_INFO(nullptr, "New pool allocated, key: " << preallocated_pool_memory_ptr << + ", surface count: " << pool.size() << + ", surface size bytes: " << surface_size_bytes); + if (!pool_table.emplace(preallocated_pool_memory_ptr, std::move(pool)).second) { + GAPI_LOG_WARNING(nullptr, "Cannot insert pool, table size: " + std::to_string(pool_table.size()) << + ", key: " << preallocated_pool_memory_ptr << " exists"); + GAPI_Assert(false && "Cannot create pool in VPLCPUAccelerationPolicy"); + } + + return preallocated_pool_memory_ptr; +} + +VPLCPUAccelerationPolicy::surface_weak_ptr_t VPLCPUAccelerationPolicy::get_free_surface(pool_key_t key) { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + throw std::runtime_error("VPLCPUAccelerationPolicy::get_free_surface - " + "key is not found, table size: " + + std::to_string(pool_table.size())); + } + + pool_t& requested_pool = pool_it->second; +#ifdef TEST_PERF + return requested_pool.find_free(); +#else // TEST_PERF + auto it = + std::find_if(requested_pool.begin(), requested_pool.end(), + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + + // Limitation realloc pool might be a future extension + if (it == requested_pool.end()) { + std::stringstream ss; + ss << "cannot get free surface from pool, key: " << key << ", size: " << requested_pool.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + return *it; +#endif // TEST_PERF +} + +size_t VPLCPUAccelerationPolicy::get_free_surface_count(pool_key_t key) const { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + GAPI_LOG_WARNING(nullptr, "key is not found: " << key << + ", table size: " << pool_table.size()); + return 0; + } +#ifdef TEST_PERF + return 0; +#else // TEST_PERF + const pool_t& requested_pool = pool_it->second; + size_t free_surf_count = + std::count_if(requested_pool.begin(), requested_pool.end(), + [](const surface_ptr_t& val) { + GAPI_Assert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + return free_surf_count; +#endif // TEST_PERF +} + +size_t VPLCPUAccelerationPolicy::get_surface_count(pool_key_t key) const { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + GAPI_LOG_DEBUG(nullptr, "key is not found: " << key << + ", table size: " << pool_table.size()); + return 0; + } +#ifdef TEST_PERF + return 0; +#else // TEST_PERF + return pool_it->second.size(); +#endif // TEST_PERF +} + +cv::MediaFrame::AdapterPtr VPLCPUAccelerationPolicy::create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) { + auto pool_it = pool_table.find(key); + if (pool_it == pool_table.end()) { + std::stringstream ss; + ss << "key is not found: " << key << ", table size: " << pool_table.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + pool_t& requested_pool = pool_it->second; +#ifdef TEST_PERF + return cv::MediaFrame::AdapterPtr{new VPLMediaFrameCPUAdapter(requested_pool.find_by_handle(surface))}; +#else // TEST_PERF + auto it = + std::find_if(requested_pool.begin(), requested_pool.end(), + [surface](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return val->get_handle() == surface; + }); + + // Limitation realloc pool might be a future extension + if (it == requested_pool.end()) { + std::stringstream ss; + ss << "cannot get requested surface from pool, key: " << key << ", surf: " + << surface << ", pool size: " << requested_pool.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + + return cv::MediaFrame::AdapterPtr{new VPLMediaFrameCPUAdapter(*it)}; +#endif // TEST_PERF +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp new file mode 100644 index 0000000000..dc5a9347b5 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_cpu.hpp @@ -0,0 +1,57 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#ifdef TEST_PERF +#include "streaming/onevpl/accelerators/surface/surface_pool.hpp" +#endif // TEST_PERF + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS VPLCPUAccelerationPolicy final : public VPLAccelerationPolicy +{ + VPLCPUAccelerationPolicy(); + ~VPLCPUAccelerationPolicy(); +#ifdef TEST_PERF + using pool_t = CachedPool; +#else // TEST_PERF + using pool_t = std::vector; +#endif // TEST_PERF + + void init(session_t session) override; + void deinit(session_t session) override; + pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) override; + surface_weak_ptr_t get_free_surface(pool_key_t key) override; + size_t get_free_surface_count(pool_key_t key) const override; + size_t get_surface_count(pool_key_t key) const override; + + cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) override; + +private: + std::map pool_table; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_CPU_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp new file mode 100644 index 0000000000..cb27df8661 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.cpp @@ -0,0 +1,119 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL +#include "streaming/onevpl/accelerators/accel_policy_dx11.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +#define D3D11_NO_HELPERS +#include +#include +#include +#include "opencv2/core/directx.hpp" + +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +VPLDX11AccelerationPolicy::VPLDX11AccelerationPolicy() +{ +#ifdef CPU_ACCEL_ADAPTER + adapter.reset(new VPLCPUAccelerationPolicy); +#endif +} + +VPLDX11AccelerationPolicy::~VPLDX11AccelerationPolicy() +{ + if (hw_handle) + { + GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy release ID3D11Device"); + hw_handle->Release(); + } +} + +void VPLDX11AccelerationPolicy::init(session_t session) { + mfxStatus sts = MFXVideoCORE_GetHandle(session, MFX_HANDLE_D3D11_DEVICE, reinterpret_cast(&hw_handle)); + if (sts != MFX_ERR_NONE) + { + throw std::logic_error("Cannot create VPLDX11AccelerationPolicy, MFXVideoCORE_GetHandle error: " + + mfxstatus_to_string(sts)); + } + + GAPI_LOG_INFO(nullptr, "VPLDX11AccelerationPolicy initialized, session: " << session); +} + +void VPLDX11AccelerationPolicy::deinit(session_t session) { + GAPI_LOG_INFO(nullptr, "deinitialize session: " << session); +} + +VPLDX11AccelerationPolicy::pool_key_t +VPLDX11AccelerationPolicy::create_surface_pool(size_t pool_size, size_t surface_size_bytes, + surface_ptr_ctr_t creator) { + GAPI_LOG_DEBUG(nullptr, "pool size: " << pool_size << ", surface size bytes: " << surface_size_bytes); + +#ifdef CPU_ACCEL_ADAPTER + return adapter->create_surface_pool(pool_size, surface_size_bytes, creator); +#endif + (void)pool_size; + (void)surface_size_bytes; + (void)creator; + throw std::runtime_error("VPLDX11AccelerationPolicy::create_surface_pool() is not implemented"); +} + +VPLDX11AccelerationPolicy::surface_weak_ptr_t VPLDX11AccelerationPolicy::get_free_surface(pool_key_t key) +{ +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_free_surface(key); +#endif + (void)key; + throw std::runtime_error("VPLDX11AccelerationPolicy::get_free_surface() is not implemented"); +} + +size_t VPLDX11AccelerationPolicy::get_free_surface_count(pool_key_t key) const { +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_free_surface_count(key); +#endif + (void)key; + throw std::runtime_error("get_free_surface_count() is not implemented"); +} + +size_t VPLDX11AccelerationPolicy::get_surface_count(pool_key_t key) const { +#ifdef CPU_ACCEL_ADAPTER + return adapter->get_surface_count(key); +#endif + (void)key; + throw std::runtime_error("VPLDX11AccelerationPolicy::get_surface_count() is not implemented"); +} + +cv::MediaFrame::AdapterPtr VPLDX11AccelerationPolicy::create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) { + +#ifdef CPU_ACCEL_ADAPTER + return adapter->create_frame_adapter(key, surface); +#endif + (void)key; + (void)surface; + throw std::runtime_error("VPLDX11AccelerationPolicy::create_frame_adapter() is not implemented"); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp new file mode 100644 index 0000000000..a875f57085 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_dx11.hpp @@ -0,0 +1,69 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS +//TODO Remove the next MACRO right after DX11 implementation +#define CPU_ACCEL_ADAPTER + +#ifdef HAVE_ONEVPL +#include +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" + +#ifdef CPU_ACCEL_ADAPTER +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#endif + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#define D3D11_NO_HELPERS +#include +#include +#include "opencv2/core/directx.hpp" +#ifdef HAVE_OPENCL +#include +#endif + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS VPLDX11AccelerationPolicy final: public VPLAccelerationPolicy +{ + // GAPI_EXPORTS for tests + VPLDX11AccelerationPolicy(); + ~VPLDX11AccelerationPolicy(); + + void init(session_t session) override; + void deinit(session_t session) override; + pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) override; + surface_weak_ptr_t get_free_surface(pool_key_t key) override; + size_t get_free_surface_count(pool_key_t key) const override; + size_t get_surface_count(pool_key_t key) const override; + + cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) override; + +private: + ID3D11Device *hw_handle; + +#ifdef CPU_ACCEL_ADAPTER + std::unique_ptr adapter; +#endif +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_DX11_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp new file mode 100644 index 0000000000..31ee91535c --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/accel_policy_interface.hpp @@ -0,0 +1,61 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP + +#include +#include +#include + +#include + +#ifdef HAVE_ONEVPL +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class Surface; +struct VPLAccelerationPolicy +{ + virtual ~VPLAccelerationPolicy() {} + + using pool_key_t = void*; + + using session_t = mfxSession; + using surface_t = Surface; + using surface_ptr_t = std::shared_ptr; + using surface_weak_ptr_t = std::weak_ptr; + using surface_ptr_ctr_t = std::function out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_ptr_size)>; + + virtual void init(session_t session) = 0; + virtual void deinit(session_t session) = 0; + + // Limitation: cannot give guarantee in succesful memory realloccation + // for existing workspace in existing pool (see realloc) + // thus it is not implemented, + // PLEASE provide initial memory area large enough + virtual pool_key_t create_surface_pool(size_t pool_size, size_t surface_size_bytes, surface_ptr_ctr_t creator) = 0; + + virtual surface_weak_ptr_t get_free_surface(pool_key_t key) = 0; + virtual size_t get_free_surface_count(pool_key_t key) const = 0; + virtual size_t get_surface_count(pool_key_t key) const = 0; + + virtual cv::MediaFrame::AdapterPtr create_frame_adapter(pool_key_t key, + mfxFrameSurface1* surface) = 0; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_ACCEL_POLICY_INTERFACE_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp new file mode 100644 index 0000000000..d3020ab168 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -0,0 +1,119 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +VPLMediaFrameCPUAdapter::VPLMediaFrameCPUAdapter(std::shared_ptr surface): + parent_surface_ptr(surface) { + + GAPI_Assert(parent_surface_ptr && "Surface is nullptr"); + parent_surface_ptr->obtain_lock(); + + GAPI_LOG_DEBUG(nullptr, "surface: " << parent_surface_ptr->get_handle() << + ", w: " << parent_surface_ptr->get_info().Width << + ", h: " << parent_surface_ptr->get_info().Height << + ", p: " << parent_surface_ptr->get_data().Pitch); +} + +VPLMediaFrameCPUAdapter::~VPLMediaFrameCPUAdapter() { + + // Each VPLMediaFrameCPUAdapter releases mfx surface counter + // The last VPLMediaFrameCPUAdapter releases shared Surface pointer + // The last surface pointer releases workspace memory + parent_surface_ptr->release_lock(); +} + +cv::GFrameDesc VPLMediaFrameCPUAdapter::meta() const { + GFrameDesc desc; + const Surface::info_t& info = parent_surface_ptr->get_info(); + switch(info.FourCC) + { + case MFX_FOURCC_I420: + throw std::runtime_error("MediaFrame doesn't support I420 type"); + break; + case MFX_FOURCC_NV12: + desc.fmt = MediaFormat::NV12; + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } + + desc.size = cv::Size{info.Width, info.Height}; + return desc; +} + +MediaFrame::View VPLMediaFrameCPUAdapter::access(MediaFrame::Access) { + const Surface::data_t& data = parent_surface_ptr->get_data(); + const Surface::info_t& info = parent_surface_ptr->get_info(); + using stride_t = typename cv::MediaFrame::View::Strides::value_type; + + stride_t pitch = static_cast(data.Pitch); + switch(info.FourCC) { + case MFX_FOURCC_I420: + { + cv::MediaFrame::View::Ptrs pp = { + data.Y, + data.U, + data.V, + nullptr + }; + cv::MediaFrame::View::Strides ss = { + pitch, + pitch / 2, + pitch / 2, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + case MFX_FOURCC_NV12: + { + cv::MediaFrame::View::Ptrs pp = { + data.Y, + data.UV, nullptr, nullptr + }; + cv::MediaFrame::View::Strides ss = { + pitch, + pitch, 0u, 0u + }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + break; + default: + throw std::runtime_error("MediaFrame unknown 'fmt' type: " + std::to_string(info.FourCC)); + } +} + +cv::util::any VPLMediaFrameCPUAdapter::blobParams() const { + GAPI_Assert("VPLMediaFrameCPUAdapter::blobParams() is not implemented"); + return {}; +} + +void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) { + GAPI_Assert("VPLMediaFrameCPUAdapter::serialize() is not implemented"); +} + +void VPLMediaFrameCPUAdapter::deserialize(cv::gapi::s11n::IIStream&) { + GAPI_Assert("VPLMediaFrameCPUAdapter::deserialize() is not implemented"); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp new file mode 100644 index 0000000000..04a9bdc275 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp @@ -0,0 +1,43 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP +#include + +#include +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class Surface; +class VPLMediaFrameCPUAdapter : public cv::MediaFrame::IAdapter { +public: + // GAPI_EXPORTS for tests + GAPI_EXPORTS explicit VPLMediaFrameCPUAdapter(std::shared_ptr assoc_surface); + GAPI_EXPORTS ~VPLMediaFrameCPUAdapter(); + cv::GFrameDesc meta() const override; + MediaFrame::View access(MediaFrame::Access) override; + + // The default implementation does nothing + cv::util::any blobParams() const override; + void serialize(cv::gapi::s11n::IOStream&) override; + void deserialize(cv::gapi::s11n::IIStream&) override; +private: + std::shared_ptr parent_surface_ptr; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_CPU_FRAME_ADAPTER_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp new file mode 100644 index 0000000000..3f5fd00305 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.cpp @@ -0,0 +1,78 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +Surface::Surface(std::unique_ptr&& surf, std::shared_ptr associated_memory) : + workspace_memory_ptr(associated_memory), + mfx_surface(std::move(surf)), + mirrored_locked_count() { + + GAPI_Assert(mfx_surface && "Surface is nullptr"); + mirrored_locked_count.store(mfx_surface->Data.Locked); + GAPI_LOG_DEBUG(nullptr, "create surface: " << mfx_surface << + ", locked count: " << mfx_surface->Data.Locked); +} + +Surface::~Surface() { + GAPI_LOG_DEBUG(nullptr, "destroy surface: " << mfx_surface << + ", worspace memory counter: " << workspace_memory_ptr.use_count()); +} + +std::shared_ptr Surface::create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory) { + surface_ptr_t ret {new Surface(std::move(surf), accociated_memory)}; + return ret; +} + +Surface::handle_t* Surface::get_handle() const { + return mfx_surface.get(); +} + +const Surface::info_t& Surface::get_info() const { + return mfx_surface->Info; +} + +const Surface::data_t& Surface::get_data() const { + return mfx_surface->Data; +} + +size_t Surface::get_locks_count() const { + return mirrored_locked_count.load(); +} + +size_t Surface::obtain_lock() { + size_t locked_count = mirrored_locked_count.fetch_add(1); + GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); + mfx_surface->Data.Locked = static_cast(locked_count + 1); + GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << + ", locked times: " << locked_count + 1); + return locked_count; // return preceding value +} + +size_t Surface::release_lock() { + size_t locked_count = mirrored_locked_count.fetch_sub(1); + GAPI_Assert(locked_count < std::numeric_limits::max() && "Too many references "); + GAPI_Assert(locked_count && "Surface lock counter is invalid"); + mfx_surface->Data.Locked = static_cast(locked_count - 1); + GAPI_LOG_DEBUG(nullptr, "surface: " << mfx_surface.get() << + ", locked times: " << locked_count - 1); + return locked_count; // return preceding value +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp new file mode 100644 index 0000000000..828e5cb1c7 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface.hpp @@ -0,0 +1,104 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP +#define GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP + +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +/** + * @brief Inner class for managing oneVPL surface through interface `mfxFrameSurface1`. + * + * Surface has no own memory and shares accelerator allocated memory using reference counter semantics. + * So it lives till a last memory consumer (surface/accelerator/media frame) lives. + * This approach allows to support different scenarious in releasing allocated memory + * + * VPL surface `mfxFrameSurface1` support Lock-Free semantics and application MUST NOT operate with a + * surface in locked state. But VPL inner counter is not threadsafe so it would be failed in any concurrent scenario. + * std::atomic counter introduced in a way to overcome this problem. + * But only few scenarious for concurrency are supported here because it is not assumed to implement entire Surface in + * for a fully multithread approach. + * Supported concurrency scenarios deal only with transaction pair: @ref Surface::get_locks_count() against + * @ref Surface::release_lock() - which may be called from different threads. On the other hand @ref Surface::get_locks_count() against + * @ref Surface::obtain_lock() happens in single thread only. Surface doesn't support shared ownership that + * because it doesn't require thread safe guarantee between transactions: + * - @ref Surface::obtain_lock() against @ref Surface::obtain_lock() + * - @ref Surface::obtain_lock() against @ref Surface::release_lock() + * - @ref Surface::release_lock() against @ref Surface::release_lock() + */ +class Surface { + using handle_t = mfxFrameSurface1; + + std::shared_ptr workspace_memory_ptr; + std::unique_ptr mfx_surface; + std::atomic mirrored_locked_count; +public: + using info_t = mfxFrameInfo; + using data_t = mfxFrameData; + + // GAPI_EXPORTS for tests + GAPI_EXPORTS static std::shared_ptr create_surface(std::unique_ptr&& surf, + std::shared_ptr accociated_memory); + GAPI_EXPORTS ~Surface(); + + GAPI_EXPORTS handle_t* get_handle() const; + GAPI_EXPORTS const info_t& get_info() const; + GAPI_EXPORTS const data_t& get_data() const; + + /** + * Extract value thread-safe lock counter (see @ref Surface description). + * It's usual situation that counter may be instantly decreased in other thread after this method called. + * We need instantaneous value. This method syncronized in inter-threading way with @ref Surface::release_lock() + * + * @return fetched locks count. + */ + GAPI_EXPORTS size_t get_locks_count() const; + + /** + * Atomically increase value of thread-safe lock counter (see @ref Surface description). + * This method is single-threaded happens-after @ref Surface::get_locks_count() and + * multi-threaded happens-before @ref Surface::release_lock() + * + * @return locks count just before its increasing. + */ + GAPI_EXPORTS size_t obtain_lock(); + + /** + * Atomically decrease value of thread-safe lock counter (see @ref Surface description). + * This method is synchronized with @ref Surface::get_locks_count() and + * multi-threaded happens-after @ref Surface::obtain_lock() + * + * @return locks count just before its decreasing. + */ + GAPI_EXPORTS size_t release_lock(); +private: + Surface(std::unique_ptr&& surf, std::shared_ptr accociated_memory); +}; + +using surface_ptr_t = std::shared_ptr; +using surface_weak_ptr_t = std::weak_ptr; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ACCELERATORS_SURFACE_HPP diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp new file mode 100644 index 0000000000..729b37f7ea --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.cpp @@ -0,0 +1,71 @@ +#include "streaming/onevpl/accelerators/surface/surface_pool.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +void CachedPool::reserve(size_t size) { + surfaces.reserve(size); +} + +size_t CachedPool::size() const { + return surfaces.size(); +} + +void CachedPool::clear() { + surfaces.clear(); + next_free_it = surfaces.begin(); + cache.clear(); +} + +void CachedPool::push_back(surface_ptr_t &&surf) { + cache.insert(std::make_pair(surf->get_handle(), surf)); + surfaces.push_back(std::move(surf)); + next_free_it = surfaces.begin(); +} + +CachedPool::surface_ptr_t CachedPool::find_free() { + auto it = + std::find_if(next_free_it, surfaces.end(), + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + + // Limitation realloc pool might be a future extension + if (it == surfaces.end()) { + it = std::find_if(surfaces.begin(), next_free_it, + [](const surface_ptr_t& val) { + GAPI_DbgAssert(val && "Pool contains empty surface"); + return !val->get_locks_count(); + }); + if (it == next_free_it) { + std::stringstream ss; + ss << "cannot get free surface from pool, size: " << surfaces.size(); + const std::string& str = ss.str(); + GAPI_LOG_WARNING(nullptr, str); + throw std::runtime_error(std::string(__FUNCTION__) + " - " + str); + } + } + + next_free_it = it; + ++next_free_it; + + return *it; +} + +CachedPool::surface_ptr_t CachedPool::find_by_handle(mfxFrameSurface1* handle) { + auto it = cache.find(handle); + GAPI_Assert(it != cache.end() && "Cannot find cached surface from pool. Data corruption is possible"); + return it->second; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp new file mode 100644 index 0000000000..029fa350db --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/surface_pool.hpp @@ -0,0 +1,47 @@ +#ifndef GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP +#define GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP + +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class Surface; +// GAPI_EXPORTS for tests +class GAPI_EXPORTS CachedPool { +public: + using surface_ptr_t = std::shared_ptr; + using surface_container_t = std::vector; + using free_surface_iterator_t = typename surface_container_t::iterator; + using cached_surface_container_t = std::map; + + void push_back(surface_ptr_t &&surf); + void reserve(size_t size); + size_t size() const; + void clear(); + surface_ptr_t find_free(); + surface_ptr_t find_by_handle(mfxFrameSurface1* handle); +private: + surface_container_t surfaces; + free_surface_iterator_t next_free_it; + cached_surface_container_t cache; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_SURFACE_SURFACE_POOL_HPP diff --git a/modules/gapi/src/streaming/onevpl/cfg_params.cpp b/modules/gapi/src/streaming/onevpl/cfg_params.cpp new file mode 100644 index 0000000000..456725508f --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/cfg_params.cpp @@ -0,0 +1,137 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +namespace util { +struct variant_comparator : cv::util::static_visitor { + variant_comparator(const CfgParam::value_t& rhs_value) : + rhs(rhs_value) {} + + template + bool visit(const ValueType& lhs) const { + return lhs < cv::util::get(rhs); + } +private: + const CfgParam::value_t& rhs; +}; +} // namespace util + +struct CfgParam::Priv { + Priv(const std::string& param_name, CfgParam::value_t&& param_value, bool is_major_param) : + name(param_name), value(std::forward(param_value)), major_flag(is_major_param) { + } + + const CfgParam::name_t& get_name_impl() const { + return name; + } + + const CfgParam::value_t& get_value_impl() const { + return value; + } + + bool is_major_impl() const { + return major_flag; + } + + // comparison implementation + bool operator< (const Priv& rhs) const { + // implement default pair comparison + if (get_name_impl() < rhs.get_name_impl()) { + return true; + } else if (get_name_impl() > rhs.get_name_impl()) { + return false; + } + + //TODO implement operator < for cv::util::variant + const CfgParam::value_t& lvar = get_value_impl(); + const CfgParam::value_t& rvar = rhs.get_value_impl(); + if (lvar.index() < rvar.index()) { + return true; + } else if (lvar.index() > rvar.index()) { + return false; + } + + util::variant_comparator comp(rvar); + return cv::util::visit(comp, lvar); + } + + bool operator==(const Priv& rhs) const { + return (get_name_impl() == rhs.get_name_impl()) + && (get_value_impl() == rhs.get_value_impl()); + } + + bool operator!=(const Priv& rhs) const { + return !(*this == rhs); + } + + CfgParam::name_t name; + CfgParam::value_t value; + bool major_flag; +}; + +CfgParam::CfgParam (const std::string& param_name, value_t&& param_value, bool is_major_param) : + m_priv(new Priv(param_name, std::move(param_value), is_major_param)) { +} + +CfgParam::~CfgParam() = default; + +CfgParam& CfgParam::operator=(const CfgParam& src) { + if (this != &src) { + m_priv = src.m_priv; + } + return *this; +} + +CfgParam& CfgParam::operator=(CfgParam&& src) { + if (this != &src) { + m_priv = std::move(src.m_priv); + } + return *this; +} + +CfgParam::CfgParam(const CfgParam& src) : + m_priv(src.m_priv) { +} + +CfgParam::CfgParam(CfgParam&& src) : + m_priv(std::move(src.m_priv)) { +} + +const CfgParam::name_t& CfgParam::get_name() const { + return m_priv->get_name_impl(); +} + +const CfgParam::value_t& CfgParam::get_value() const { + return m_priv->get_value_impl(); +} + +bool CfgParam::is_major() const { + return m_priv->is_major_impl(); +} + +bool CfgParam::operator< (const CfgParam& rhs) const { + return *m_priv < *rhs.m_priv; +} + +bool CfgParam::operator==(const CfgParam& rhs) const { + return *m_priv == *rhs.m_priv; +} + +bool CfgParam::operator!=(const CfgParam& rhs) const { + return *m_priv != *rhs.m_priv; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp b/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp new file mode 100644 index 0000000000..feaae7a727 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/data_provider_interface_exception.cpp @@ -0,0 +1,22 @@ +#include +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +DataProviderSystemErrorException::DataProviderSystemErrorException(int error_code, const std::string& desription) { + reason = desription + ", error: " + std::to_string(error_code) + ", desctiption: " + strerror(error_code); +} + +DataProviderSystemErrorException::~DataProviderSystemErrorException() = default; + +const char* DataProviderSystemErrorException::what() const noexcept { + return reason.c_str(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp new file mode 100644 index 0000000000..ad4a4eca8e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp @@ -0,0 +1,310 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include +#include + +#include + +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/engine/decode/decode_session.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +/* UTILS */ +mfxU32 GetSurfaceSize(mfxU32 FourCC, mfxU32 width, mfxU32 height) { + mfxU32 nbytes = 0; + + mfxU32 half_width = width / 2; + mfxU32 half_height = height / 2; + switch (FourCC) { + case MFX_FOURCC_I420: + case MFX_FOURCC_NV12: + nbytes = width * height + 2 * half_width * half_height; + break; + case MFX_FOURCC_I010: + case MFX_FOURCC_P010: + nbytes = width * height + 2 * half_width * half_height; + nbytes *= 2; + break; + case MFX_FOURCC_RGB4: + nbytes = width * height * 4; + break; + default: + GAPI_LOG_WARNING(nullptr, "Unsupported FourCC requested: " << FourCC); + GAPI_Assert(false && "Unsupported FourCC requested"); + break; + } + return nbytes; +} + +surface_ptr_t create_surface_RGB4(mfxFrameInfo frameInfo, + std::shared_ptr out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_size) +{ + mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); + mfxU16 surfW = frameInfo.Width * 4; + mfxU16 surfH = frameInfo.Height; + (void)surfH; + + // TODO more intelligent check + if (out_buf_size <= out_buf_ptr_offset) { + throw std::runtime_error(std::string("Insufficient buffer size: ") + + std::to_string(out_buf_size) + ", buffer offset: " + + std::to_string(out_buf_ptr_offset) + + ", expected surface width: " + std::to_string(surfW) + + ", height: " + std::to_string(surfH)); + } + + std::unique_ptr handle(new mfxFrameSurface1); + memset(handle.get(), 0, sizeof(mfxFrameSurface1)); + + handle->Info = frameInfo; + handle->Data.B = buf + out_buf_ptr_offset; + handle->Data.G = handle->Data.B + 1; + handle->Data.R = handle->Data.B + 2; + handle->Data.A = handle->Data.B + 3; + handle->Data.Pitch = surfW; + + return Surface::create_surface(std::move(handle), out_buf_ptr); +} + +surface_ptr_t create_surface_other(mfxFrameInfo frameInfo, + std::shared_ptr out_buf_ptr, + size_t out_buf_ptr_offset, + size_t out_buf_size) +{ + mfxU8* buf = reinterpret_cast(out_buf_ptr.get()); + mfxU16 surfH = frameInfo.Height; + mfxU16 surfW = (frameInfo.FourCC == MFX_FOURCC_P010) ? frameInfo.Width * 2 : frameInfo.Width; + + // TODO more intelligent check + if (out_buf_size <= + out_buf_ptr_offset + (surfW * surfH) + ((surfW / 2) * (surfH / 2))) { + throw std::runtime_error(std::string("Insufficient buffer size: ") + + std::to_string(out_buf_size) + ", buffer offset: " + + std::to_string(out_buf_ptr_offset) + + ", expected surface width: " + std::to_string(surfW) + + ", height: " + std::to_string(surfH)); + } + + std::unique_ptr handle(new mfxFrameSurface1); + memset(handle.get(), 0, sizeof(mfxFrameSurface1)); + + handle->Info = frameInfo; + handle->Data.Y = buf + out_buf_ptr_offset; + handle->Data.U = buf + out_buf_ptr_offset + (surfW * surfH); + handle->Data.V = handle->Data.U + ((surfW / 2) * (surfH / 2)); + handle->Data.Pitch = surfW; + + return Surface::create_surface(std::move(handle), out_buf_ptr); +} + +VPLLegacyDecodeEngine::VPLLegacyDecodeEngine(std::unique_ptr&& accel) + : ProcessingEngineBase(std::move(accel)) { + + GAPI_LOG_INFO(nullptr, "Create Legacy Decode Engine"); + create_pipeline( + // 1) Read File + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyDecodeSession &my_sess = static_cast(sess); + my_sess.last_status = ReadEncodedStream(my_sess.stream, my_sess.data_provider); + if (my_sess.last_status != MFX_ERR_NONE) { + my_sess.data_provider.reset(); //close source + } + return ExecutionStatus::Continue; + }, + // 2) enqueue ASYNC decode + [this] (EngineSession& sess) -> ExecutionStatus + { + LegacyDecodeSession &my_sess = static_cast(sess); + + my_sess.last_status = + MFXVideoDECODE_DecodeFrameAsync(my_sess.session, + my_sess.last_status == MFX_ERR_NONE + ? &my_sess.stream + : nullptr, /* No more data to read, start decode draining mode*/ + my_sess.procesing_surface_ptr.lock()->get_handle(), + &my_sess.output_surface_ptr, + &my_sess.sync); + return ExecutionStatus::Continue; + }, + // 3) Wait for ASYNC decode result + [this] (EngineSession& sess) -> ExecutionStatus + { + if (sess.last_status == MFX_ERR_NONE) // Got 1 decoded frame + { + do { + //TODO try to extract TIMESTAMP + sess.last_status = MFXVideoCORE_SyncOperation(sess.session, sess.sync, 100); + if (MFX_ERR_NONE == sess.last_status) { + + LegacyDecodeSession& my_sess = static_cast(sess); + on_frame_ready(my_sess); + } + } while (sess.last_status == MFX_WRN_IN_EXECUTION); + } + return ExecutionStatus::Continue; + }, + // 4) Falls back on generic status procesing + [this] (EngineSession& sess) -> ExecutionStatus + { + return this->process_error(sess.last_status, static_cast(sess)); + } + ); +} + +void VPLLegacyDecodeEngine::initialize_session(mfxSession mfx_session, + DecoderParams&& decoder_param, + std::shared_ptr provider) +{ + mfxFrameAllocRequest decRequest = {}; + // Query number required surfaces for decoder + MFXVideoDECODE_QueryIOSurf(mfx_session, &decoder_param.param, &decRequest); + + // External (application) allocation of decode surfaces + GAPI_LOG_DEBUG(nullptr, "Query IOSurf for session: " << mfx_session << + ", mfxFrameAllocRequest.NumFrameSuggested: " << decRequest.NumFrameSuggested << + ", mfxFrameAllocRequest.Type: " << decRequest.Type); + + mfxU32 singleSurfaceSize = GetSurfaceSize(decoder_param.param.mfx.FrameInfo.FourCC, + decoder_param.param.mfx.FrameInfo.Width, + decoder_param.param.mfx.FrameInfo.Height); + if (!singleSurfaceSize) { + throw std::runtime_error("Cannot determine surface size for: fourCC" + + std::to_string(decoder_param.param.mfx.FrameInfo.FourCC) + + ", width: " + std::to_string(decoder_param.param.mfx.FrameInfo.Width) + + ", height: " + std::to_string(decoder_param.param.mfx.FrameInfo.Height)); + } + + const auto &frameInfo = decoder_param.param.mfx.FrameInfo; + auto surface_creator = + [&frameInfo] (std::shared_ptr out_buf_ptr, size_t out_buf_ptr_offset, + size_t out_buf_size) -> surface_ptr_t { + return (frameInfo.FourCC == MFX_FOURCC_RGB4) ? + create_surface_RGB4(frameInfo, out_buf_ptr, out_buf_ptr_offset, + out_buf_size) : + create_surface_other(frameInfo, out_buf_ptr, out_buf_ptr_offset, + out_buf_size);}; + + //TODO Configure preallocation size (how many frames we can hold) + const size_t preallocated_frames_count = 30; + VPLAccelerationPolicy::pool_key_t decode_pool_key = + acceleration_policy->create_surface_pool(decRequest.NumFrameSuggested * preallocated_frames_count, + singleSurfaceSize, + surface_creator); + + // create session + std::shared_ptr sess_ptr = + register_session(mfx_session, + std::move(decoder_param), + provider); + + sess_ptr->init_surface_pool(decode_pool_key); + // prepare working decode surface + sess_ptr->swap_surface(*this); +} + +ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::execute_op(operation_t& op, EngineSession& sess) { + return op(sess); +} + +void VPLLegacyDecodeEngine::on_frame_ready(LegacyDecodeSession& sess) +{ + GAPI_LOG_DEBUG(nullptr, "[" << sess.session << "], frame ready"); + + // manage memory ownership rely on acceleration policy + auto frame_adapter = acceleration_policy->create_frame_adapter(sess.decoder_pool_id, + sess.output_surface_ptr); + ready_frames.emplace(cv::MediaFrame(std::move(frame_adapter)), sess.generate_frame_meta()); +} + +ProcessingEngineBase::ExecutionStatus VPLLegacyDecodeEngine::process_error(mfxStatus status, LegacyDecodeSession& sess) +{ + GAPI_LOG_DEBUG(nullptr, "status: " << mfxstatus_to_string(status)); + + switch (status) { + case MFX_ERR_NONE: + return ExecutionStatus::Continue; + case MFX_ERR_MORE_DATA: // The function requires more bitstream at input before decoding can proceed + if (!sess.data_provider || sess.data_provider->empty()) { + // No more data to drain from decoder, start encode draining mode + return ExecutionStatus::Processed; + } + else + return ExecutionStatus::Continue; // read more data + break; + case MFX_ERR_MORE_SURFACE: + { + // The function requires more frame surface at output before decoding can proceed. + // This applies to external memory allocations and should not be expected for + // a simple internal allocation case like this + try { + sess.swap_surface(*this); + return ExecutionStatus::Continue; + } catch (const std::exception& ex) { + GAPI_LOG_WARNING(nullptr, "[" << sess.session << "] error: " << ex.what()); + } + break; + } + case MFX_ERR_DEVICE_LOST: + // For non-CPU implementations, + // Cleanup if device is lost + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_DEVICE_LOST is not processed"); + break; + case MFX_WRN_DEVICE_BUSY: + // For non-CPU implementations, + // Wait a few milliseconds then try again + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_WRN_DEVICE_BUSY is not processed"); + break; + case MFX_WRN_VIDEO_PARAM_CHANGED: + // The decoder detected a new sequence header in the bitstream. + // Video parameters may have changed. + // In external memory allocation case, might need to reallocate the output surface + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_WRN_VIDEO_PARAM_CHANGED is not processed"); + break; + case MFX_ERR_INCOMPATIBLE_VIDEO_PARAM: + // The function detected that video parameters provided by the application + // are incompatible with initialization parameters. + // The application should close the component and then reinitialize it + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_INCOMPATIBLE_VIDEO_PARAM is not processed"); + break; + case MFX_ERR_REALLOC_SURFACE: + // Bigger surface_work required. May be returned only if + // mfxInfoMFX::EnableReallocRequest was set to ON during initialization. + // This applies to external memory allocations and should not be expected for + // a simple internal allocation case like this + GAPI_DbgAssert(false && "VPLLegacyDecodeEngine::process_error - " + "MFX_ERR_REALLOC_SURFACE is not processed"); + break; + default: + GAPI_LOG_WARNING(nullptr, "Unknown status code: " << mfxstatus_to_string(status)); + break; + } + + return ExecutionStatus::Failed; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp new file mode 100644 index 0000000000..5db54c3199 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.hpp @@ -0,0 +1,48 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP +#define GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP +#include +#include + +#include "streaming/onevpl/engine/processing_engine_base.hpp" + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) + #include +#endif +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +class LegacyDecodeSession; +struct DecoderParams; +struct IDataProvider; +struct VPLAccelerationPolicy; + +class VPLLegacyDecodeEngine : public ProcessingEngineBase { +public: + + VPLLegacyDecodeEngine(std::unique_ptr&& accel); + void initialize_session(mfxSession mfx_session, DecoderParams&& decoder_param, + std::shared_ptr provider) override; + +private: + ExecutionStatus execute_op(operation_t& op, EngineSession& sess) override; + ExecutionStatus process_error(mfxStatus status, LegacyDecodeSession& sess); + + void on_frame_ready(LegacyDecodeSession& sess); +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONVPL_DECODE_ENGINE_LEGACY_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp new file mode 100644 index 0000000000..3468869e15 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.cpp @@ -0,0 +1,78 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include +#include + +#include "streaming/onevpl/engine/decode/decode_session.hpp" +#include "streaming/onevpl/engine/decode/decode_engine_legacy.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/utils.hpp" + +#include "logger.hpp" +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +LegacyDecodeSession::LegacyDecodeSession(mfxSession sess, + DecoderParams&& decoder_param, + std::shared_ptr provider) : + EngineSession(sess, std::move(decoder_param.stream)), + mfx_decoder_param(std::move(decoder_param.param)), + data_provider(std::move(provider)), + procesing_surface_ptr(), + output_surface_ptr(), + decoded_frames_count() +{ +} + +LegacyDecodeSession::~LegacyDecodeSession() +{ + GAPI_LOG_INFO(nullptr, "Close Decode for session: " << session); + MFXVideoDECODE_Close(session); +} + +void LegacyDecodeSession::swap_surface(VPLLegacyDecodeEngine& engine) { + VPLAccelerationPolicy* acceleration_policy = engine.get_accel(); + GAPI_Assert(acceleration_policy && "Empty acceleration_policy"); + auto old_locked = procesing_surface_ptr.lock(); + try { + auto cand = acceleration_policy->get_free_surface(decoder_pool_id).lock(); + + GAPI_LOG_DEBUG(nullptr, "[" << session << "] swap surface" + ", old: " << (old_locked ? old_locked->get_handle() : nullptr) << + ", new: "<< cand->get_handle()); + + procesing_surface_ptr = cand; + } catch (const std::exception& ex) { + GAPI_LOG_WARNING(nullptr, "[" << session << "] error: " << ex.what() << + "Abort"); + } +} + +void LegacyDecodeSession::init_surface_pool(VPLAccelerationPolicy::pool_key_t key) { + GAPI_Assert(key && "Init decode pull with empty key"); + decoder_pool_id = key; +} + +Data::Meta LegacyDecodeSession::generate_frame_meta() { + const auto now = std::chrono::system_clock::now(); + const auto dur = std::chrono::duration_cast + (now.time_since_epoch()); + Data::Meta meta { + {cv::gapi::streaming::meta_tag::timestamp, int64_t{dur.count()} }, + {cv::gapi::streaming::meta_tag::seq_id, int64_t{decoded_frames_count++}} + }; + return meta; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp new file mode 100644 index 0000000000..46b1decc81 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_session.hpp @@ -0,0 +1,60 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP +#define GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP +#include +#include + +#include + +#include "streaming/onevpl/engine/engine_session.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) + #include +#endif +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct IDataProvider; +class Surface; +struct VPLAccelerationPolicy; + +class LegacyDecodeSession : public EngineSession { +public: + friend class VPLLegacyDecodeEngine; + + LegacyDecodeSession(mfxSession sess, DecoderParams&& decoder_param, std::shared_ptr provider); + ~LegacyDecodeSession(); + using EngineSession::EngineSession; + + void swap_surface(VPLLegacyDecodeEngine& engine); + void init_surface_pool(VPLAccelerationPolicy::pool_key_t key); + + mfxVideoParam mfx_decoder_param; + std::shared_ptr data_provider; + + Data::Meta generate_frame_meta(); +private: + VPLAccelerationPolicy::pool_key_t decoder_pool_id; + mfxFrameAllocRequest request; + + std::weak_ptr procesing_surface_ptr; + mfxFrameSurface1* output_surface_ptr; + + int64_t decoded_frames_count; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONVPL_ENGINE_DECODE_DECODE_SESSION_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp b/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp new file mode 100644 index 0000000000..9f8028361a --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/engine_session.cpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation +#ifdef HAVE_ONEVPL + +#include "streaming/onevpl/engine/engine_session.hpp" +#include "streaming/onevpl/utils.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +EngineSession::EngineSession(mfxSession sess, mfxBitstream&& str) : + session(sess), stream(std::move(str)) {} +EngineSession::~EngineSession() +{ + GAPI_LOG_INFO(nullptr, "Close session: " << session); + MFXClose(session); +} + +std::string EngineSession::error_code_to_str() const +{ + return mfxstatus_to_string(last_status); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp new file mode 100644 index 0000000000..d8f2f6b312 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/engine_session.hpp @@ -0,0 +1,49 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP +#define GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP + +#include +#include +#include +#include +#include +#include + +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +#ifdef HAVE_ONEVPL +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +// GAPI_EXPORTS for tests +struct GAPI_EXPORTS DecoderParams { + mfxBitstream stream; + mfxVideoParam param; +}; + +struct GAPI_EXPORTS EngineSession { + mfxSession session; + mfxBitstream stream; + mfxSyncPoint sync; + mfxStatus last_status; + + EngineSession(mfxSession sess, mfxBitstream&& str); + std::string error_code_to_str() const; + virtual ~EngineSession(); +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ENGINE_ENGINE_SESSION_HPP diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp new file mode 100644 index 0000000000..3161b1627e --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.cpp @@ -0,0 +1,134 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifdef HAVE_ONEVPL + +#include + +#include +#include "streaming/onevpl/engine/processing_engine_base.hpp" +#include "streaming/onevpl/accelerators/accel_policy_interface.hpp" +#include "logger.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +ProcessingEngineBase::ProcessingEngineBase(std::unique_ptr&& accel) : + acceleration_policy(std::move(accel)) { +} + +ProcessingEngineBase::~ProcessingEngineBase() { + GAPI_LOG_INFO(nullptr, "destroyed"); +} + +ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::process(mfxSession session) { + auto sess_it = sessions.find(session); + if (sess_it == sessions.end()) { + return ExecutionStatus::SessionNotFound; + } + + session_ptr processing_session = sess_it->second; + ExecutionData& exec_data = execution_table[session]; + + GAPI_LOG_DEBUG(nullptr, "[" << session <<"] start op id: " << exec_data.op_id); + ExecutionStatus status = execute_op(pipeline.at(exec_data.op_id), *processing_session); + size_t old_op_id = exec_data.op_id++; + if (exec_data.op_id == pipeline.size()) + { + exec_data.op_id = 0; + } + GAPI_LOG_DEBUG(nullptr, "[" << session <<"] finish op id: " << old_op_id << + ", " << processing_session->error_code_to_str() << + ", " << ProcessingEngineBase::status_to_string(status) << + ", next op id: " << exec_data.op_id); + + if (status == ExecutionStatus::Failed) { + + GAPI_LOG_WARNING(nullptr, "Operation for session: " << session << + ", " << ProcessingEngineBase::status_to_string(status) << + " - remove it"); + sessions.erase(sess_it); + execution_table.erase(session); + } + + if (status == ExecutionStatus::Processed) { + sessions.erase(sess_it); + execution_table.erase(session); + } + + return status; +} + +const char* ProcessingEngineBase::status_to_string(ExecutionStatus status) +{ + switch(status) { + case ExecutionStatus::Continue: return "CONTINUE"; + case ExecutionStatus::Processed: return "PROCESSED"; + case ExecutionStatus::SessionNotFound: return "NOT_FOUND_SESSION"; + case ExecutionStatus::Failed: return "FAILED"; + default: + return "UNKNOWN"; + } +} + +ProcessingEngineBase::ExecutionStatus ProcessingEngineBase::execute_op(operation_t& op, EngineSession& sess) +{ + return op(sess); +} + +size_t ProcessingEngineBase::get_ready_frames_count() const +{ + return ready_frames.size(); +} + +void ProcessingEngineBase::get_frame(Data &data) +{ + data = ready_frames.front(); + ready_frames.pop(); +} + +const VPLAccelerationPolicy* ProcessingEngineBase::get_accel() const { + return acceleration_policy.get(); +} + +VPLAccelerationPolicy* ProcessingEngineBase::get_accel() { + return const_cast(static_cast(this)->get_accel()); +} + + +// Read encoded stream from file +mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& data_provider) { + + if (!data_provider) { + return MFX_ERR_MORE_DATA; + } + + mfxU8 *p0 = bs.Data; + mfxU8 *p1 = bs.Data + bs.DataOffset; + if (bs.DataOffset > bs.MaxLength - 1) { + return MFX_ERR_NOT_ENOUGH_BUFFER; + } + if (bs.DataLength + bs.DataOffset > bs.MaxLength) { + return MFX_ERR_NOT_ENOUGH_BUFFER; + } + + std::copy_n(p0, bs.DataLength, p1); + + bs.DataOffset = 0; + bs.DataLength += static_cast(data_provider->fetch_data(bs.MaxLength - bs.DataLength, + bs.Data + bs.DataLength)); + if (bs.DataLength == 0) + return MFX_ERR_MORE_DATA; + + return MFX_ERR_NONE; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp new file mode 100644 index 0000000000..7d4869ac66 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/engine/processing_engine_base.hpp @@ -0,0 +1,96 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP +#define GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP + +#include +#include "streaming/onevpl/engine/engine_session.hpp" +#include "opencv2/gapi/own/exports.hpp" // GAPI_EXPORTS + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct VPLAccelerationPolicy; +struct IDataProvider; + +// GAPI_EXPORTS for tests +class GAPI_EXPORTS ProcessingEngineBase { +public: + enum class ExecutionStatus { + Continue, + Processed, + SessionNotFound, + Failed + }; + struct ExecutionData { + size_t op_id = 0; + }; + + using file_ptr = std::unique_ptr; + + using session_ptr = std::shared_ptr; + using SessionsTable = std::map; + using ExecutionDataTable = std::map; + + using frame_t = cv::gapi::wip::Data; + using frames_container_t = std::queue; + using operation_t = std::function; + + static const char * status_to_string(ExecutionStatus); + + ProcessingEngineBase(std::unique_ptr&& accel); + virtual ~ProcessingEngineBase(); + + virtual void initialize_session(mfxSession mfx_session, + DecoderParams&& decoder_param, + std::shared_ptr provider) = 0; + + ExecutionStatus process(mfxSession session); + size_t get_ready_frames_count() const; + void get_frame(Data &data); + + const VPLAccelerationPolicy* get_accel() const; + VPLAccelerationPolicy* get_accel(); +protected: + SessionsTable sessions; + frames_container_t ready_frames; + ExecutionDataTable execution_table; + + std::vector pipeline; + std::unique_ptr acceleration_policy; + + virtual ExecutionStatus execute_op(operation_t& op, EngineSession& sess); + + template + void create_pipeline(Ops&&...ops) + { + GAPI_DbgAssert(pipeline.empty() && "Pipeline must be empty"); + std::vector({std::forward(ops)...}).swap(pipeline); + } + + template + std::shared_ptr register_session(mfxSession key, + SessionArgs&& ...args) + { + auto sess_impl = std::make_shared(key, + std::forward(args)...); + sessions.emplace(key, sess_impl); + execution_table.emplace(key, ExecutionData{}); + return sess_impl; + } +}; + + +mfxStatus ReadEncodedStream(mfxBitstream &bs, std::shared_ptr& data_provider); +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ENGINE_PROCESSING_ENGINE_BASE_HPP diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.cpp b/modules/gapi/src/streaming/onevpl/file_data_provider.cpp new file mode 100644 index 0000000000..2a3df07c51 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.cpp @@ -0,0 +1,47 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation +#include + +#include "streaming/onevpl/file_data_provider.hpp" + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +FileDataProvider::FileDataProvider(const std::string& file_path) : + source_handle(fopen(file_path.c_str(), "rb"), &fclose) { + if (!source_handle) { + throw DataProviderSystemErrorException(errno, + "FileDataProvider: cannot open source file: " + file_path); + } +} + +FileDataProvider::~FileDataProvider() = default; + +size_t FileDataProvider::fetch_data(size_t out_data_bytes_size, void* out_data) { + if (empty()) { + return 0; + } + + size_t ret = fread(out_data, 1, out_data_bytes_size, source_handle.get()); + if (ret == 0) { + if (feof(source_handle.get())) { + source_handle.reset(); + } else { + throw DataProviderSystemErrorException (errno, "FileDataProvider::fetch_data error read"); + } + } + return ret; +} + +bool FileDataProvider::empty() const { + return !source_handle; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/file_data_provider.hpp b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp new file mode 100644 index 0000000000..22348290be --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/file_data_provider.hpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +struct FileDataProvider : public IDataProvider { + + using file_ptr = std::unique_ptr; + FileDataProvider(const std::string& file_path); + ~FileDataProvider(); + + size_t fetch_data(size_t out_data_bytes_size, void* out_data) override; + bool empty() const override; +private: + file_ptr source_handle; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_FILE_DATA_PROVIDER_HPP diff --git a/modules/gapi/src/streaming/onevpl/source.cpp b/modules/gapi/src/streaming/onevpl/source.cpp new file mode 100644 index 0000000000..0decb1358b --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/source.cpp @@ -0,0 +1,58 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include + +#include "streaming/onevpl/source_priv.hpp" +#include "streaming/onevpl/file_data_provider.hpp" +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +#ifdef HAVE_ONEVPL +GSource::GSource(const std::string& filePath, const CfgParams& cfg_params) : + GSource(std::unique_ptr(new GSource::Priv(std::make_shared(filePath), + cfg_params))) { + + if (filePath.empty()) { + util::throw_error(std::logic_error("Cannot create 'GSource' on empty source file name")); + } +} + +GSource::GSource(std::shared_ptr source, const CfgParams& cfg_params) : + GSource(std::unique_ptr(new GSource::Priv(source, cfg_params))) { +} +#else +GSource::GSource(const std::string&, const CfgParams&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} + +GSource::GSource(std::shared_ptr, const CfgParams&) { + GAPI_Assert(false && "Unsupported: G-API compiled without `WITH_GAPI_ONEVPL=ON`"); +} +#endif + +GSource::GSource(std::unique_ptr&& impl) : + IStreamSource(), + m_priv(std::move(impl)) { +} + +GSource::~GSource() = default; + +bool GSource::pull(cv::gapi::wip::Data& data) +{ + return m_priv->pull(data); +} + +GMetaArg GSource::descr_of() const +{ + return m_priv->descr_of(); +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv diff --git a/modules/gapi/src/streaming/onevpl/source_priv.cpp b/modules/gapi/src/streaming/onevpl/source_priv.cpp new file mode 100644 index 0000000000..ed838d6adc --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/source_priv.cpp @@ -0,0 +1,67 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#include +#include + +#include "streaming/onevpl/source_priv.hpp" +#include "logger.hpp" + +#ifndef HAVE_ONEVPL +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +bool GSource::Priv::pull(cv::gapi::wip::Data&) { + return true; +} +GMetaArg GSource::Priv::descr_of() const { + return {}; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +GSource::Priv::Priv() : + mfx_handle(MFXLoad()) +{ + GAPI_LOG_INFO(nullptr, "Initialized MFX handle: " << mfx_handle); + description_is_valid = false; +} + +GSource::Priv::Priv(std::shared_ptr, const std::vector&) : + GSource::Priv() +{ +} + +GSource::Priv::~Priv() +{ + GAPI_LOG_INFO(nullptr, "Unload MFX handle: " << mfx_handle); + MFXUnload(mfx_handle); +} + +bool GSource::Priv::pull(cv::gapi::wip::Data&) +{ + return false; +} + +GMetaArg GSource::Priv::descr_of() const +{ + return {}; +} +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/source_priv.hpp b/modules/gapi/src/streaming/onevpl/source_priv.hpp new file mode 100644 index 0000000000..c278934253 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/source_priv.hpp @@ -0,0 +1,67 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP + +#include + +#include +#include + +#include +#include +#include + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include + +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +struct GSource::Priv +{ + explicit Priv(std::shared_ptr provider, + const std::vector& params); + ~Priv(); + + bool pull(cv::gapi::wip::Data& data); + GMetaArg descr_of() const; +private: + Priv(); + mfxLoader mfx_handle; + bool description_is_valid; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { +struct GSource::Priv final +{ + bool pull(cv::gapi::wip::Data&); + GMetaArg descr_of() const; +}; +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // OPENCV_GAPI_STREAMING_ONEVPL_ONEVPL_SOURCE_PRIV_HPP diff --git a/modules/gapi/src/streaming/onevpl/utils.hpp b/modules/gapi/src/streaming/onevpl/utils.hpp new file mode 100644 index 0000000000..0512c4f687 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/utils.hpp @@ -0,0 +1,38 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + +#ifndef GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP +#define GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP + +#ifdef HAVE_ONEVPL +#if (MFX_VERSION >= 2000) +#include +#endif // MFX_VERSION + +#include +#include + +#include +#include + +#include + + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +inline std::string mfxstatus_to_string(mfxStatus) { + return "UNKNOWN"; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv +#endif // HAVE_ONEVPL +#endif // GAPI_STREAMING_ONEVPL_ONEVPL_UTILS_HPP diff --git a/modules/gapi/test/common/gapi_tests_helpers.hpp b/modules/gapi/test/common/gapi_tests_helpers.hpp index ce0c90442d..3f4e1ee0b3 100644 --- a/modules/gapi/test/common/gapi_tests_helpers.hpp +++ b/modules/gapi/test/common/gapi_tests_helpers.hpp @@ -50,6 +50,18 @@ namespace opencv_test __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ __WRAP_VAARGS(__DEFINE_PARAMS_IMPL7(index+1, __VA_ARGS__)) +#define __DEFINE_PARAMS_IMPL9(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL8(index+1, __VA_ARGS__)) + +#define __DEFINE_PARAMS_IMPL10(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL9(index+1, __VA_ARGS__)) + +#define __DEFINE_PARAMS_IMPL11(index, param_name, ...) \ + __TUPLE_PARAM_TYPE(index) param_name = getSpecificParam(); \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL10(index+1, __VA_ARGS__)) + // user interface to define member variables of specified names #define DEFINE_SPECIFIC_PARAMS_0() @@ -76,6 +88,15 @@ namespace opencv_test #define DEFINE_SPECIFIC_PARAMS_8(...) \ __WRAP_VAARGS(__DEFINE_PARAMS_IMPL8(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_9(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL9(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_10(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL10(0, __VA_ARGS__)) + +#define DEFINE_SPECIFIC_PARAMS_11(...) \ + __WRAP_VAARGS(__DEFINE_PARAMS_IMPL11(0, __VA_ARGS__)) } // namespace opencv_test #endif //OPENCV_GAPI_TESTS_HELPERS_HPP diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index e33f165ae1..b7ea891b81 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -9,6 +9,8 @@ #ifdef HAVE_INF_ENGINE #include +#include +#include #include @@ -2052,6 +2054,139 @@ TEST(IEFrameAdapter, blobParams) EXPECT_EQ(expected, actual); } +namespace +{ + +struct Sync { + std::mutex m; + std::condition_variable cv; + int counter = 0; +}; + +class GMockMediaAdapter final: public cv::MediaFrame::IAdapter { +public: + explicit GMockMediaAdapter(cv::Mat m, Sync& sync) + : m_mat(m), m_sync(sync) { + } + + cv::GFrameDesc meta() const override { + return cv::GFrameDesc{cv::MediaFormat::BGR, m_mat.size()}; + } + + cv::MediaFrame::View access(cv::MediaFrame::Access) override { + cv::MediaFrame::View::Ptrs pp = { m_mat.ptr(), nullptr, nullptr, nullptr }; + cv::MediaFrame::View::Strides ss = { m_mat.step, 0u, 0u, 0u }; + return cv::MediaFrame::View(std::move(pp), std::move(ss)); + } + + ~GMockMediaAdapter() { + { + std::lock_guard lk{m_sync.m}; + m_sync.counter--; + } + m_sync.cv.notify_one(); + } + +private: + cv::Mat m_mat; + Sync& m_sync; +}; + +// NB: This source is needed to simulate real +// cases where the memory resources are limited. +// GMockSource(int limit) - accept the number of MediaFrames that +// the source can produce until resources are over. +class GMockSource : public cv::gapi::wip::IStreamSource { +public: + explicit GMockSource(int limit) + : m_limit(limit), m_mat(cv::Size(1920, 1080), CV_8UC3) { + cv::randu(m_mat, cv::Scalar::all(0), cv::Scalar::all(255)); + } + + bool pull(cv::gapi::wip::Data& data) { + std::unique_lock lk(m_sync.m); + m_sync.counter++; + // NB: Can't produce new frames until old ones are released. + m_sync.cv.wait(lk, [this]{return m_sync.counter <= m_limit;}); + + data = cv::MediaFrame::Create(m_mat, m_sync); + return true; + } + + GMetaArg descr_of() const override { + return GMetaArg{cv::GFrameDesc{cv::MediaFormat::BGR, m_mat.size()}}; + } + +private: + int m_limit; + cv::Mat m_mat; + Sync m_sync; +}; + +struct LimitedSourceInfer: public ::testing::Test { + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + LimitedSourceInfer() + : comp([](){ + cv::GFrame in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + return cv::GComputation(cv::GIn(in), cv::GOut(age, gender)); + }) { + initDLDTDataPath(); + } + + GStreamingCompiled compileStreaming(int nireq) { + cv::gapi::ie::detail::ParamDesc params; + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + params.device_id = "CPU"; + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id } + .cfgOutputLayers({ "age_conv3", "prob" }) + .cfgNumRequests(nireq); + + return comp.compileStreaming(cv::compile_args(cv::gapi::networks(pp))); + } + + void run(const int max_frames, const int limit, const int nireq) { + auto pipeline = compileStreaming(nireq); + pipeline.setSource(limit); + pipeline.start(); + + int num_frames = 0; + while (num_frames != max_frames && + pipeline.pull(cv::gout(out_age, out_gender))) { + ++num_frames; + } + } + + cv::GComputation comp; + cv::Mat out_age, out_gender; +}; + +} // anonymous namespace + +TEST_F(LimitedSourceInfer, ReleaseFrame) +{ + constexpr int max_frames = 50; + constexpr int resources_limit = 1; + constexpr int nireq = 1; + + run(max_frames, resources_limit, nireq); +} + +TEST_F(LimitedSourceInfer, ReleaseFrameAsync) +{ + constexpr int max_frames = 50; + constexpr int resources_limit = 4; + constexpr int nireq = 8; + + run(max_frames, resources_limit, nireq); +} + } // namespace opencv_test #endif // HAVE_INF_ENGINE diff --git a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp index ef192b9d6a..b1bf9c9356 100644 --- a/modules/gapi/test/infer/gapi_infer_onnx_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_onnx_test.cpp @@ -67,17 +67,17 @@ struct ONNXInitPath { static ONNXInitPath g_init_path; cv::Mat initMatrixRandU(const int type, const cv::Size& sz_in) { - const cv::Mat in_mat1 = cv::Mat(sz_in, type); + const cv::Mat in_mat = cv::Mat(sz_in, type); if (CV_MAT_DEPTH(type) < CV_32F) { - cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar::all(255)); } else { const int fscale = 256; // avoid bits near ULP, generate stable test input - cv::Mat in_mat32s(in_mat1.size(), CV_MAKE_TYPE(CV_32S, CV_MAT_CN(type))); + cv::Mat in_mat32s(in_mat.size(), CV_MAKE_TYPE(CV_32S, CV_MAT_CN(type))); cv::randu(in_mat32s, cv::Scalar::all(0), cv::Scalar::all(255 * fscale)); - in_mat32s.convertTo(in_mat1, type, 1.0f / fscale, 0); + in_mat32s.convertTo(in_mat, type, 1.0f / fscale, 0); } - return in_mat1; + return in_mat; } } // anonymous namespace namespace opencv_test @@ -319,15 +319,13 @@ public: size_t num_in, num_out; std::vector out_gapi; std::vector out_onnx; - cv::Mat in_mat1; + cv::Mat in_mat; ONNXtest() { initTestDataPath(); env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "test"); memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); out_gapi.resize(1); - // FIXME: It should be an image from own (gapi) directory in opencv extra - in_mat1 = cv::imread(findDataFile("cv/dpm/cat.png")); } template @@ -463,13 +461,9 @@ public: cv::Rect(cv::Point{70, 10}, cv::Size{20, 260}), cv::Rect(cv::Point{5, 15}, cv::Size{200, 160}), }; - cv::Mat m_in_y; - cv::Mat m_in_uv; - virtual void SetUp() { - cv::Size sz{640, 480}; - m_in_y = initMatrixRandU(CV_8UC1, sz); - m_in_uv = initMatrixRandU(CV_8UC2, sz / 2); - } + const cv::Size sz{640, 480}; + const cv::Mat m_in_y = initMatrixRandU(CV_8UC1, sz); + const cv::Mat m_in_uv = initMatrixRandU(CV_8UC2, sz / 2); }; class ONNXGRayScale : public ONNXtest { @@ -545,20 +539,20 @@ class ONNXYoloV3 : public ONNXWithRemap { public: std::vector ins; -private: - virtual void SetUp() { + void constructYoloInputs(const cv::Mat& src) { const int yolo_in_h = 416; const int yolo_in_w = 416; cv::Mat yolov3_input, shape, prep_mat; - cv::resize(in_mat1, yolov3_input, cv::Size(yolo_in_w, yolo_in_h)); + cv::resize(src, yolov3_input, cv::Size(yolo_in_w, yolo_in_h)); shape.create(cv::Size(2, 1), CV_32F); float* ptr = shape.ptr(); - ptr[0] = in_mat1.cols; - ptr[1] = in_mat1.rows; + ptr[0] = src.cols; + ptr[1] = src.rows; preprocess(yolov3_input, prep_mat); ins = {prep_mat, shape}; } +private: void preprocess(const cv::Mat& src, cv::Mat& dst) { cv::Mat cvt; src.convertTo(cvt, CV_32F, 1.f / 255.f); @@ -571,9 +565,10 @@ private: TEST_F(ONNXClassification, Infer) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat processed_mat; - preprocess(in_mat1, processed_mat); + preprocess(in_mat, processed_mat); infer(processed_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -583,7 +578,7 @@ TEST_F(ONNXClassification, Infer) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -593,9 +588,10 @@ TEST_F(ONNXClassification, Infer) TEST_F(ONNXClassification, InferTensor) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // Create tensor cv::Mat tensor; - preprocess(in_mat1, tensor); + preprocess(in_mat, tensor); // ONNX_API code infer(tensor, out_onnx); // G_API code @@ -614,10 +610,11 @@ TEST_F(ONNXClassification, InferTensor) TEST_F(ONNXClassification, InferROI) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto ROI = rois.at(0); // ONNX_API code cv::Mat roi_mat; - preprocess(in_mat1(ROI), roi_mat); + preprocess(in_mat(ROI), roi_mat); infer(roi_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -628,7 +625,7 @@ TEST_F(ONNXClassification, InferROI) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, ROI), + comp.apply(cv::gin(in_mat, ROI), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -638,10 +635,11 @@ TEST_F(ONNXClassification, InferROI) TEST_F(ONNXClassification, InferROIList) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -653,7 +651,7 @@ TEST_F(ONNXClassification, InferROIList) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, rois), + comp.apply(cv::gin(in_mat, rois), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -663,10 +661,11 @@ TEST_F(ONNXClassification, InferROIList) TEST_F(ONNXClassification, Infer2ROIList) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -678,7 +677,7 @@ TEST_F(ONNXClassification, Infer2ROIList) // NOTE: We have to normalize U8 tensor // so cfgMeanStd() is here auto net = cv::gapi::onnx::Params { model_path }.cfgMeanStd({ mean }, { std }); - comp.apply(cv::gin(in_mat1, rois), + comp.apply(cv::gin(in_mat, rois), cv::gout(out_gapi), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -688,9 +687,10 @@ TEST_F(ONNXClassification, Infer2ROIList) TEST_F(ONNXWithRemap, InferDynamicInputTensor) { useModel("object_detection_segmentation/tiny-yolov2/model/tinyyolov2-8"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // Create tensor cv::Mat cvt, rsz, tensor; - cv::resize(in_mat1, rsz, cv::Size{416, 416}); + cv::resize(in_mat, rsz, cv::Size{416, 416}); rsz.convertTo(cvt, CV_32F, 1.f / 255.f); toCHW(cvt, tensor); tensor = tensor.reshape(1, {1, 3, 416, 416}); @@ -714,9 +714,10 @@ TEST_F(ONNXWithRemap, InferDynamicInputTensor) TEST_F(ONNXGRayScale, InferImage) { useModel("body_analysis/emotion_ferplus/model/emotion-ferplus-8"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat prep_mat; - preprocess(in_mat1, prep_mat); + preprocess(in_mat, prep_mat); infer(prep_mat, out_onnx); // G_API code G_API_NET(EmotionNet, , "emotion-ferplus"); @@ -725,7 +726,7 @@ TEST_F(ONNXGRayScale, InferImage) cv::GComputation comp(cv::GIn(in), cv::GOut(out)); auto net = cv::gapi::onnx::Params { model_path } .cfgNormalize({ false }); // model accepts 0..255 range in FP32; - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -735,8 +736,9 @@ TEST_F(ONNXGRayScale, InferImage) TEST_F(ONNXWithRemap, InferMultiOutput) { useModel("object_detection_segmentation/ssd-mobilenetv1/model/ssd_mobilenet_v1_10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code - const auto prep_mat = in_mat1.reshape(1, {1, in_mat1.rows, in_mat1.cols, in_mat1.channels()}); + const auto prep_mat = in_mat.reshape(1, {1, in_mat.rows, in_mat.cols, in_mat.channels()}); infer(prep_mat, out_onnx); cv::Mat onnx_conv_out({1, 1, 200, 7}, CV_32F); remapToIESSDOut({out_onnx[3], out_onnx[0], out_onnx[2], out_onnx[1]}, onnx_conv_out); @@ -750,7 +752,7 @@ TEST_F(ONNXWithRemap, InferMultiOutput) auto net = cv::gapi::onnx::Params{ model_path } .cfgOutputLayers({"detection_output"}) .cfgPostProc({cv::GMatDesc{CV_32F, {1, 1, 200, 7}}}, remapSSDPorts); - comp.apply(cv::gin(in_mat1), + comp.apply(cv::gin(in_mat), cv::gout(out_gapi.front()), cv::compile_args(cv::gapi::networks(net))); // Validate @@ -760,12 +762,13 @@ TEST_F(ONNXWithRemap, InferMultiOutput) TEST_F(ONNXMediaFrame, InferBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // ONNX_API code cv::Mat processed_mat; - preprocess(in_mat1, processed_mat); + preprocess(in_mat, processed_mat); infer(processed_mat, out_onnx); // G_API code - auto frame = MediaFrame::Create(in_mat1); + auto frame = MediaFrame::Create(in_mat); G_API_NET(SqueezNet, , "squeeznet"); cv::GFrame in; cv::GMat out = cv::gapi::infer(in); @@ -783,6 +786,7 @@ TEST_F(ONNXMediaFrame, InferBGR) TEST_F(ONNXMediaFrame, InferYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -808,10 +812,11 @@ TEST_F(ONNXMediaFrame, InferYUV) TEST_F(ONNXMediaFrame, InferROIBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + auto frame = MediaFrame::Create(in_mat); // ONNX_API code cv::Mat roi_mat; - preprocess(in_mat1(rois.front()), roi_mat); + preprocess(in_mat(rois.front()), roi_mat); infer(roi_mat, out_onnx); // G_API code G_API_NET(SqueezNet, , "squeeznet"); @@ -832,6 +837,7 @@ TEST_F(ONNXMediaFrame, InferROIBGR) TEST_F(ONNXMediaFrame, InferROIYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -858,11 +864,12 @@ TEST_F(ONNXMediaFrame, InferROIYUV) TEST_F(ONNXMediaFrame, InferListBGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - const auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + const auto frame = MediaFrame::Create(in_mat); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -884,6 +891,7 @@ TEST_F(ONNXMediaFrame, InferListBGR) TEST_F(ONNXMediaFrame, InferListYUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -911,8 +919,9 @@ TEST_F(ONNXMediaFrame, InferListYUV) TEST_F(ONNXRCNN, InferWithDisabledOut) { useModel("object_detection_segmentation/faster-rcnn/model/FasterRCNN-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); cv::Mat pp; - preprocess(in_mat1, pp); + preprocess(in_mat, pp); // ONNX_API code infer(pp, out_onnx, {"6379", "6383"}); // G_API code @@ -937,11 +946,12 @@ TEST_F(ONNXRCNN, InferWithDisabledOut) TEST_F(ONNXMediaFrame, InferList2BGR) { useModel("classification/squeezenet/model/squeezenet1.0-9"); - const auto frame = MediaFrame::Create(in_mat1); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + const auto frame = MediaFrame::Create(in_mat); // ONNX_API code for (size_t i = 0; i < rois.size(); ++i) { cv::Mat roi_mat; - preprocess(in_mat1(rois[i]), roi_mat); + preprocess(in_mat(rois[i]), roi_mat); infer(roi_mat, out_onnx); } // G_API code @@ -963,6 +973,7 @@ TEST_F(ONNXMediaFrame, InferList2BGR) TEST_F(ONNXMediaFrame, InferList2YUV) { useModel("classification/squeezenet/model/squeezenet1.0-9"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); const auto frame = MediaFrame::Create(m_in_y, m_in_uv); // ONNX_API code cv::Mat pp; @@ -991,6 +1002,8 @@ TEST_F(ONNXMediaFrame, InferList2YUV) TEST_F(ONNXYoloV3, InferConstInput) { useModel("object_detection_segmentation/yolov3/model/yolov3-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + constructYoloInputs(in_mat); // ONNX_API code infer(ins, out_onnx); // G_API code @@ -1022,6 +1035,8 @@ TEST_F(ONNXYoloV3, InferBSConstInput) // and all input layer names are specified. // Const input has the advantage. It is expected behavior. useModel("object_detection_segmentation/yolov3/model/yolov3-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); + constructYoloInputs(in_mat); // Tensor with incorrect image size // is used for check case when InputLayers and constInput have same names cv::Mat bad_shape; @@ -1059,8 +1074,9 @@ TEST_F(ONNXYoloV3, InferBSConstInput) TEST_F(ONNXRCNN, ConversionInt64to32) { useModel("object_detection_segmentation/faster-rcnn/model/FasterRCNN-10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); cv::Mat dst; - preprocess(in_mat1, dst); + preprocess(in_mat, dst); // ONNX_API code infer(dst, out_onnx); // G_API code @@ -1087,6 +1103,7 @@ TEST_F(ONNXRCNN, ConversionInt64to32) TEST_F(ONNXWithRemap, InferOutReallocation) { useModel("object_detection_segmentation/ssd-mobilenetv1/model/ssd_mobilenet_v1_10"); + in_mat = cv::imread(findDataFile("cv/dpm/cat.png", false)); // G_API code G_API_NET(MobileNet, , "ssd_mobilenet"); auto net = cv::gapi::onnx::Params{model_path} @@ -1096,7 +1113,7 @@ TEST_F(ONNXWithRemap, InferOutReallocation) cv::GMat out1; out1 = cv::gapi::infer(in); cv::GComputation comp(cv::GIn(in), cv::GOut(out1)); - EXPECT_THROW(comp.apply(cv::gin(in_mat1), + EXPECT_THROW(comp.apply(cv::gin(in_mat), cv::gout(out_gapi[0]), cv::compile_args(cv::gapi::networks(net))), std::exception); } diff --git a/modules/gapi/test/internal/gapi_int_executor_tests.cpp b/modules/gapi/test/internal/gapi_int_executor_tests.cpp index 04afa1fda8..90a338a3d0 100644 --- a/modules/gapi/test/internal/gapi_int_executor_tests.cpp +++ b/modules/gapi/test/internal/gapi_int_executor_tests.cpp @@ -6,10 +6,158 @@ #include "../test_precomp.hpp" +#include "../gapi_mock_kernels.hpp" namespace opencv_test { +namespace +{ + +class GMockExecutable final: public cv::gimpl::GIslandExecutable +{ + virtual inline bool canReshape() const override { + return m_priv->m_can_reshape; + } + + virtual void reshape(ade::Graph&, const GCompileArgs&) override + { + m_priv->m_reshape_counter++; + } + virtual void handleNewStream() override { } + virtual void run(std::vector&&, std::vector&&) { } + virtual bool allocatesOutputs() const override + { + return true; + } + + virtual cv::RMat allocate(const cv::GMatDesc&) const override + { + m_priv->m_allocate_counter++; + return cv::RMat(); + } + + // NB: GMockBackendImpl creates new unique_ptr + // on every compile call. Need to share counters between instances in order + // to validate it in tests. + struct Priv + { + bool m_can_reshape; + int m_reshape_counter; + int m_allocate_counter; + }; + + std::shared_ptr m_priv; + +public: + GMockExecutable(bool can_reshape = true) + : m_priv(new Priv{can_reshape, 0, 0}) + { + }; + + void setReshape(bool can_reshape) { m_priv->m_can_reshape = can_reshape; } + + int getReshapeCounter() const { return m_priv->m_reshape_counter; } + int getAllocateCounter() const { return m_priv->m_allocate_counter; } +}; + +class GMockBackendImpl final: public cv::gapi::GBackend::Priv +{ + virtual void unpackKernel(ade::Graph &, + const ade::NodeHandle &, + const cv::GKernelImpl &) override { } + + virtual EPtr compile(const ade::Graph &, + const cv::GCompileArgs &, + const std::vector &) const override + { + ++m_compile_counter; + return EPtr{new GMockExecutable(m_exec)}; + } + + mutable int m_compile_counter = 0; + GMockExecutable m_exec; + + virtual bool controlsMerge() const override { + return true; + } + + virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &, + const ade::NodeHandle &, + const ade::NodeHandle &, + const ade::NodeHandle &) const override { + return false; + } + +public: + GMockBackendImpl(const GMockExecutable& exec) : m_exec(exec) { }; + int getCompileCounter() const { return m_compile_counter; } +}; + +class GMockFunctor : public gapi::cpu::GOCVFunctor +{ +public: + GMockFunctor(cv::gapi::GBackend backend, + const char* id, + const Meta &meta, + const Impl& impl) + : gapi::cpu::GOCVFunctor(id, meta, impl), m_backend(backend) + { + } + + cv::gapi::GBackend backend() const override { return m_backend; } + +private: + cv::gapi::GBackend m_backend; +}; + +template +GMockFunctor mock_kernel(const cv::gapi::GBackend& backend, Callable c) +{ + using P = cv::detail::OCVCallHelper; + return GMockFunctor{ backend + , K::id() + , &K::getOutMeta + , std::bind(&P::callFunctor, std::placeholders::_1, c) + }; +} + +void dummyFooImpl(const cv::Mat&, cv::Mat&) { }; +void dummyBarImpl(const cv::Mat&, const cv::Mat&, cv::Mat&) { }; + +struct GExecutorReshapeTest: public ::testing::Test +{ + GExecutorReshapeTest() + : comp([](){ + cv::GMat in; + cv::GMat out = I::Bar::on(I::Foo::on(in), in); + return cv::GComputation(in, out); + }) + { + backend_impl1 = std::make_shared(island1); + backend1 = cv::gapi::GBackend{backend_impl1}; + backend_impl2 = std::make_shared(island2); + backend2 = cv::gapi::GBackend{backend_impl2}; + auto kernel1 = mock_kernel(backend1, dummyFooImpl); + auto kernel2 = mock_kernel(backend2, dummyBarImpl); + pkg = cv::gapi::kernels(kernel1, kernel2); + in_mat1 = cv::Mat::eye(32, 32, CV_8UC1); + in_mat2 = cv::Mat::eye(64, 64, CV_8UC1); + } + + cv::GComputation comp; + GMockExecutable island1; + std::shared_ptr backend_impl1; + cv::gapi::GBackend backend1; + GMockExecutable island2; + std::shared_ptr backend_impl2; + cv::gapi::GBackend backend2; + cv::gapi::GKernelPackage pkg; + cv::Mat in_mat1, in_mat2, out_mat;; +}; + +} // anonymous namespace + // FIXME: avoid code duplication // The below graph and config is taken from ComplexIslands test suite TEST(GExecutor, SmokeTest) @@ -77,6 +225,75 @@ TEST(GExecutor, SmokeTest) // with breakdown worked) } +TEST_F(GExecutorReshapeTest, ReshapeInsteadOfRecompile) +{ + // NB: Initial state + EXPECT_EQ(0, backend_impl1->getCompileCounter()); + EXPECT_EQ(0, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: GMockBackendImpl implements "reshape" method, + // so it won't be recompiled if the meta is changed. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(1, island1.getReshapeCounter()); + EXPECT_EQ(1, island2.getReshapeCounter()); +} + +TEST_F(GExecutorReshapeTest, OneBackendNotReshapable) +{ + // NB: Make first island not reshapable + island1.setReshape(false); + + // NB: Initial state + EXPECT_EQ(0, backend_impl1->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, backend_impl1->getCompileCounter()); + EXPECT_EQ(1, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); + + // NB: Since one of islands isn't reshapable + // the entire graph isn't reshapable as well. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(2, backend_impl1->getCompileCounter()); + EXPECT_EQ(2, backend_impl2->getCompileCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + EXPECT_EQ(0, island2.getReshapeCounter()); +} + +TEST_F(GExecutorReshapeTest, ReshapeCallAllocate) +{ + // NB: Initial state + EXPECT_EQ(0, island1.getAllocateCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + + // NB: First compilation. + comp.apply(cv::gin(in_mat1), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(1, island1.getAllocateCounter()); + EXPECT_EQ(0, island1.getReshapeCounter()); + + // NB: The entire graph is reshapable, so it won't be recompiled, but reshaped. + // Check that reshape call "allocate" to reallocate buffers. + comp.apply(cv::gin(in_mat2), cv::gout(out_mat), cv::compile_args(pkg)); + EXPECT_EQ(2, island1.getAllocateCounter()); + EXPECT_EQ(1, island1.getReshapeCounter()); +} + // FIXME: Add explicit tests on GMat/GScalar/GArray being connectors // between executed islands diff --git a/modules/gapi/test/render/gapi_render_tests_ocv.cpp b/modules/gapi/test/render/gapi_render_tests_ocv.cpp index 010df5dff7..95f6954156 100644 --- a/modules/gapi/test/render/gapi_render_tests_ocv.cpp +++ b/modules/gapi/test/render/gapi_render_tests_ocv.cpp @@ -639,8 +639,8 @@ INSTANTIATE_TEST_CASE_P(RenderBGROCVTestRectsImpl, RenderBGROCVTestRects, Values(cv::Rect(100, 100, 200, 200)), Values(cv::Scalar(100, 50, 150)), Values(2), - Values(LINE_8), - Values(0))); + Values(LINE_8, LINE_4), + Values(0, 1))); INSTANTIATE_TEST_CASE_P(RenderNV12OCVTestRectsImpl, RenderNV12OCVTestRects, Combine(Values(cv::Size(1280, 720)), @@ -673,8 +673,8 @@ INSTANTIATE_TEST_CASE_P(RenderNV12OCVTestCirclesImpl, RenderNV12OCVTestCircles, Values(10), Values(cv::Scalar(100, 50, 150)), Values(2), - Values(LINE_8), - Values(0))); + Values(LINE_8, LINE_4), + Values(0, 1))); INSTANTIATE_TEST_CASE_P(RenderMFrameOCVTestCirclesImpl, RenderMFrameOCVTestCircles, Combine(Values(cv::Size(1280, 720)), diff --git a/modules/gapi/test/s11n/gapi_s11n_tests.cpp b/modules/gapi/test/s11n/gapi_s11n_tests.cpp index f4a30b3946..4c6e63b552 100644 --- a/modules/gapi/test/s11n/gapi_s11n_tests.cpp +++ b/modules/gapi/test/s11n/gapi_s11n_tests.cpp @@ -2,6 +2,7 @@ #include "backends/common/serialization.hpp" #include +#include #include <../src/backends/common/gbackend.hpp> // asView namespace { @@ -148,6 +149,29 @@ public: int getVal() { return m_value; } std::string getStr() { return m_str; } }; + +class MyMediaFrameAdapter : public cv::MediaFrame::IAdapter { + cv::Mat m_mat; + int m_value; + std::string m_str; +public: + MyMediaFrameAdapter() = default; + MyMediaFrameAdapter(cv::Mat m, int value, const std::string& str) + : m_mat(m), m_value(value), m_str(str) + {} + virtual cv::MediaFrame::View access(cv::MediaFrame::Access) override { + return cv::MediaFrame::View({m_mat.data}, {m_mat.step}); + } + virtual cv::GFrameDesc meta() const override { return {cv::MediaFormat::BGR, m_mat.size()}; } + virtual void serialize(cv::gapi::s11n::IOStream& os) override { + os << m_value << m_str; + } + virtual void deserialize(cv::gapi::s11n::IIStream& is) override { + is >> m_value >> m_str; + } + int getVal() { return m_value; } + std::string getStr() { return m_str; } +}; } namespace opencv_test { @@ -581,6 +605,17 @@ TEST_F(S11N_Basic, Test_Vector_Of_Strings) { EXPECT_EQ("42", des[2]); } +TEST_F(S11N_Basic, Test_RunArg) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto v = cv::GRunArgs{ cv::GRunArg{ mat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::Mat out_mat = cv::util::get(out[0]); + + EXPECT_EQ(0, cv::norm(mat, out_mat)); +} + TEST_F(S11N_Basic, Test_RunArg_RMat) { cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); cv::RMat rmat = cv::make_rmat(mat, 42, "It actually works"); @@ -614,6 +649,87 @@ TEST_F(S11N_Basic, Test_RunArg_RMat_Scalar_Mat) { EXPECT_EQ(0, cv::norm(mat, out_mat)); } +TEST_F(S11N_Basic, Test_RunArg_MediaFrame) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto v = cv::GRunArgs{ cv::GRunArg{ frame } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::MediaFrame out_mat = cv::util::get(out[0]); + auto adapter = out_mat.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); +} + +TEST_F(S11N_Basic, Test_RunArg_MediaFrame_Scalar_Mat) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + cv::Scalar sc(111); + auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ sc }, cv::GRunArg{ mat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + cv::MediaFrame out_frame = cv::util::get(out[0]); + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + cv::Scalar out_sc = cv::util::get(out[1]); + EXPECT_EQ(sc, out_sc); + + cv::Mat out_mat = cv::util::get(out[2]); + EXPECT_EQ(0, cv::norm(mat, out_mat)); +} + +TEST_F(S11N_Basic, Test_RunArg_MediaFrame_RMat) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto rmat = cv::make_rmat(mat2, 24, "Hello there"); + + auto v = cv::GRunArgs{ cv::GRunArg{ frame }, cv::GRunArg{ rmat } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + + cv::MediaFrame out_frame = cv::util::get(out[0]); + cv::RMat out_rmat = cv::util::get(out[1]); + + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + auto adapter2 = out_rmat.get(); + EXPECT_EQ(24, adapter2->getVal()); + EXPECT_EQ("Hello there", adapter2->getStr()); +} + +TEST_F(S11N_Basic, Test_RunArg_RMat_MediaFrame) { + cv::Mat mat = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + cv::Mat mat2 = cv::Mat::eye(cv::Size(128, 64), CV_8UC3); + + auto frame = cv::MediaFrame::Create(mat, 42, "It actually works"); + auto rmat = cv::make_rmat(mat2, 24, "Hello there"); + + auto v = cv::GRunArgs{ cv::GRunArg{ rmat }, cv::GRunArg{ frame } }; + + const std::vector sargsin = cv::gapi::serialize(v); + cv::GRunArgs out = cv::gapi::deserialize(sargsin); + + cv::RMat out_rmat = cv::util::get(out[0]); + cv::MediaFrame out_frame = cv::util::get(out[1]); + + auto adapter = out_frame.get(); + EXPECT_EQ(42, adapter->getVal()); + EXPECT_EQ("It actually works", adapter->getStr()); + + auto adapter2 = out_rmat.get(); + EXPECT_EQ(24, adapter2->getVal()); + EXPECT_EQ("Hello there", adapter2->getStr()); +} + namespace { template bool verifyOpaqueKind(T&& in) { @@ -754,8 +870,6 @@ TEST_F(S11N_Basic, Test_Deserialize_CompileArgs_RandomOrder) { std::vector sArgs = cv::gapi::serialize( cv::compile_args(simpleCustomVar, simpleCustomVar2)); GCompileArgs dArgs = cv::gapi::deserialize(sArgs); diff --git a/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp b/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp index 885457cd90..c3d21a3f6f 100644 --- a/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp +++ b/modules/gapi/test/s11n/gapi_sample_pipelines_s11n.cpp @@ -806,4 +806,33 @@ TEST(S11N, Pipeline_Render_RGB) EXPECT_EQ(cv::norm(input, ref_mat), 0); } + +TEST(S11N, Pipeline_Const_GScalar) +{ + static constexpr auto in_scalar = 10; + + cv::GMat a; + cv::GScalar s; + + cv::GComputation computation(GIn(a), GOut(cv::gapi::addC(a, in_scalar))); + auto p = cv::gapi::serialize(computation); + auto deserialized_computation = cv::gapi::deserialize(p); + + cv::Mat in_mat = cv::Mat::eye(32, 32, CV_8UC1); + cv::Mat ref_mat; + cv::add(in_mat, in_scalar, ref_mat); + + cv::Mat out_mat; + computation.apply(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); + + out_mat = cv::Mat(); + deserialized_computation.apply(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); + + out_mat = cv::Mat(); + auto cc = deserialized_computation.compile(cv::descr_of(in_mat)); + cc(cv::gin(in_mat/*, in_scalar*/), cv::gout(out_mat)); + EXPECT_EQ(0, cvtest::norm(out_mat, ref_mat, NORM_INF)); +} } // namespace opencv_test diff --git a/modules/gapi/test/streaming/gapi_streaming_tests.cpp b/modules/gapi/test/streaming/gapi_streaming_tests.cpp index f3179a7081..4f8b2563ac 100644 --- a/modules/gapi/test/streaming/gapi_streaming_tests.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_tests.cpp @@ -2,7 +2,7 @@ // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html. // -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation #include "../test_precomp.hpp" @@ -25,6 +25,17 @@ #include #include +#include + +#ifdef HAVE_ONEVPL + +#if (MFX_VERSION >= 2000) +#include +#endif + +#include +#endif // HAVE_ONEVPL + namespace opencv_test { namespace @@ -244,6 +255,51 @@ public: } }; +void checkPullOverload(const cv::Mat& ref, + const bool has_output, + cv::util::variant& args) { + EXPECT_TRUE(has_output); + using runArgs = cv::util::variant; + cv::Mat out_mat; + switch (args.index()) { + case runArgs::index_of(): + { + auto outputs = util::get(args); + EXPECT_EQ(1u, outputs.size()); + out_mat = cv::util::get(outputs[0]); + break; + } + case runArgs::index_of(): + { + auto outputs = util::get(args); + EXPECT_EQ(1u, outputs.size()); + auto opt_mat = cv::util::get>(outputs[0]); + ASSERT_TRUE(opt_mat.has_value()); + out_mat = *opt_mat; + break; + } + default: GAPI_Assert(false && "Incorrect type of Args"); + } + + EXPECT_EQ(0., cv::norm(ref, out_mat, cv::NORM_INF)); +} + +struct StreamDataProvider : public cv::gapi::wip::onevpl::IDataProvider { + + StreamDataProvider(std::istream& in) : data_stream (in) { + EXPECT_TRUE(in); + } + + size_t fetch_data(size_t out_data_size, void* out_data_buf) override { + data_stream.read(reinterpret_cast(out_data_buf), out_data_size); + return (size_t)data_stream.gcount(); + } + bool empty() const override { + return data_stream.eof() || data_stream.bad(); + } +private: + std::istream& data_stream; +}; } // anonymous namespace TEST_P(GAPI_Streaming, SmokeTest_ConstInput_GMat) @@ -1336,13 +1392,45 @@ TEST(Streaming, Python_Pull_Overload) bool has_output; cv::GRunArgs outputs; - std::tie(has_output, outputs) = ccomp.pull(); + using RunArgs = cv::util::variant; + RunArgs args; - EXPECT_TRUE(has_output); - EXPECT_EQ(1u, outputs.size()); + std::tie(has_output, args) = ccomp.pull(); - auto out_mat = cv::util::get(outputs[0]); - EXPECT_EQ(0., cv::norm(in_mat, out_mat, cv::NORM_INF)); + checkPullOverload(in_mat, has_output, args); + + ccomp.stop(); + EXPECT_FALSE(ccomp.running()); +} + +TEST(GAPI_Streaming_Desync, Python_Pull_Overload) +{ + cv::GMat in; + cv::GMat out = cv::gapi::streaming::desync(in); + cv::GComputation c(in, out); + + cv::Size sz(3,3); + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar(255)); + + auto ccomp = c.compileStreaming(); + + EXPECT_TRUE(ccomp); + EXPECT_FALSE(ccomp.running()); + + ccomp.setSource(cv::gin(in_mat)); + + ccomp.start(); + EXPECT_TRUE(ccomp.running()); + + bool has_output; + cv::GRunArgs outputs; + using RunArgs = cv::util::variant; + RunArgs args; + + std::tie(has_output, args) = ccomp.pull(); + + checkPullOverload(in_mat, has_output, args); ccomp.stop(); EXPECT_FALSE(ccomp.running()); @@ -2132,9 +2220,17 @@ TEST(GAPI_Streaming, TestPythonAPI) bool is_over = false; cv::GRunArgs out_args; + using RunArgs = cv::util::variant; + RunArgs args; // NB: Used by python bridge - std::tie(is_over, out_args) = cc.pull(); + std::tie(is_over, args) = cc.pull(); + + switch (args.index()) { + case RunArgs::index_of(): + out_args = util::get(args); break; + default: GAPI_Assert(false && "Incorrect type of return value"); + } ASSERT_EQ(1u, out_args.size()); ASSERT_TRUE(cv::util::holds_alternative(out_args[0])); @@ -2145,4 +2241,113 @@ TEST(GAPI_Streaming, TestPythonAPI) cc.stop(); } +#ifdef HAVE_ONEVPL +const unsigned char hevc_header[] = { + 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0C, 0x06, 0xFF, 0xFF, 0x01, 0x40, 0x00, + 0x00, 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, + 0x00, 0x04, 0x02, 0x10, 0x30, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, + 0x01, 0xE5, 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x06, 0x01, 0x40, 0x00, 0x00, + 0x03, 0x00, 0x80, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, 0x00, 0x00, + 0xA0, 0x10, 0x20, 0x61, 0x63, 0x41, 0x00, 0x86, 0x49, 0x1B, 0x2B, 0x20, 0x00, + 0x00, 0x00, 0x01, 0x44, 0x01, 0xC0, 0x71, 0xC0, 0xD9, 0x20, 0x00, 0x00, 0x00, + 0x01, 0x26, 0x01, 0xAF, 0x0C +}; +TEST(OneVPL_Source, Init) +{ + using CfgParam = cv::gapi::wip::onevpl::CfgParam; + + std::vector src_params; + src_params.push_back(CfgParam::create("mfxImplDescription.Impl", + MFX_IMPL_TYPE_HARDWARE)); + src_params.push_back(CfgParam::create("mfxImplDescription.AccelerationMode", + MFX_ACCEL_MODE_VIA_D3D11, false)); + src_params.push_back(CfgParam::create("mfxImplDescription.mfxDecoderDescription.decoder.CodecID", + MFX_CODEC_HEVC)); + std::stringstream stream(std::ios_base::in | std::ios_base::out | std::ios_base::binary); + EXPECT_TRUE(stream.write(reinterpret_cast(const_cast(hevc_header)), + sizeof(hevc_header))); + std::shared_ptr stream_data_provider = std::make_shared(stream); + + cv::Ptr cap; + bool cap_created = false; + try { + cap = cv::gapi::wip::make_onevpl_src(stream_data_provider, src_params); + cap_created = true; + } catch (const std::exception&) { + } + ASSERT_TRUE(cap_created); + + cv::gapi::wip::Data out; + while (cap->pull(out)) { + (void)out; + } + EXPECT_TRUE(stream_data_provider->empty()); +} +#endif + +TEST(GAPI_Streaming, TestDesyncRMat) { + cv::GMat in; + auto blurred = cv::gapi::blur(in, cv::Size{3,3}); + auto desynced = cv::gapi::streaming::desync(blurred); + auto out = in - blurred; + auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)).compileStreaming(); + + cv::Size sz(32,32); + cv::Mat in_mat(sz, CV_8UC3); + cv::randu(in_mat, cv::Scalar::all(0), cv::Scalar(255)); + pipe.setSource(cv::gin(in_mat)); + pipe.start(); + + cv::optional out_desync; + cv::optional out_rmat; + while (true) { + // Initially it throwed "bad variant access" since there was + // no RMat handling in wrap_opt_arg + EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_rmat))); + if (out_rmat) break; + } +} + +G_API_OP(GTestBlur, , "test.blur") { + static GFrameDesc outMeta(GFrameDesc d) { return d; } +}; +GAPI_OCV_KERNEL(GOcvTestBlur, GTestBlur) { + static void run(const cv::MediaFrame& in, cv::MediaFrame& out) { + auto d = in.desc(); + GAPI_Assert(d.fmt == cv::MediaFormat::BGR); + auto view = in.access(cv::MediaFrame::Access::R); + cv::Mat mat(d.size, CV_8UC3, view.ptr[0]); + cv::Mat blurred; + cv::blur(mat, blurred, cv::Size{3,3}); + out = cv::MediaFrame::Create(blurred); + } +}; + +TEST(GAPI_Streaming, TestDesyncMediaFrame) { + initTestDataPath(); + cv::GFrame in; + auto blurred = GTestBlur::on(in); + auto desynced = cv::gapi::streaming::desync(blurred); + auto out = GTestBlur::on(blurred); + auto pipe = cv::GComputation(cv::GIn(in), cv::GOut(desynced, out)) + .compileStreaming(cv::compile_args(cv::gapi::kernels())); + + std::string filepath = findDataFile("cv/video/768x576.avi"); + try { + pipe.setSource(filepath); + } catch(...) { + throw SkipTestException("Video file can not be opened"); + } + pipe.start(); + + cv::optional out_desync; + cv::optional out_frame; + while (true) { + // Initially it throwed "bad variant access" since there was + // no MediaFrame handling in wrap_opt_arg + EXPECT_NO_THROW(pipe.pull(cv::gout(out_desync, out_frame))); + if (out_frame) break; + } +} + } // namespace opencv_test diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp new file mode 100644 index 0000000000..0b8822b366 --- /dev/null +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -0,0 +1,446 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2021 Intel Corporation + + +#include "../test_precomp.hpp" + +#include "../common/gapi_tests_common.hpp" + +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +#ifdef HAVE_ONEVPL +#include + +#include "streaming/onevpl/accelerators/surface/surface.hpp" +#include "streaming/onevpl/accelerators/surface/cpu_frame_adapter.hpp" +#include "streaming/onevpl/accelerators/accel_policy_cpu.hpp" +#include "streaming/onevpl/engine/processing_engine_base.hpp" +#include "streaming/onevpl/engine/engine_session.hpp" + +namespace opencv_test +{ +namespace +{ + +struct EmptyDataProvider : public cv::gapi::wip::onevpl::IDataProvider { + + size_t fetch_data(size_t, void*) override { + return 0; + } + bool empty() const override { + return true; + } +}; + +struct TestProcessingSession : public cv::gapi::wip::onevpl::EngineSession { + TestProcessingSession(mfxSession mfx_session) : + EngineSession(mfx_session, {}) { + } +}; + +struct TestProcessingEngine: public cv::gapi::wip::onevpl::ProcessingEngineBase { + + size_t pipeline_stage_num = 0; + + TestProcessingEngine(std::unique_ptr&& accel) : + cv::gapi::wip::onevpl::ProcessingEngineBase(std::move(accel)) { + using cv::gapi::wip::onevpl::EngineSession; + create_pipeline( + // 0) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 0; + return ExecutionStatus::Continue; + }, + // 1) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 1; + return ExecutionStatus::Continue; + }, + // 2) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 2; + return ExecutionStatus::Continue; + }, + // 3) + [this] (EngineSession&) -> ExecutionStatus + { + pipeline_stage_num = 3; + ready_frames.emplace(cv::MediaFrame()); + return ExecutionStatus::Processed; + } + ); + } + + void initialize_session(mfxSession mfx_session, + cv::gapi::wip::onevpl::DecoderParams&&, + std::shared_ptr) override { + + register_session(mfx_session); + } +}; + +cv::gapi::wip::onevpl::surface_ptr_t create_test_surface(std::shared_ptr out_buf_ptr, + size_t, size_t) { + std::unique_ptr handle(new mfxFrameSurface1{}); + return cv::gapi::wip::onevpl::Surface::create_surface(std::move(handle), out_buf_ptr); +} + +TEST(OneVPL_Source_Surface, InitSurface) +{ + using namespace cv::gapi::wip::onevpl; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + mfxFrameSurface1 *mfx_core_handle = handle.get(); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check self consistency + EXPECT_EQ(reinterpret_cast(surf->get_handle()), + reinterpret_cast(mfx_core_handle)); + EXPECT_EQ(surf->get_locks_count(), 0); + EXPECT_EQ(surf->obtain_lock(), 0); + EXPECT_EQ(surf->get_locks_count(), 1); + EXPECT_EQ(surf->release_lock(), 1); + EXPECT_EQ(surf->get_locks_count(), 0); +} + +TEST(OneVPL_Source_Surface, ConcurrentLock) +{ + using namespace cv::gapi::wip::onevpl; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check self consistency + EXPECT_EQ(surf->get_locks_count(), 0); + + // MFX internal limitation: do not exceede U16 range + // so I16 is using here + int16_t lock_counter = std::numeric_limits::max() - 1; + std::promise barrier; + std::future sync = barrier.get_future(); + + + std::thread worker_thread([&barrier, surf, lock_counter] () { + barrier.set_value(); + + // concurrent lock + for (int16_t i = 0; i < lock_counter; i ++) { + surf->obtain_lock(); + } + }); + sync.wait(); + + // concurrent lock + for (int16_t i = 0; i < lock_counter; i ++) { + surf->obtain_lock(); + } + + worker_thread.join(); + EXPECT_EQ(surf->get_locks_count(), lock_counter * 2); +} + +TEST(OneVPL_Source_Surface, MemoryLifeTime) +{ + using namespace cv::gapi::wip::onevpl; + + // create preallocate surface memory + std::unique_ptr preallocated_memory_ptr(new char); + std::shared_ptr associated_memory (preallocated_memory_ptr.get(), + [&preallocated_memory_ptr] (void* ptr) { + EXPECT_TRUE(preallocated_memory_ptr); + EXPECT_EQ(preallocated_memory_ptr.get(), ptr); + preallocated_memory_ptr.reset(); + }); + + // generate surfaces + constexpr size_t surface_num = 10000; + std::vector> surfaces(surface_num); + std::generate(surfaces.begin(), surfaces.end(), [surface_num, associated_memory](){ + std::unique_ptr handle(new mfxFrameSurface1{}); + return Surface::create_surface(std::move(handle), associated_memory); + }); + + // destroy surfaces + { + std::thread deleter_thread([&surfaces]() { + surfaces.clear(); + }); + deleter_thread.join(); + } + + // workspace memory must be alive + EXPECT_EQ(surfaces.size(), 0); + EXPECT_TRUE(associated_memory != nullptr); + EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); + + // generate surfaces again + 1 + constexpr size_t surface_num_plus_one = 10001; + surfaces.resize(surface_num_plus_one); + std::generate(surfaces.begin(), surfaces.end(), [surface_num_plus_one, associated_memory](){ + std::unique_ptr handle(new mfxFrameSurface1{}); + return Surface::create_surface(std::move(handle), associated_memory); + }); + + // remember one surface + std::shared_ptr last_surface = surfaces.back(); + + // destroy another surfaces + surfaces.clear(); + + // destroy associated_memory + associated_memory.reset(); + + // workspace memory must be still alive + EXPECT_EQ(surfaces.size(), 0); + EXPECT_TRUE(associated_memory == nullptr); + EXPECT_TRUE(preallocated_memory_ptr.get() != nullptr); + + // destroy last surface + last_surface.reset(); + + // workspace memory must be freed + EXPECT_TRUE(preallocated_memory_ptr.get() == nullptr); +} + +TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) +{ + using namespace cv::gapi::wip::onevpl; + + // create raw MFX handle + std::unique_ptr handle(new mfxFrameSurface1{}); + + // create preallocate surface memory: empty for test + std::shared_ptr associated_memory {}; + auto surf = Surface::create_surface(std::move(handle), associated_memory); + + // check consistency + EXPECT_EQ(surf->get_locks_count(), 0); + + { + VPLMediaFrameCPUAdapter adapter(surf); + EXPECT_EQ(surf->get_locks_count(), 1); + } + EXPECT_EQ(surf->get_locks_count(), 0); +} + +TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) +{ + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + size_t pool_count = 3; + std::vector pool_export_keys; + pool_export_keys.reserve(pool_count); + + // create several pools + for (size_t i = 0; i < pool_count; i++) + { + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + pool_export_keys.push_back(key); + } + + EXPECT_NO_THROW(acceleration_policy.reset()); +} + +TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConsume) +{ + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::Surface; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + // consume available surfaces + std::vector> surfaces; + surfaces.reserve(surface_count); + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + surfaces.push_back(std::move(surf)); + } + + // check consistency (no free surfaces) + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), 0); + + // fail consume non-free surfaces + for (size_t i = 0; i < surface_count; i++) { + EXPECT_THROW(acceleration_policy->get_free_surface(key), std::runtime_error); + } + + // release surfaces + for (auto& surf : surfaces) { + EXPECT_EQ(surf->release_lock(), 1); + } + surfaces.clear(); + + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + //check availability after release + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + } +} + +TEST(OneVPL_Source_CPU_Accelerator, PoolProduceConcurrentConsume) +{ + using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy; + using cv::gapi::wip::onevpl::VPLAccelerationPolicy; + using cv::gapi::wip::onevpl::Surface; + + auto acceleration_policy = std::make_shared(); + + size_t surface_count = 10; + size_t surface_size_bytes = 1024; + + VPLAccelerationPolicy::pool_key_t key = + acceleration_policy->create_surface_pool(surface_count, + surface_size_bytes, + create_test_surface); + + // check consistency + EXPECT_EQ(acceleration_policy->get_surface_count(key), surface_count); + EXPECT_EQ(acceleration_policy->get_free_surface_count(key), surface_count); + + // consume available surfaces + std::vector> surfaces; + surfaces.reserve(surface_count); + for (size_t i = 0; i < surface_count; i++) { + std::shared_ptr surf = acceleration_policy->get_free_surface(key).lock(); + EXPECT_TRUE(surf.get() != nullptr); + EXPECT_EQ(surf->obtain_lock(), 0); + surfaces.push_back(std::move(surf)); + } + + std::promise launch_promise; + std::future sync = launch_promise.get_future(); + std::promise surface_released_promise; + std::future released_result = surface_released_promise.get_future(); + std::thread worker_thread([&launch_promise, &surface_released_promise, &surfaces] () { + launch_promise.set_value(); + + // concurrent release surfaces + size_t surfaces_count = surfaces.size(); + for (auto& surf : surfaces) { + EXPECT_EQ(surf->release_lock(), 1); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + surfaces.clear(); + + surface_released_promise.set_value(surfaces_count); + }); + sync.wait(); + + // check free surface concurrently + std::future_status status; + size_t free_surface_count = 0; + size_t free_surface_count_prev = 0; + do { + status = released_result.wait_for(std::chrono::seconds(1)); + free_surface_count = acceleration_policy->get_free_surface_count(key); + EXPECT_TRUE(free_surface_count >= free_surface_count_prev); + free_surface_count_prev = free_surface_count; + } while (status != std::future_status::ready); + std::cerr<< "Ready" << std::endl; + free_surface_count = acceleration_policy->get_free_surface_count(key); + worker_thread.join(); + EXPECT_TRUE(free_surface_count >= free_surface_count_prev); +} + +TEST(OneVPL_Source_ProcessingEngine, Init) +{ + using namespace cv::gapi::wip::onevpl; + std::unique_ptr accel; + TestProcessingEngine engine(std::move(accel)); + + mfxSession mfx_session{}; + engine.initialize_session(mfx_session, DecoderParams{}, std::shared_ptr{}); + + EXPECT_EQ(engine.get_ready_frames_count(), 0); + ProcessingEngineBase::ExecutionStatus ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 0); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 1); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Continue); + EXPECT_EQ(engine.pipeline_stage_num, 2); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::Processed); + EXPECT_EQ(engine.pipeline_stage_num, 3); + EXPECT_EQ(engine.get_ready_frames_count(), 1); + + ret = engine.process(mfx_session); + EXPECT_EQ(ret, ProcessingEngineBase::ExecutionStatus::SessionNotFound); + EXPECT_EQ(engine.pipeline_stage_num, 3); + EXPECT_EQ(engine.get_ready_frames_count(), 1); + + cv::gapi::wip::Data frame; + engine.get_frame(frame); +} +} +} // namespace opencv_test +#endif // HAVE_ONEVPL diff --git a/modules/gapi/test/util/variant_tests.cpp b/modules/gapi/test/util/variant_tests.cpp index 65d5e579f8..7725f9a702 100644 --- a/modules/gapi/test/util/variant_tests.cpp +++ b/modules/gapi/test/util/variant_tests.cpp @@ -354,6 +354,20 @@ TEST(Variant, Get) EXPECT_THROW(util::get(cv2), util::bad_variant_access); } +TEST(Variant, GetIndexed) +{ + const TestVar cv(42); + + // Test const& get() + EXPECT_EQ(42, util::get<0>(cv)); + EXPECT_THROW(util::get<1>(cv), util::bad_variant_access); + + // Test &get + TestVar cv2(std::string("42")); + EXPECT_EQ("42", util::get<1>(cv2)); + EXPECT_THROW(util::get<0>(cv2), util::bad_variant_access); +} + TEST(Variant, GetWrite) { util::variant v(42); @@ -486,4 +500,240 @@ TEST(Variant, EXT_IndexOf) static_assert(6u == V::index_of(), "Index is incorrect"); } +namespace test_validation +{ +struct MyType +{ + friend std::ostream& operator<<(std::ostream& out, const MyType& src) + { + return out << "MyType"; (void) src; + } +}; +class MyClass +{ + friend std::ostream& operator<<(std::ostream& out, const MyClass& src) + { + return out << "MyClass"; (void) src; + } +}; + +struct MyBoolParamIndexedVisitor : cv::util::static_indexed_visitor +{ + MyBoolParamIndexedVisitor(std::ostream &output) : out(output) {} + + template + bool visit(std::size_t index, Type val, int check) + { + bool result = false; + out << index << ":" << val <<","; + if(std::is_same::value) + { + result = !memcmp(&val, &check, sizeof(int)); + } + return result; + } + + std::ostream &out; +}; + +struct MyBoolNoParamNonIndexedVisitor : cv::util::static_indexed_visitor +{ + MyBoolNoParamNonIndexedVisitor(std::ostream &output) : out(output) {} + + template + bool visit(std::size_t index, Type val) + { + out << index << ":" << val <<","; + return true; + } + std::ostream &out; +}; + + +struct MyVoidNoParamNonIndexedVisitor : cv::util::static_visitor +{ + MyVoidNoParamNonIndexedVisitor(std::ostream &output) : out(output) {} + + template + void visit(Type val) + { + out << val << ","; + } + + std::ostream &out; +}; + + +struct MyVoidNoParamIndexedVisitor : cv::util::static_indexed_visitor +{ + MyVoidNoParamIndexedVisitor(std::ostream &output) : out(output) {} + + template + void visit(std::size_t Index, Type val) + { + out << Index << ":" << val <<","; + } + + std::ostream &out; +}; +} + +TEST(Variant, DynamicVisitor) +{ + using V = cv::util::variant; + V var{42}; + { + std::stringstream ss; + test_validation::MyBoolParamIndexedVisitor visitor(ss); + + EXPECT_TRUE(cv::util::visit(visitor, var, int{42})); + EXPECT_EQ(ss.str(), std::string("0:42,")); + } + + std::stringstream ss; + test_validation::MyBoolNoParamNonIndexedVisitor visitor(ss); + + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("0:42,")); + + var = double{1.0}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + + var = char{'a'}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + + var = float{6.0}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + + var = test_validation::MyType{}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + + var = test_validation::MyClass{}; + EXPECT_TRUE(cv::util::visit(visitor, var)); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); +} + +TEST(Variant, StaticVisitor) +{ + using V = cv::util::variant; + V var{42}; + std::stringstream ss; + test_validation::MyVoidNoParamNonIndexedVisitor visitor(ss); + + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,")); + + var = double{1.0}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,")); + + var = char{'a'}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,")); + + var = float{6.0}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,")); + + var = test_validation::MyType{}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,")); + + var = test_validation::MyClass{}; + cv::util::visit(visitor, var); + EXPECT_EQ(ss.str(), std::string("42,1,a,6,MyType,MyClass,")); +} + +TEST(Variant, StaticIndexedVisitor) +{ + using V = cv::util::variant; + V var{42}; + + std::stringstream ss; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor {ss}, var); + EXPECT_EQ(ss.str(), std::string("0:42,")); + + var = double{1.0}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,")); + + var = char{'a'}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,")); + + var = float{6.0}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,")); + + var = test_validation::MyType{}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,")); + + var = test_validation::MyClass{}; + cv::util::visit(test_validation::MyVoidNoParamIndexedVisitor (ss), var); + EXPECT_EQ(ss.str(), std::string("0:42,1:1,2:a,3:6,4:MyType,5:MyClass,")); +} + + +TEST(Variant, LambdaVisitor) +{ + using V = cv::util::variant; + V var{42}; + { + cv::util::visit(cv::util::overload_lambdas( + [](int value) { + EXPECT_EQ(value, 42); + }, + [](double) { + ADD_FAILURE() << "can't be called for `double`"; + }, + [](char) { + ADD_FAILURE() << "can't be called for `char`"; + }, + [](float) { + ADD_FAILURE() << "can't be called for `float`"; + }, + [](test_validation::MyType) { + ADD_FAILURE() << "can't be called for `MyType`"; + }, + [](test_validation::MyClass) { + ADD_FAILURE() << "can't be called for `MyClass`"; + }, + [](std::string) { + ADD_FAILURE() << "can't be called for `std::string`, invalid type"; + } + ), var); + } + + var = 'c'; + { + cv::util::visit(cv::util::overload_lambdas( + [](int) { + ADD_FAILURE() << "can't be called for `int`"; + }, + [](double) { + ADD_FAILURE() << "can't be called for `double`"; + }, + [](char value) { + EXPECT_EQ(value, 'c'); + }, + [](float) { + ADD_FAILURE() << "can't be called for `float`"; + }, + [](test_validation::MyType) { + ADD_FAILURE() << "can't be called for `MyType`"; + }, + [](test_validation::MyClass) { + ADD_FAILURE() << "can't be called for `MyClass`"; + }, + [](std::string) { + ADD_FAILURE() << "can't be called for `std::string`, invalid type"; + } + ), var); + } +} } // namespace opencv_test diff --git a/modules/highgui/CMakeLists.txt b/modules/highgui/CMakeLists.txt index b4d4b9f503..5c0f1e3278 100644 --- a/modules/highgui/CMakeLists.txt +++ b/modules/highgui/CMakeLists.txt @@ -49,52 +49,59 @@ list(REMOVE_ITEM highgui_ext_hdrs "${CMAKE_CURRENT_LIST_DIR}/include/opencv2/${n set(OPENCV_HIGHGUI_BUILTIN_BACKEND "") -if(HAVE_QT5) - set(OPENCV_HIGHGUI_BUILTIN_BACKEND "QT5") +if(HAVE_QT) + set(OPENCV_HIGHGUI_BUILTIN_BACKEND "QT${QT_VERSION_MAJOR}") add_definitions(-DHAVE_QT) - # "Automoc" doesn't work properly with opencv_world build, use QT5_WRAP_CPP() directly - #set(CMAKE_AUTOMOC ON) + if(QT_VERSION_MAJOR GREATER 4) + # "Automoc" doesn't work properly with opencv_world build, use QT_WRAP_CPP() directly + #set(CMAKE_AUTOMOC ON) - set(CMAKE_INCLUDE_CURRENT_DIR ON) + set(CMAKE_INCLUDE_CURRENT_DIR ON) - QT5_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) - QT5_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) - list(APPEND highgui_srcs - ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h - ${_MOC_OUTFILES} - ${_RCC_OUTFILES}) + if(QT_VERSION_MAJOR EQUAL 6) + QT6_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT6_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + elseif(QT_VERSION_MAJOR EQUAL 5) + QT5_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT5_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + else() + message(FATAL_ERROR "Unsuported QT version: ${QT_VERSION_MAJOR}") + endif() - foreach(dt5_dep Core Gui Widgets Test Concurrent) - add_definitions(${Qt5${dt5_dep}_DEFINITIONS}) - include_directories(${Qt5${dt5_dep}_INCLUDE_DIRS}) - list(APPEND HIGHGUI_LIBRARIES ${Qt5${dt5_dep}_LIBRARIES}) - endforeach() + list(APPEND highgui_srcs + ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h + ${_MOC_OUTFILES} + ${_RCC_OUTFILES}) - if(HAVE_QT_OPENGL) - add_definitions(-DHAVE_QT_OPENGL) - add_definitions(${Qt5OpenGL_DEFINITIONS}) - include_directories(${Qt5OpenGL_INCLUDE_DIRS}) - list(APPEND HIGHGUI_LIBRARIES ${Qt5OpenGL_LIBRARIES}) - endif() -elseif(HAVE_QT) - set(OPENCV_HIGHGUI_BUILTIN_BACKEND "QT4") - add_definitions(-DHAVE_QT) - if(HAVE_QT_OPENGL) - add_definitions(-DHAVE_QT_OPENGL) - set(QT_USE_QTOPENGL TRUE) - endif() - include(${QT_USE_FILE}) + set(qt_deps Core Gui Widgets Test Concurrent) + if(HAVE_QT_OPENGL) + add_definitions(-DHAVE_QT_OPENGL) + list(APPEND qt_deps OpenGL) + endif() - QT4_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) - QT4_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + foreach(dt_dep ${qt_deps}) + add_definitions(${Qt${QT_VERSION_MAJOR}${dt_dep}_DEFINITIONS}) + include_directories(${Qt${QT_VERSION_MAJOR}${dt_dep}_INCLUDE_DIRS}) + list(APPEND HIGHGUI_LIBRARIES ${Qt${QT_VERSION_MAJOR}${dt_dep}_LIBRARIES}) + endforeach() + else() + ocv_assert(QT_VERSION_MAJOR EQUAL 4) + if (HAVE_QT_OPENGL) + set(QT_USE_QTOPENGL TRUE) + endif() + include(${QT_USE_FILE}) - list(APPEND HIGHGUI_LIBRARIES ${QT_LIBRARIES}) - list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp ${_MOC_OUTFILES} ${_RCC_OUTFILES}) - ocv_check_flag_support(CXX -Wno-missing-declarations _have_flag "") - if(${_have_flag}) - set_source_files_properties(${_RCC_OUTFILES} PROPERTIES COMPILE_FLAGS -Wno-missing-declarations) + QT4_ADD_RESOURCES(_RCC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.qrc) + QT4_WRAP_CPP(_MOC_OUTFILES ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.h) + + list(APPEND HIGHGUI_LIBRARIES ${QT_LIBRARIES}) + list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_QT.cpp ${_MOC_OUTFILES} ${_RCC_OUTFILES}) + ocv_check_flag_support(CXX -Wno-missing-declarations _have_flag "") + if(${_have_flag}) + set_source_files_properties(${_RCC_OUTFILES} PROPERTIES COMPILE_FLAGS -Wno-missing-declarations) + endif() endif() elseif(WINRT) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WINRT") @@ -131,12 +138,6 @@ elseif(WINRT) message(STATUS " ${name}: Removing 'comctl32.lib, gdi32.lib, ole32.lib, setupapi.lib'") message(STATUS " ${name}: Leaving '${HIGHGUI_LIBRARIES}'") endif() -elseif(HAVE_WIN32UI) - set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WIN32UI") - list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_w32.cpp) - if(OpenCV_ARCH STREQUAL "ARM64") - list(APPEND HIGHGUI_LIBRARIES "comdlg32" "advapi32") - endif() elseif(HAVE_COCOA) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "COCOA") add_definitions(-DHAVE_COCOA) @@ -144,6 +145,16 @@ elseif(HAVE_COCOA) list(APPEND HIGHGUI_LIBRARIES "-framework Cocoa") endif() +if(TARGET ocv.3rdparty.win32ui) + if("win32ui" IN_LIST HIGHGUI_PLUGIN_LIST OR HIGHGUI_PLUGIN_LIST STREQUAL "all") + ocv_create_builtin_highgui_plugin(opencv_highgui_win32 ocv.3rdparty.win32ui "window_w32.cpp") + elseif(NOT OPENCV_HIGHGUI_BUILTIN_BACKEND) + set(OPENCV_HIGHGUI_BUILTIN_BACKEND "WIN32UI") + list(APPEND highgui_srcs ${CMAKE_CURRENT_LIST_DIR}/src/window_w32.cpp) + list(APPEND tgts ocv.3rdparty.win32ui) + endif() +endif() + if(TARGET ocv.3rdparty.gtk3 OR TARGET ocv.3rdparty.gtk2) if(TARGET ocv.3rdparty.gtk3 AND NOT WITH_GTK_2_X) set(__gtk_dependency "ocv.3rdparty.gtk3") @@ -214,7 +225,8 @@ endif() if(NOT OPENCV_HIGHGUI_BUILTIN_BACKEND) set(OPENCV_HIGHGUI_BUILTIN_BACKEND "NONE") endif() -message(STATUS "highgui: using builtin backend: ${OPENCV_HIGHGUI_BUILTIN_BACKEND}") # FIXIT: propagate to root CMake +message(STATUS "highgui: using builtin backend: ${OPENCV_HIGHGUI_BUILTIN_BACKEND}") +set(OPENCV_HIGHGUI_BUILTIN_BACKEND "${OPENCV_HIGHGUI_BUILTIN_BACKEND}" PARENT_SCOPE) # informational if(TRUE) # these variables are set by 'ocv_append_build_options(HIGHGUI ...)' @@ -254,7 +266,7 @@ if(APPLE) add_apple_compiler_options(${the_module}) endif() -if(MSVC) +if(MSVC AND NOT BUILD_SHARED_LIBS AND BUILD_WITH_STATIC_CRT) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /NODEFAULTLIB:libcmt.lib /DEBUG") endif() diff --git a/modules/highgui/cmake/detect_gtk.cmake b/modules/highgui/cmake/detect_gtk.cmake index cdc054fad0..c58246ac54 100644 --- a/modules/highgui/cmake/detect_gtk.cmake +++ b/modules/highgui/cmake/detect_gtk.cmake @@ -6,8 +6,6 @@ if(WITH_GTK) if(HAVE_GTK3) ocv_add_external_target(gtk3 "${GTK3_INCLUDE_DIRS}" "${GTK3_LIBRARIES}" "HAVE_GTK3;HAVE_GTK") set(HAVE_GTK TRUE) - set(HAVE_GTK3 ${HAVE_GTK3} PARENT_SCOPE) - set(GTK3_VERSION "${GTK3_VERSION}" PARENT_SCOPE) # informational endif() endif() if((PROJECT_NAME STREQUAL "OpenCV" AND HIGHGUI_ENABLE_PLUGINS) OR NOT HAVE_GTK3) @@ -19,8 +17,6 @@ if(WITH_GTK) else() ocv_add_external_target(gtk2 "${GTK2_INCLUDE_DIRS}" "${GTK2_LIBRARIES}" "HAVE_GTK2;HAVE_GTK") set(HAVE_GTK TRUE) - set(HAVE_GTK2 ${HAVE_GTK2} PARENT_SCOPE) - set(GTK2_VERSION "${GTK2_VERSION}" PARENT_SCOPE) # informational endif() endif() endif() @@ -29,15 +25,11 @@ if(WITH_GTK) message(FATAL_ERROR "gthread not found. This library is required when building with GTK support") else() ocv_add_external_target(gthread "${GTHREAD_INCLUDE_DIRS}" "${GTHREAD_LIBRARIES}" "HAVE_GTHREAD") - set(HAVE_GTHREAD "${HAVE_GTHREAD}" PARENT_SCOPE) # informational - set(GTHREAD_VERSION "${GTHREAD_VERSION}" PARENT_SCOPE) # informational endif() if((WITH_OPENGL OR HAVE_OPENGL) AND HAVE_GTK2) ocv_check_modules(GTKGLEXT gtkglext-1.0) if(HAVE_GTKGLEXT) ocv_add_external_target(gtkglext "${GTKGLEXT_INCLUDE_DIRS}" "${GTKGLEXT_LIBRARIES}" "HAVE_GTKGLEXT") - set(HAVE_GTKGLEXT "${HAVE_GTKGLEXT}" PARENT_SCOPE) # informational - set(GTKGLEXT_VERSION "${GTKGLEXT_VERSION}" PARENT_SCOPE) # informational endif() endif() elseif(HAVE_GTK) @@ -48,9 +40,6 @@ if(WITH_OPENGL AND HAVE_GTKGLEXT) find_package(OpenGL QUIET) if(OPENGL_FOUND) set(HAVE_OPENGL TRUE) - #set(HAVE_OPENGL ${HAVE_OPENGL} PARENT_SCOPE) ocv_add_external_target(gtk_opengl "${OPENGL_INCLUDE_DIRS}" "${OPENGL_LIBRARIES}" "HAVE_OPENGL") endif() endif() - -set(HAVE_GTK ${HAVE_GTK} PARENT_SCOPE) diff --git a/modules/highgui/cmake/detect_win32ui.cmake b/modules/highgui/cmake/detect_win32ui.cmake new file mode 100644 index 0000000000..c5e358ffa7 --- /dev/null +++ b/modules/highgui/cmake/detect_win32ui.cmake @@ -0,0 +1,15 @@ +#--- Win32 UI --- +ocv_clear_vars(HAVE_WIN32UI) +if(WITH_WIN32UI) + try_compile(HAVE_WIN32UI + "${CMAKE_CURRENT_BINARY_DIR}" + "${OpenCV_SOURCE_DIR}/cmake/checks/win32uitest.cpp" + CMAKE_FLAGS "-DLINK_LIBRARIES:STRING=user32;gdi32") + if(HAVE_WIN32UI) + set(__libs "user32" "gdi32") + if(OpenCV_ARCH STREQUAL "ARM64") + list(APPEND __libs "comdlg32" "advapi32") + endif() + ocv_add_external_target(win32ui "" "${__libs}" "HAVE_WIN32UI") + endif() +endif() diff --git a/modules/highgui/cmake/init.cmake b/modules/highgui/cmake/init.cmake index 3b766b3758..2002ff0e9d 100644 --- a/modules/highgui/cmake/init.cmake +++ b/modules/highgui/cmake/init.cmake @@ -25,26 +25,18 @@ endif() # Detect available dependencies # -include(FindPkgConfig) - -# FIXIT: stop using PARENT_SCOPE in dependencies -if(PROJECT_NAME STREQUAL "OpenCV") - macro(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endmacro() -else() - function(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endfunction() +if(NOT PROJECT_NAME STREQUAL "OpenCV") + include(FindPkgConfig) endif() -add_backend("gtk" WITH_GTK) +macro(add_backend backend_id cond_var) + if(${cond_var}) + include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") + endif() +endmacro() -# TODO win32 +add_backend("gtk" WITH_GTK) +add_backend("win32ui" WITH_WIN32UI) # TODO cocoa # TODO qt # TODO opengl diff --git a/modules/highgui/include/opencv2/highgui.hpp b/modules/highgui/include/opencv2/highgui.hpp index cdab0ada78..7ae2395737 100644 --- a/modules/highgui/include/opencv2/highgui.hpp +++ b/modules/highgui/include/opencv2/highgui.hpp @@ -383,10 +383,12 @@ cv::WINDOW_AUTOSIZE flag, the image is shown with its original size, however it Otherwise, the image is scaled to fit the window. The function may scale the image, depending on its depth: - If the image is 8-bit unsigned, it is displayed as is. -- If the image is 16-bit unsigned or 32-bit integer, the pixels are divided by 256. That is, the +- If the image is 16-bit unsigned, the pixels are divided by 256. That is, the value range [0,255\*256] is mapped to [0,255]. - If the image is 32-bit or 64-bit floating-point, the pixel values are multiplied by 255. That is, the value range [0,1] is mapped to [0,255]. +- 32-bit integer images are not processed anymore due to ambiguouty of required transform. + Convert to 8-bit unsigned matrix using a custom preprocessing specific to image's context. If window was created with OpenGL support, cv::imshow also support ogl::Buffer , ogl::Texture2D and cuda::GpuMat as input. diff --git a/modules/highgui/src/backend.hpp b/modules/highgui/src/backend.hpp index 14c88b2387..7c32846ce4 100644 --- a/modules/highgui/src/backend.hpp +++ b/modules/highgui/src/backend.hpp @@ -114,6 +114,10 @@ bool setUIBackend(const std::string& backendName); #ifndef BUILD_PLUGIN +#ifdef HAVE_WIN32UI +std::shared_ptr createUIBackendWin32UI(); +#endif + #ifdef HAVE_GTK std::shared_ptr createUIBackendGTK(); #endif diff --git a/modules/highgui/src/plugin_wrapper.impl.hpp b/modules/highgui/src/plugin_wrapper.impl.hpp index 97aea69098..23f0ecf6bf 100644 --- a/modules/highgui/src/plugin_wrapper.impl.hpp +++ b/modules/highgui/src/plugin_wrapper.impl.hpp @@ -224,6 +224,9 @@ std::vector getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_DEBUG(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } diff --git a/modules/highgui/src/precomp.hpp b/modules/highgui/src/precomp.hpp index 6ad5bce8b4..0d26b957ad 100644 --- a/modules/highgui/src/precomp.hpp +++ b/modules/highgui/src/precomp.hpp @@ -67,7 +67,6 @@ #include #include #include -#include #if defined _WIN32 || defined WINCE #include @@ -127,6 +126,13 @@ void cvSetPropTopmost_COCOA(const char* name, const bool topmost); double cvGetPropVsync_W32(const char* name); void cvSetPropVsync_W32(const char* name, const bool enabled); +void setWindowTitle_W32(const cv::String& name, const cv::String& title); +void setWindowTitle_GTK(const cv::String& name, const cv::String& title); +void setWindowTitle_QT(const cv::String& name, const cv::String& title); +void setWindowTitle_COCOA(const cv::String& name, const cv::String& title); + +int pollKey_W32(); + //for QT #if defined (HAVE_QT) CvRect cvGetWindowRect_QT(const char* name); diff --git a/modules/highgui/src/registry.impl.hpp b/modules/highgui/src/registry.impl.hpp index ccf81f9280..66693f1b07 100644 --- a/modules/highgui/src/registry.impl.hpp +++ b/modules/highgui/src/registry.impl.hpp @@ -50,6 +50,14 @@ std::vector& getBuiltinBackendsInfo() #elif defined(ENABLE_PLUGINS) DECLARE_DYNAMIC_BACKEND("QT") #endif +#endif + +#ifdef _WIN32 +#ifdef HAVE_WIN32UI + DECLARE_STATIC_BACKEND("WIN32", createUIBackendWin32UI) +#elif defined(ENABLE_PLUGINS) + DECLARE_DYNAMIC_BACKEND("WIN32") +#endif #endif }; return g_backends; diff --git a/modules/highgui/src/window.cpp b/modules/highgui/src/window.cpp index dca6ddbaf8..d9481de6da 100644 --- a/modules/highgui/src/window.cpp +++ b/modules/highgui/src/window.cpp @@ -586,6 +586,46 @@ void cv::moveWindow( const String& winname, int x, int y ) #endif } +void cv::setWindowTitle(const String& winname, const String& title) +{ + CV_TRACE_FUNCTION(); + + { + cv::AutoLock lock(cv::getWindowMutex()); + auto window = findWindow_(winname); + if (window) + { + return window->setTitle(title); + } + } + +#if defined(OPENCV_HIGHGUI_WITHOUT_BUILTIN_BACKEND) && defined(ENABLE_PLUGINS) + auto backend = getCurrentUIBackend(); + if (backend) + { + CV_LOG_WARNING(NULL, "Can't find window with name: '" << winname << "'. Do nothing"); + CV_NOT_FOUND_DEPRECATION; + } + else + { + CV_LOG_WARNING(NULL, "No UI backends available. Use OPENCV_LOG_LEVEL=DEBUG for investigation"); + } + return; +#elif defined(HAVE_WIN32UI) + return setWindowTitle_W32(winname, title); +#elif defined (HAVE_GTK) + return setWindowTitle_GTK(winname, title); +#elif defined (HAVE_QT) + return setWindowTitle_QT(winname, title); +#elif defined (HAVE_COCOA) + return setWindowTitle_COCOA(winname, title); +#else + CV_Error(Error::StsNotImplemented, "The function is not implemented. " + "Rebuild the library with Windows, GTK+ 2.x or Cocoa support. " + "If you are on Ubuntu or Debian, install libgtk2.0-dev and pkg-config, then re-run cmake or configure script"); +#endif +} + void cv::setWindowProperty(const String& winname, int prop_id, double prop_value) { CV_TRACE_FUNCTION(); @@ -630,10 +670,9 @@ int cv::waitKey(int delay) return (code != -1) ? (code & 0xff) : -1; } -#if defined(HAVE_WIN32UI) -// pollKey() implemented in window_w32.cpp -#elif defined(HAVE_GTK) || defined(HAVE_COCOA) || defined(HAVE_QT) || (defined (WINRT) && !defined (WINRT_8_0)) -// pollKey() fallback implementation +/* + * process until queue is empty but don't wait. + */ int cv::pollKey() { CV_TRACE_FUNCTION(); @@ -647,10 +686,13 @@ int cv::pollKey() } } +#if defined(HAVE_WIN32UI) + return pollKey_W32(); +#else // fallback. please implement a proper polling function return cvWaitKey(1); -} #endif +} int cv::createTrackbar(const String& trackbarName, const String& winName, int* value, int count, TrackbarCallback callback, @@ -948,7 +990,7 @@ void cv::imshow( const String& winname, InputArray _img ) auto backend = getCurrentUIBackend(); if (backend) { - auto window = backend->createWindow(winname, WINDOW_NORMAL); + auto window = backend->createWindow(winname, WINDOW_AUTOSIZE); if (!window) { CV_LOG_ERROR(NULL, "OpenCV/UI: Can't create window: '" << winname << "'"); @@ -1202,13 +1244,6 @@ int cv::createButton(const String&, ButtonCallback, void*, int , bool ) // version with a more capable one without a need to recompile dependent // applications or libraries. -void cv::setWindowTitle(const String&, const String&) -{ - CV_Error(Error::StsNotImplemented, "The function is not implemented. " - "Rebuild the library with Windows, GTK+ 2.x or Cocoa support. " - "If you are on Ubuntu or Debian, install libgtk2.0-dev and pkg-config, then re-run cmake or configure script"); -} - #define CV_NO_GUI_ERROR(funcname) \ cv::error(cv::Error::StsError, \ "The function is not implemented. " \ @@ -1359,11 +1394,6 @@ CV_IMPL int cvCreateButton(const char*, void (*)(int, void*), void*, int, int) CV_NO_GUI_ERROR("cvCreateButton"); } -int cv::pollKey() -{ - CV_NO_GUI_ERROR("cv::pollKey()"); -} - #endif /* End of file. */ diff --git a/modules/highgui/src/window_QT.cpp b/modules/highgui/src/window_QT.cpp index 60d7d69a59..e3c831ab50 100644 --- a/modules/highgui/src/window_QT.cpp +++ b/modules/highgui/src/window_QT.cpp @@ -63,6 +63,39 @@ #endif #endif +using namespace cv; + +#if QT_VERSION >= QT_VERSION_CHECK(5, 15, 0) +#define Qt_MiddleButton Qt::MiddleButton +inline Qt::Orientation wheelEventOrientation(QWheelEvent *we) { + if (std::abs(we->angleDelta().x()) < std::abs(we->angleDelta().y())) + return Qt::Vertical; + else + return Qt::Horizontal; +} +inline int wheelEventDelta(QWheelEvent *we) { + if(wheelEventOrientation(we) == Qt::Vertical) + return we->angleDelta().y(); + else + return we->angleDelta().x(); +} +inline QPoint wheelEventPos(QWheelEvent *we) { + return we->position().toPoint(); +} +#else +#define Qt_MiddleButton Qt::MidButton +inline Qt::Orientation wheelEventOrientation(QWheelEvent *we) { + return we->orientation(); +} +inline int wheelEventDelta(QWheelEvent *we) { + return we->delta(); +} +inline QPoint wheelEventPos(QWheelEvent *we) { + return we->pos(); +} + +#endif + //Static and global first static GuiReceiver *guiMainThread = NULL; @@ -197,7 +230,7 @@ void cvSetPropWindow_QT(const char* name,double prop_value) Q_ARG(double, prop_value)); } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_QT(const String& winname, const String& title) { if (!guiMainThread) CV_Error(Error::StsNullPtr, "NULL guiReceiver (please create a window)"); @@ -1579,7 +1612,9 @@ CvWinProperties::CvWinProperties(QString name_paraWindow, QObject* /*parent*/) myLayout->setObjectName(QString::fromUtf8("boxLayout")); myLayout->setContentsMargins(0, 0, 0, 0); myLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myLayout->setMargin(0); +#endif myLayout->setSizeConstraint(QLayout::SetFixedSize); setLayout(myLayout); @@ -1957,7 +1992,9 @@ void CvWindow::createBarLayout() myBarLayout->setObjectName(QString::fromUtf8("barLayout")); myBarLayout->setContentsMargins(0, 0, 0, 0); myBarLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myBarLayout->setMargin(0); +#endif } @@ -1967,7 +2004,9 @@ void CvWindow::createGlobalLayout() myGlobalLayout->setObjectName(QString::fromUtf8("boxLayout")); myGlobalLayout->setContentsMargins(0, 0, 0, 0); myGlobalLayout->setSpacing(0); +#if QT_VERSION < QT_VERSION_CHECK(5, 13, 0) myGlobalLayout->setMargin(0); +#endif setMinimumSize(1, 1); if (param_flags == CV_WINDOW_AUTOSIZE) @@ -2205,7 +2244,7 @@ void CvWindow::icvLoadControlPanel() } if (t->type == type_CvButtonbar) { - int subsize = settings.beginReadArray(QString("buttonbar")+i); + int subsize = settings.beginReadArray(QString("buttonbar%1").arg(i)); if ( subsize == ((CvButtonbar*)t)->layout()->count() ) icvLoadButtonbar((CvButtonbar*)t,&settings); @@ -2236,7 +2275,7 @@ void CvWindow::icvSaveControlPanel() } if (t->type == type_CvButtonbar) { - settings.beginWriteArray(QString("buttonbar")+i); + settings.beginWriteArray(QString("buttonbar%1").arg(i)); icvSaveButtonbar((CvButtonbar*)t,&settings); settings.endArray(); } @@ -2396,14 +2435,14 @@ void OCVViewPort::icvmouseHandler(QMouseEvent* evnt, type_mouse_event category, flags |= CV_EVENT_FLAG_LBUTTON; if(buttons & Qt::RightButton) flags |= CV_EVENT_FLAG_RBUTTON; - if(buttons & Qt::MidButton) + if(buttons & Qt_MiddleButton) flags |= CV_EVENT_FLAG_MBUTTON; if (cv_event == -1) { if (category == mouse_wheel) { QWheelEvent *we = (QWheelEvent *) evnt; - cv_event = ((we->orientation() == Qt::Vertical) ? CV_EVENT_MOUSEWHEEL : CV_EVENT_MOUSEHWHEEL); - flags |= (we->delta() & 0xffff)<<16; + cv_event = ((wheelEventOrientation(we) == Qt::Vertical) ? CV_EVENT_MOUSEWHEEL : CV_EVENT_MOUSEHWHEEL); + flags |= (wheelEventDelta(we) & 0xffff)<<16; return; } switch(evnt->button()) @@ -2416,7 +2455,7 @@ void OCVViewPort::icvmouseHandler(QMouseEvent* evnt, type_mouse_event category, cv_event = tableMouseButtons[category][1]; flags |= CV_EVENT_FLAG_RBUTTON; break; - case Qt::MidButton: + case Qt_MiddleButton: cv_event = tableMouseButtons[category][2]; flags |= CV_EVENT_FLAG_MBUTTON; break; @@ -2771,7 +2810,7 @@ void DefaultViewPort::wheelEvent(QWheelEvent* evnt) { icvmouseEvent((QMouseEvent *)evnt, mouse_wheel); - scaleView(evnt->delta() / 240.0, evnt->pos()); + scaleView(wheelEventDelta(evnt) / 240.0, wheelEventPos(evnt)); viewport()->update(); QWidget::wheelEvent(evnt); @@ -2882,18 +2921,19 @@ inline bool DefaultViewPort::isSameSize(IplImage* img1, IplImage* img2) void DefaultViewPort::controlImagePosition() { qreal left, top, right, bottom; + qreal factor = 1.0 / param_matrixWorld.m11(); //after check top-left, bottom right corner to avoid getting "out" during zoom/panning param_matrixWorld.map(0,0,&left,&top); if (left > 0) { - param_matrixWorld.translate(-left,0); + param_matrixWorld.translate(-left * factor, 0); left = 0; } if (top > 0) { - param_matrixWorld.translate(0,-top); + param_matrixWorld.translate(0, -top * factor); top = 0; } //------- @@ -2902,12 +2942,12 @@ void DefaultViewPort::controlImagePosition() param_matrixWorld.map(sizeImage.width(),sizeImage.height(),&right,&bottom); if (right < sizeImage.width()) { - param_matrixWorld.translate(sizeImage.width()-right,0); + param_matrixWorld.translate((sizeImage.width() - right) * factor, 0); right = sizeImage.width(); } if (bottom < sizeImage.height()) { - param_matrixWorld.translate(0,sizeImage.height()-bottom); + param_matrixWorld.translate(0, (sizeImage.height() - bottom) * factor); bottom = sizeImage.height(); } diff --git a/modules/highgui/src/window_cocoa.mm b/modules/highgui/src/window_cocoa.mm index 29a0278c98..e8e9034406 100644 --- a/modules/highgui/src/window_cocoa.mm +++ b/modules/highgui/src/window_cocoa.mm @@ -795,18 +795,18 @@ void cvSetPropTopmost_COCOA( const char* name, const bool topmost ) __END__; } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_COCOA(const cv::String& winname, const cv::String& title) { CVWindow *window = cvGetWindow(winname.c_str()); if (window == NULL) { - namedWindow(winname); + cv::namedWindow(winname); window = cvGetWindow(winname.c_str()); } if (window == NULL) - CV_Error(Error::StsNullPtr, "NULL window"); + CV_Error(cv::Error::StsNullPtr, "NULL window"); NSAutoreleasePool* localpool = [[NSAutoreleasePool alloc] init]; diff --git a/modules/highgui/src/window_gtk.cpp b/modules/highgui/src/window_gtk.cpp index efa3fbd96f..3428586ea3 100644 --- a/modules/highgui/src/window_gtk.cpp +++ b/modules/highgui/src/window_gtk.cpp @@ -364,7 +364,7 @@ static void cvImageWidget_set_size(GtkWidget * widget, int max_width, int max_he } - assert( image_widget->scaled_image ); + CV_Assert(image_widget->scaled_image); } static void @@ -849,7 +849,7 @@ static bool setModeWindow_(const std::shared_ptr& window, int mode) return false; } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_GTK(const String& winname, const String& title) { CV_LOCK_MUTEX(); @@ -2023,6 +2023,7 @@ static gboolean icvOnMouse( GtkWidget *widget, GdkEvent *event, gpointer user_da (unsigned)pt.y < (unsigned)(image_widget->original_image->height) )) { + state &= gtk_accelerator_get_default_mod_mask(); flags |= BIT_MAP(state, GDK_SHIFT_MASK, CV_EVENT_FLAG_SHIFTKEY) | BIT_MAP(state, GDK_CONTROL_MASK, CV_EVENT_FLAG_CTRLKEY) | BIT_MAP(state, GDK_MOD1_MASK, CV_EVENT_FLAG_ALTKEY) | diff --git a/modules/highgui/src/window_w32.cpp b/modules/highgui/src/window_w32.cpp index c4f2ddd2a6..43a19baf89 100644 --- a/modules/highgui/src/window_w32.cpp +++ b/modules/highgui/src/window_w32.cpp @@ -41,12 +41,17 @@ #include "precomp.hpp" +#ifdef HAVE_WIN32UI + +#include +#include + +#include "backend.hpp" + using namespace cv; #include // required for GET_X_LPARAM() and GET_Y_LPARAM() macros -#if defined _WIN32 - #ifdef __GNUC__ # pragma GCC diagnostic ignored "-Wmissing-declarations" #endif @@ -60,14 +65,12 @@ using namespace cv; #include #include #include -#include #ifdef HAVE_OPENGL #include #include #include #include -#include "opencv2/highgui.hpp" #include #include "opencv2/core/opengl.hpp" #endif @@ -78,7 +81,7 @@ static const char* trackbar_text = #if defined _M_X64 || defined __x86_64 || defined _M_ARM64 #define icvGetWindowLongPtr GetWindowLongPtr -#define icvSetWindowLongPtr( hwnd, id, ptr ) SetWindowLongPtr( hwnd, id, (LONG_PTR)(ptr) ) +#define icvSetWindowLongPtr(hwnd, id, ptr) SetWindowLongPtr(hwnd, id, (LONG_PTR)(ptr)) #define icvGetClassLongPtr GetClassLongPtr #define CV_USERDATA GWLP_USERDATA @@ -89,7 +92,7 @@ static const char* trackbar_text = #else #define icvGetWindowLongPtr GetWindowLong -#define icvSetWindowLongPtr( hwnd, id, ptr ) SetWindowLong( hwnd, id, (size_t)ptr ) +#define icvSetWindowLongPtr(hwnd, id, ptr) SetWindowLong(hwnd, id, (size_t)ptr) #define icvGetClassLongPtr GetClassLong #define CV_USERDATA GWL_USERDATA @@ -116,13 +119,13 @@ static inline void mingw_strcat_s(char *dest, size_t destsz, const char *src){ #define strcat_s mingw_strcat_s #endif -static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int origin ) +static void FillBitmapInfo(BITMAPINFO* bmi, int width, int height, int bpp, int origin) { - assert( bmi && width >= 0 && height >= 0 && (bpp == 8 || bpp == 24 || bpp == 32)); + CV_Assert(bmi && width >= 0 && height >= 0 && (bpp == 8 || bpp == 24 || bpp == 32)); BITMAPINFOHEADER* bmih = &(bmi->bmiHeader); - memset( bmih, 0, sizeof(*bmih)); + memset(bmih, 0, sizeof(*bmih)); bmih->biSize = sizeof(BITMAPINFOHEADER); bmih->biWidth = width; bmih->biHeight = origin ? abs(height) : -abs(height); @@ -130,11 +133,11 @@ static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int bmih->biBitCount = (unsigned short)bpp; bmih->biCompression = BI_RGB; - if( bpp == 8 ) + if (bpp == 8) { RGBQUAD* palette = bmi->bmiColors; int i; - for( i = 0; i < 256; i++ ) + for (i = 0; i < 256; i++) { palette[i].rgbBlue = palette[i].rgbGreen = palette[i].rgbRed = (BYTE)i; palette[i].rgbReserved = 0; @@ -144,68 +147,91 @@ static void FillBitmapInfo( BITMAPINFO* bmi, int width, int height, int bpp, int struct CvWindow; -typedef struct CvTrackbar +struct CvTrackbar : public std::enable_shared_from_this { + CvTrackbar(CvWindow& window, const std::string& name_) + : signature(CV_TRACKBAR_MAGIC_VAL) + , name(name_) + , parent(&window) + { + // nothing + } + ~CvTrackbar() + { + signature = -1; + } + int signature; - HWND hwnd; - char* name; - CvTrackbar* next; - CvWindow* parent; - HWND buddy; - int* data; - int pos; - int maxval; - int minval; - void (*notify)(int); - void (*notify2)(int, void*); - void* userdata; - int id; -} -CvTrackbar; + HWND hwnd = 0; + std::string name; + CvWindow* parent; // TODO weak_ptr + HWND buddy = 0; + int* data = nullptr; + int pos = 0; + int maxval = 0; + int minval = 0; + void (*notify)(int) = nullptr; // deprecated + void (*notify2)(int, void*) = nullptr; // deprecated + TrackbarCallback onChangeCallback = nullptr; + void* userdata = nullptr; + int id = -1; +}; -typedef struct CvWindow +struct CvWindow : public std::enable_shared_from_this { + CvWindow(const std::string& name_) + : signature(CV_WINDOW_MAGIC_VAL) + , name(name_) + { + // nothing + } + + ~CvWindow() + { + signature = -1; + } + + void destroy(); + int signature; - HWND hwnd; - char* name; - CvWindow* prev; - CvWindow* next; - HWND frame; + cv::Mutex mutex; + HWND hwnd = 0; + std::string name; + HWND frame = 0; - HDC dc; - HGDIOBJ image; - int last_key; - int flags; - int status;//0 normal, 1 fullscreen (YV) + HDC dc = 0; + HGDIOBJ image = 0; + int last_key = 0; + int flags = 0; + int status = 0;//0 normal, 1 fullscreen (YV) - CvMouseCallback on_mouse; - void* on_mouse_param; + CvMouseCallback on_mouse = nullptr; + void* on_mouse_param = nullptr; struct { - HWND toolbar; - int pos; - int rows; - WNDPROC toolBarProc; - CvTrackbar* first; + HWND toolbar = 0; + int pos = 0; + int rows = 0; + WNDPROC toolBarProc = nullptr; + std::vector< std::shared_ptr > trackbars; } toolbar; - int width; - int height; + int width = -1; + int height = -1; // OpenGL support #ifdef HAVE_OPENGL - bool useGl; - HGLRC hGLRC; + bool useGl = false; + HGLRC hGLRC = 0; - CvOpenGlDrawCallback glDrawCallback; - void* glDrawData; + CvOpenGlDrawCallback glDrawCallback = nullptr; + void* glDrawData = nullptr; #endif -} -CvWindow; +}; #define HG_BUDDY_WIDTH 130 @@ -221,19 +247,65 @@ CvWindow; #define TBM_GETTOOLTIPS (WM_USER + 30) #endif -static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static LRESULT CALLBACK MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); -static void icvUpdateWindowPos( CvWindow* window ); +static +std::vector< std::shared_ptr >& getWindowsList() +{ + static std::vector< std::shared_ptr > g_windows; + return g_windows; +} -static CvWindow* hg_windows = 0; + +// Mutex must be locked +static +std::shared_ptr icvFindWindowByName(const std::string& name) +{ + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window = *it; + if (!window) + continue; + if (window->name == name) + return window; + } + return std::shared_ptr(); +} + +static inline +std::shared_ptr icvFindWindowByName(const char* name) +{ + CV_Assert(name); + return icvFindWindowByName(std::string(name)); +} + +// Mutex must be locked +static +std::shared_ptr icvFindWindowByHandle(HWND hwnd) +{ + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window = *it; + if (!window) + continue; + if (window->hwnd == hwnd || window->frame == hwnd) + return window; + } + return std::shared_ptr(); +} + + +static LRESULT CALLBACK HighGUIProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static LRESULT CALLBACK MainWindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam); +static void icvUpdateWindowPos(CvWindow& window); typedef int (CV_CDECL * CvWin32WindowCallback)(HWND, UINT, WPARAM, LPARAM, int*); static CvWin32WindowCallback hg_on_preprocess = 0, hg_on_postprocess = 0; static HINSTANCE hg_hinstance = 0; -static const char* highGUIclassName = "HighGUI class"; -static const char* mainHighGUIclassName = "Main HighGUI class"; +static const char* const highGUIclassName = "HighGUI class"; +static const char* const mainHighGUIclassName = "Main HighGUI class"; static void icvCleanupHighgui() { @@ -242,15 +314,15 @@ static void icvCleanupHighgui() UnregisterClass(mainHighGUIclassName, hg_hinstance); } -CV_IMPL int cvInitSystem( int, char** ) +CV_IMPL int cvInitSystem(int, char**) { static int wasInitialized = 0; // check initialization status - if( !wasInitialized ) + if (!wasInitialized) { - // Initialize the storage - hg_windows = 0; + (void)getWindowMutex(); // force mutex initialization + (void)getWindowsList(); // Initialize the storage // Register the class WNDCLASS wndc; @@ -262,7 +334,7 @@ CV_IMPL int cvInitSystem( int, char** ) wndc.lpszClassName = highGUIclassName; wndc.lpszMenuName = highGUIclassName; wndc.hIcon = LoadIcon(0, IDI_APPLICATION); - wndc.hCursor = (HCURSOR)LoadCursor(0, (LPSTR)(size_t)IDC_CROSS ); + wndc.hCursor = (HCURSOR)LoadCursor(0, (LPSTR)(size_t)IDC_CROSS); wndc.hbrBackground = (HBRUSH)GetStockObject(DKGRAY_BRUSH); RegisterClass(&wndc); @@ -273,12 +345,12 @@ CV_IMPL int cvInitSystem( int, char** ) wndc.lpfnWndProc = MainWindowProc; RegisterClass(&wndc); - atexit( icvCleanupHighgui ); + atexit(icvCleanupHighgui); wasInitialized = 1; } - setlocale(LC_NUMERIC,"C"); + setlocale(LC_NUMERIC,"C"); // FIXIT must be removed return 0; } @@ -287,50 +359,58 @@ CV_IMPL int cvStartWindowThread(){ return 0; } -static CvWindow* icvFindWindowByName( const char* name ) + +static std::shared_ptr icvWindowByHWND(HWND hwnd) { - CvWindow* window = hg_windows; - - for( ; window != 0 && strcmp( name, window->name) != 0; window = window->next ) - ; - - return window; -} - - -static CvWindow* icvWindowByHWND( HWND hwnd ) -{ - CvWindow* window = (CvWindow*)icvGetWindowLongPtr( hwnd, CV_USERDATA ); - return window != 0 && hg_windows != 0 && + AutoLock lock(getWindowMutex()); + CvWindow* window = (CvWindow*)icvGetWindowLongPtr(hwnd, CV_USERDATA); + window = window != 0 && window->signature == CV_WINDOW_MAGIC_VAL ? window : 0; + if (window) + { + return window->shared_from_this(); + } + else + { + return std::shared_ptr(); + } } -static CvTrackbar* icvTrackbarByHWND( HWND hwnd ) +static std::shared_ptr icvTrackbarByHWND(HWND hwnd) { - CvTrackbar* trackbar = (CvTrackbar*)icvGetWindowLongPtr( hwnd, CV_USERDATA ); - return trackbar != 0 && trackbar->signature == CV_TRACKBAR_MAGIC_VAL && + AutoLock lock(getWindowMutex()); + CvTrackbar* trackbar = (CvTrackbar*)icvGetWindowLongPtr(hwnd, CV_USERDATA); + trackbar = trackbar != 0 && trackbar->signature == CV_TRACKBAR_MAGIC_VAL && trackbar->hwnd == hwnd ? trackbar : 0; + if (trackbar) + { + return trackbar->shared_from_this(); + } + else + { + return std::shared_ptr(); + } } -static const char* icvWindowPosRootKey = "Software\\OpenCV\\HighGUI\\Windows\\"; +static const char* const icvWindowPosRootKey = "Software\\OpenCV\\HighGUI\\Windows\\"; // Window positions saving/loading added by Philip Gruebele. //pgruebele@cox.net // Restores the window position from the registry saved position. static void -icvLoadWindowPos( const char* name, CvRect& rect ) +icvLoadWindowPos(const char* name, CvRect& rect) { HKEY hkey; char szKey[1024]; - strcpy_s( szKey, 1024, icvWindowPosRootKey ); - strcat_s( szKey, 1024, name ); + strcpy_s(szKey, 1024, icvWindowPosRootKey); + strcat_s(szKey, 1024, name); rect.x = rect.y = CW_USEDEFAULT; rect.width = rect.height = 320; - if( RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_QUERY_VALUE,&hkey) == ERROR_SUCCESS ) + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_QUERY_VALUE,&hkey) == ERROR_SUCCESS) { // Yes we are installed. DWORD dwType = 0; @@ -379,16 +459,16 @@ icvLoadWindowPos( const char* name, CvRect& rect ) //pgruebele@cox.net // philipg. Saves the window position in the registry static void -icvSaveWindowPos( const char* name, CvRect rect ) +icvSaveWindowPos(const char* name, CvRect rect) { static const DWORD MAX_RECORD_COUNT = 100; HKEY hkey; char szKey[1024]; char rootKey[1024]; - strcpy_s( szKey, 1024, icvWindowPosRootKey ); - strcat_s( szKey, 1024, name ); + strcpy_s(szKey, 1024, icvWindowPosRootKey); + strcat_s(szKey, 1024, name); - if( RegOpenKeyEx( HKEY_CURRENT_USER,szKey,0,KEY_READ,&hkey) != ERROR_SUCCESS ) + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_READ,&hkey) != ERROR_SUCCESS) { HKEY hroot; DWORD count = 0; @@ -396,40 +476,40 @@ icvSaveWindowPos( const char* name, CvRect rect ) char oldestKey[1024]; char currentKey[1024]; - strcpy_s( rootKey, 1024, icvWindowPosRootKey ); + strcpy_s(rootKey, 1024, icvWindowPosRootKey); rootKey[strlen(rootKey)-1] = '\0'; - if( RegCreateKeyEx(HKEY_CURRENT_USER, rootKey, 0, NULL, REG_OPTION_NON_VOLATILE, KEY_READ+KEY_WRITE, 0, &hroot, NULL) != ERROR_SUCCESS ) - //RegOpenKeyEx( HKEY_CURRENT_USER,rootKey,0,KEY_READ,&hroot) != ERROR_SUCCESS ) + if (RegCreateKeyEx(HKEY_CURRENT_USER, rootKey, 0, NULL, REG_OPTION_NON_VOLATILE, KEY_READ+KEY_WRITE, 0, &hroot, NULL) != ERROR_SUCCESS) + //RegOpenKeyEx(HKEY_CURRENT_USER,rootKey,0,KEY_READ,&hroot) != ERROR_SUCCESS) return; for(;;) { DWORD csize = sizeof(currentKey); FILETIME accesstime = { 0, 0 }; - LONG code = RegEnumKeyEx( hroot, count, currentKey, &csize, NULL, NULL, NULL, &accesstime ); - if( code != ERROR_SUCCESS && code != ERROR_MORE_DATA ) + LONG code = RegEnumKeyEx(hroot, count, currentKey, &csize, NULL, NULL, NULL, &accesstime); + if (code != ERROR_SUCCESS && code != ERROR_MORE_DATA) break; count++; - if( oldestTime.dwHighDateTime > accesstime.dwHighDateTime || + if (oldestTime.dwHighDateTime > accesstime.dwHighDateTime || (oldestTime.dwHighDateTime == accesstime.dwHighDateTime && - oldestTime.dwLowDateTime > accesstime.dwLowDateTime) ) + oldestTime.dwLowDateTime > accesstime.dwLowDateTime)) { oldestTime = accesstime; - strcpy_s( oldestKey, 1024, currentKey ); + strcpy_s(oldestKey, 1024, currentKey); } } - if( count >= MAX_RECORD_COUNT ) - RegDeleteKey( hroot, oldestKey ); - RegCloseKey( hroot ); + if (count >= MAX_RECORD_COUNT) + RegDeleteKey(hroot, oldestKey); + RegCloseKey(hroot); - if( RegCreateKeyEx(HKEY_CURRENT_USER,szKey,0,NULL,REG_OPTION_NON_VOLATILE, KEY_WRITE, 0, &hkey, NULL) != ERROR_SUCCESS ) + if (RegCreateKeyEx(HKEY_CURRENT_USER,szKey,0,NULL,REG_OPTION_NON_VOLATILE, KEY_WRITE, 0, &hkey, NULL) != ERROR_SUCCESS) return; } else { - RegCloseKey( hkey ); - if( RegOpenKeyEx( HKEY_CURRENT_USER,szKey,0,KEY_WRITE,&hkey) != ERROR_SUCCESS ) + RegCloseKey(hkey); + if (RegOpenKeyEx(HKEY_CURRENT_USER,szKey,0,KEY_WRITE,&hkey) != ERROR_SUCCESS) return; } @@ -440,96 +520,101 @@ icvSaveWindowPos( const char* name, CvRect rect ) RegCloseKey(hkey); } +static Rect getImageRect_(CvWindow& window); + CvRect cvGetWindowRect_W32(const char* name) { - RECT rect = { 0 }; - CvRect result = cvRect(-1, -1, -1, -1); + CV_FUNCNAME("cvGetWindowRect_W32"); - CV_FUNCNAME( "cvGetWindowRect_W32" ); - - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - window = icvFindWindowByName( name ); - if (!window) - EXIT; // keep silence here + CV_Error(Error::StsNullPtr, "NULL name string"); - GetClientRect(window->hwnd, &rect); - { + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); + + Rect r = getImageRect_(*window); + + CvRect result = cvRect(r.x, r.y, r.width, r.height); + return result; +} + +static Rect getImageRect_(CvWindow& window) +{ + RECT rect = { 0 }; + GetClientRect(window.hwnd, &rect); POINT pt = {rect.left, rect.top}; - ClientToScreen(window->hwnd, &pt); - result = cvRect(pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top); - } - __END__; + ClientToScreen(window.hwnd, &pt); + Rect result(pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top); return result; } double cvGetModeWindow_W32(const char* name)//YV { - double result = -1; + CV_FUNCNAME("cvGetModeWindow_W32"); - CV_FUNCNAME( "cvGetModeWindow_W32" ); - - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - result = window->status; - - __END__; - return result; + return window->status; } -void cvSetModeWindow_W32( const char* name, double prop_value)//Yannick Verdie +static bool setModeWindow_(CvWindow& window, int mode); + +void cvSetModeWindow_W32(const char* name, double prop_value)//Yannick Verdie { - CV_FUNCNAME( "cvSetModeWindow_W32" ); + CV_FUNCNAME("cvSetModeWindow_W32"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - window = icvFindWindowByName( name ); - if( !window ) - CV_ERROR( CV_StsNullPtr, "NULL window" ); + (void)setModeWindow_(*window, (int)prop_value); +} - if(window->flags & CV_WINDOW_AUTOSIZE)//if the flag CV_WINDOW_AUTOSIZE is set - EXIT; +static bool setModeWindow_(CvWindow& window, int mode) +{ + if (window.flags & CV_WINDOW_AUTOSIZE)//if the flag CV_WINDOW_AUTOSIZE is set + return false; + + if (window.status == mode) + return true; { - DWORD dwStyle = (DWORD)GetWindowLongPtr(window->frame, GWL_STYLE); + DWORD dwStyle = (DWORD)GetWindowLongPtr(window.frame, GWL_STYLE); CvRect position; - if (window->status==CV_WINDOW_FULLSCREEN && prop_value==CV_WINDOW_NORMAL) + if (window.status == CV_WINDOW_FULLSCREEN && mode == CV_WINDOW_NORMAL) { - icvLoadWindowPos(window->name,position ); - SetWindowLongPtr(window->frame, GWL_STYLE, dwStyle | WS_CAPTION | WS_THICKFRAME); + icvLoadWindowPos(window.name.c_str(), position); + SetWindowLongPtr(window.frame, GWL_STYLE, dwStyle | WS_CAPTION | WS_THICKFRAME); - SetWindowPos(window->frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); - window->status=CV_WINDOW_NORMAL; + SetWindowPos(window.frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); + window.status=CV_WINDOW_NORMAL; - EXIT; + return true; } - if (window->status==CV_WINDOW_NORMAL && prop_value==CV_WINDOW_FULLSCREEN) + if (window.status == CV_WINDOW_NORMAL && mode == CV_WINDOW_FULLSCREEN) { //save dimension RECT rect = { 0 }; - GetWindowRect(window->frame, &rect); - CvRect RectCV = cvRect(rect.left, rect.top,rect.right - rect.left, rect.bottom - rect.top); - icvSaveWindowPos(window->name,RectCV ); + GetWindowRect(window.frame, &rect); + CvRect rectCV = cvRect(rect.left, rect.top,rect.right - rect.left, rect.bottom - rect.top); + icvSaveWindowPos(window.name.c_str(), rectCV); //Look at coordinate for fullscreen HMONITOR hMonitor; @@ -542,60 +627,75 @@ void cvSetModeWindow_W32( const char* name, double prop_value)//Yannick Verdie //fullscreen position.x=mi.rcMonitor.left;position.y=mi.rcMonitor.top; position.width=mi.rcMonitor.right - mi.rcMonitor.left;position.height=mi.rcMonitor.bottom - mi.rcMonitor.top; - SetWindowLongPtr(window->frame, GWL_STYLE, dwStyle & ~WS_CAPTION & ~WS_THICKFRAME); + SetWindowLongPtr(window.frame, GWL_STYLE, dwStyle & ~WS_CAPTION & ~WS_THICKFRAME); - SetWindowPos(window->frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); - window->status=CV_WINDOW_FULLSCREEN; + SetWindowPos(window.frame, HWND_TOP, position.x, position.y , position.width,position.height, SWP_NOZORDER | SWP_FRAMECHANGED); + window.status=CV_WINDOW_FULLSCREEN; - EXIT; + return true; } } - __END__; + return false; } +static double getPropTopmost_(CvWindow& window); + double cvGetPropTopmost_W32(const char* name) { - double result = -1; - CV_Assert(name); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error(Error::StsNullPtr, "NULL window"); - LONG style = GetWindowLongA(window->frame, GWL_EXSTYLE); // -20 + return getPropTopmost_(*window); +} + +static double getPropTopmost_(CvWindow& window) +{ + LONG style = GetWindowLongA(window.frame, GWL_EXSTYLE); // -20 if (!style) { std::ostringstream errorMsg; - errorMsg << "window(" << name << "): failed to retrieve extended window style using GetWindowLongA(); error code: " << GetLastError(); - CV_Error(Error::StsError, errorMsg.str().c_str()); + errorMsg << "window(" << window.name << "): failed to retrieve extended window style using GetWindowLongA(); error code: " << GetLastError(); + CV_Error(Error::StsError, errorMsg.str()); } - result = (style & WS_EX_TOPMOST) == WS_EX_TOPMOST; - - return result; + bool result = (style & WS_EX_TOPMOST) == WS_EX_TOPMOST; + return result ? 1.0 : 0.0; } +static bool setPropTopmost_(CvWindow& window, bool topmost); + void cvSetPropTopmost_W32(const char* name, const bool topmost) { CV_Assert(name); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error(Error::StsNullPtr, "NULL window"); + (void)setPropTopmost_(*window, topmost); +} + +static bool setPropTopmost_(CvWindow& window, bool topmost) +{ HWND flag = topmost ? HWND_TOPMOST : HWND_TOP; - BOOL success = SetWindowPos(window->frame, flag, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); + BOOL success = SetWindowPos(window.frame, flag, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE); if (!success) { std::ostringstream errorMsg; - errorMsg << "window(" << name << "): error reported by SetWindowPos(" << (topmost ? "HWND_TOPMOST" : "HWND_TOP") << "), error code: " << GetLastError(); - CV_Error(Error::StsError, errorMsg.str().c_str()); + errorMsg << "window(" << window.name << "): error reported by SetWindowPos(" << (topmost ? "HWND_TOPMOST" : "HWND_TOP") << "), error code: " << GetLastError(); + CV_Error(Error::StsError, errorMsg.str()); + return false; } + return true; } +static double getPropVsync_(CvWindow& window); + double cvGetPropVsync_W32(const char* name) { #ifndef HAVE_OPENGL @@ -605,40 +705,53 @@ double cvGetPropVsync_W32(const char* name) if (!name) CV_Error(Error::StsNullPtr, "'name' argument must not be NULL"); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error_(Error::StsBadArg, ("there is no window named '%s'", name)); + double result = getPropVsync_(*window); + return cvIsNaN(result) ? -1.0 : result; +#endif +} + +static double getPropVsync_(CvWindow& window) +{ +#ifndef HAVE_OPENGL + CV_UNUSED(window); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); +#else // https://www.khronos.org/opengl/wiki/Swap_Interval // https://www.khronos.org/registry/OpenGL/extensions/EXT/WGL_EXT_extensions_string.txt // https://www.khronos.org/registry/OpenGL/extensions/EXT/WGL_EXT_swap_control.txt - if (!wglMakeCurrent(window->dc, window->hGLRC)) + if (!wglMakeCurrent(window.dc, window.hGLRC)) CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); typedef const char* (APIENTRY* PFNWGLGETEXTENSIONSSTRINGEXTPROC)(void); PFNWGLGETEXTENSIONSSTRINGEXTPROC wglGetExtensionsString = NULL; wglGetExtensionsString = (PFNWGLGETEXTENSIONSSTRINGEXTPROC)wglGetProcAddress("wglGetExtensionsStringEXT"); if (wglGetExtensionsString == NULL) - return -1; // wglGetProcAddress failed to get wglGetExtensionsStringEXT + return std::numeric_limits::quiet_NaN(); // wglGetProcAddress failed to get wglGetExtensionsStringEXT const char* wgl_extensions = wglGetExtensionsString(); if (wgl_extensions == NULL) - return -1; // Can't get WGL extensions string + return std::numeric_limits::quiet_NaN(); // Can't get WGL extensions string if (strstr(wgl_extensions, "WGL_EXT_swap_control") == NULL) - return -1; // WGL extensions don't contain WGL_EXT_swap_control + return std::numeric_limits::quiet_NaN(); // WGL extensions don't contain WGL_EXT_swap_control typedef int (APIENTRY* PFNWGLGETSWAPINTERVALPROC)(void); PFNWGLGETSWAPINTERVALPROC wglGetSwapInterval = 0; wglGetSwapInterval = (PFNWGLGETSWAPINTERVALPROC)wglGetProcAddress("wglGetSwapIntervalEXT"); if (wglGetSwapInterval == NULL) - return -1; // wglGetProcAddress failed to get wglGetSwapIntervalEXT + return std::numeric_limits::quiet_NaN(); // wglGetProcAddress failed to get wglGetSwapIntervalEXT return wglGetSwapInterval(); #endif } +static bool setPropVsync_(CvWindow& window, bool enable_vsync); + void cvSetPropVsync_W32(const char* name, const bool enable_vsync) { #ifndef HAVE_OPENGL @@ -649,11 +762,22 @@ void cvSetPropVsync_W32(const char* name, const bool enable_vsync) if (!name) CV_Error(Error::StsNullPtr, "'name' argument must not be NULL"); - CvWindow* window = icvFindWindowByName(name); + auto window = icvFindWindowByName(name); if (!window) CV_Error_(Error::StsBadArg, ("there is no window named '%s'", name)); - if (!wglMakeCurrent(window->dc, window->hGLRC)) + (void)setPropVsync_(*window, enable_vsync); +#endif +} + +static bool setPropVsync_(CvWindow& window, bool enable_vsync) +{ +#ifndef HAVE_OPENGL + CV_UNUSED(window); + CV_UNUSED(enable_vsync); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); +#else + if (!wglMakeCurrent(window.dc, window.hGLRC)) CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); typedef const char* (APIENTRY* PFNWGLGETEXTENSIONSSTRINGEXTPROC)(void); @@ -676,47 +800,44 @@ void cvSetPropVsync_W32(const char* name, const bool enable_vsync) CV_Error(Error::OpenGlApiCallError, "wglGetProcAddress failed to get wglSwapIntervalEXT"); wglSwapInterval(enable_vsync); + return true; #endif } -void cv::setWindowTitle(const String& winname, const String& title) +void setWindowTitle_W32(const std::string& name, const std::string& title) { - CvWindow* window = icvFindWindowByName(winname.c_str()); + auto window = icvFindWindowByName(name); if (!window) { - namedWindow(winname); - window = icvFindWindowByName(winname.c_str()); + namedWindow(name); + window = icvFindWindowByName(name); } if (!window) CV_Error(Error::StsNullPtr, "NULL window"); if (!SetWindowText(window->frame, title.c_str())) - CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", winname.c_str(), title.c_str())); + CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", name.c_str(), title.c_str())); } double cvGetPropWindowAutoSize_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvSetCloseCallback" ); + CV_FUNCNAME("cvSetCloseCallback"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); result = window->flags & CV_WINDOW_AUTOSIZE; - __END__; - return result; } @@ -724,23 +845,19 @@ double cvGetRatioWindow_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvGetRatioWindow_W32" ); + CV_FUNCNAME("cvGetRatioWindow_W32"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); result = static_cast(window->width) / window->height; - __END__; - return result; } @@ -749,23 +866,20 @@ double cvGetOpenGlProp_W32(const char* name) double result = -1; #ifdef HAVE_OPENGL - CV_FUNCNAME( "cvGetOpenGlProp_W32" ); + CV_FUNCNAME("cvGetOpenGlProp_W32"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; // keep silence here + return -1; result = window->useGl; - - __END__; #endif + CV_UNUSED(name); return result; @@ -775,16 +889,15 @@ double cvGetPropVisible_W32(const char* name) { double result = -1; - CV_FUNCNAME( "cvGetPropVisible_W32" ); + CV_FUNCNAME("cvGetPropVisible_W32"); - __BEGIN__; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - result = (icvFindWindowByName( name ) != NULL); - - __END__; + auto window = icvFindWindowByName(name); + result = (bool)window ? 1.0 : 0.0; return result; } @@ -798,9 +911,9 @@ namespace { void createGlContext(HWND hWnd, HDC& hGLDC, HGLRC& hGLRC, bool& useGl) { - CV_FUNCNAME( "createGlContext" ); + CV_FUNCNAME("createGlContext"); - __BEGIN__; + AutoLock lock(getWindowMutex()); useGl = false; @@ -830,120 +943,119 @@ namespace hGLDC = GetDC(hWnd); if (!hGLDC) - CV_ERROR( CV_OpenGlApiCallError, "Can't Create A GL Device Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Create A GL Device Context"); PixelFormat = ChoosePixelFormat(hGLDC, &pfd); if (!PixelFormat) - CV_ERROR( CV_OpenGlApiCallError, "Can't Find A Suitable PixelFormat" ); + CV_Error(Error::OpenGlApiCallError, "Can't Find A Suitable PixelFormat"); if (!SetPixelFormat(hGLDC, PixelFormat, &pfd)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Set The PixelFormat" ); + CV_Error(Error::OpenGlApiCallError, "Can't Set The PixelFormat"); hGLRC = wglCreateContext(hGLDC); if (!hGLRC) - CV_ERROR( CV_OpenGlApiCallError, "Can't Create A GL Rendering Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Create A GL Rendering Context"); if (!wglMakeCurrent(hGLDC, hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); useGl = true; - - __END__; } - void releaseGlContext(CvWindow* window) + void releaseGlContext(CvWindow& window) { - //CV_FUNCNAME( "releaseGlContext" ); + //CV_FUNCNAME("releaseGlContext"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - if (window->hGLRC) + if (window.hGLRC) { - wglDeleteContext(window->hGLRC); - window->hGLRC = NULL; + wglDeleteContext(window.hGLRC); + window.hGLRC = NULL; } - if (window->dc) + if (window.dc) { - ReleaseDC(window->hwnd, window->dc); - window->dc = NULL; + ReleaseDC(window.hwnd, window.dc); + window.dc = NULL; } - window->useGl = false; - - __END__; + window.useGl = false; } - void drawGl(CvWindow* window) + void drawGl(CvWindow& window) { - CV_FUNCNAME( "drawGl" ); + CV_FUNCNAME("drawGl"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + if (!wglMakeCurrent(window.dc, window.hGLRC)) + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); - if (window->glDrawCallback) - window->glDrawCallback(window->glDrawData); + if (window.glDrawCallback) + window.glDrawCallback(window.glDrawData); - if (!SwapBuffers(window->dc)) - CV_ERROR( CV_OpenGlApiCallError, "Can't swap OpenGL buffers" ); - - __END__; + if (!SwapBuffers(window.dc)) + CV_Error(Error::OpenGlApiCallError, "Can't swap OpenGL buffers"); } - void resizeGl(CvWindow* window) + void resizeGl(CvWindow& window) { - CV_FUNCNAME( "resizeGl" ); + CV_FUNCNAME("resizeGl"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); + if (!wglMakeCurrent(window.dc, window.hGLRC)) + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); - glViewport(0, 0, window->width, window->height); - - __END__; + glViewport(0, 0, window.width, window.height); } } #endif // HAVE_OPENGL +static std::shared_ptr namedWindow_(const std::string& name, int flags); -CV_IMPL int cvNamedWindow( const char* name, int flags ) +CV_IMPL int cvNamedWindow(const char* name, int flags) { - int result = 0; - CV_FUNCNAME( "cvNamedWindow" ); + CV_FUNCNAME("cvNamedWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); + + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); + + // Check the name in the storage + auto window = icvFindWindowByName(name); + if (window) + { + return 1; + } + + window = namedWindow_(name, flags); + return (bool)window; +} + +static std::shared_ptr namedWindow_(const std::string& name, int flags) +{ + AutoLock lock(getWindowMutex()); + + cvInitSystem(0,0); HWND hWnd, mainhWnd; - CvWindow* window; DWORD defStyle = WS_VISIBLE | WS_MINIMIZEBOX | WS_MAXIMIZEBOX | WS_SYSMENU; - int len; - CvRect rect; #ifdef HAVE_OPENGL bool useGl; HDC hGLDC; HGLRC hGLRC; #endif - cvInitSystem(0,0); + CvRect rect; + icvLoadWindowPos(name.c_str(), rect); - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - - // Check the name in the storage - window = icvFindWindowByName( name ); - if (window != 0) - { - result = 1; - EXIT; - } - - if( !(flags & CV_WINDOW_AUTOSIZE))//YV add border in order to resize the window + if (!(flags & CV_WINDOW_AUTOSIZE))//YV add border in order to resize the window defStyle |= WS_SIZEBOX; #ifdef HAVE_OPENGL @@ -951,23 +1063,21 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) defStyle |= WS_CLIPCHILDREN | WS_CLIPSIBLINGS; #endif - icvLoadWindowPos( name, rect ); - - mainhWnd = CreateWindow( "Main HighGUI class", name, defStyle | WS_OVERLAPPED, - rect.x, rect.y, rect.width, rect.height, 0, 0, hg_hinstance, 0 ); - if( !mainhWnd ) - CV_ERROR( CV_StsError, "Frame window can not be created" ); + mainhWnd = CreateWindow(mainHighGUIclassName, name.c_str(), defStyle | WS_OVERLAPPED, + rect.x, rect.y, rect.width, rect.height, 0, 0, hg_hinstance, 0); + if (!mainhWnd) + CV_Error_(Error::StsError, ("Frame window can not be created: '%s'", name.c_str())); ShowWindow(mainhWnd, SW_SHOW); //YV- remove one border by changing the style - hWnd = CreateWindow("HighGUI class", "", (defStyle & ~WS_SIZEBOX) | WS_CHILD, CW_USEDEFAULT, 0, rect.width, rect.height, mainhWnd, 0, hg_hinstance, 0); - if( !hWnd ) - CV_ERROR( CV_StsError, "Frame window can not be created" ); + hWnd = CreateWindow(highGUIclassName, "", (defStyle & ~WS_SIZEBOX) | WS_CHILD, CW_USEDEFAULT, 0, rect.width, rect.height, mainhWnd, 0, hg_hinstance, 0); + if (!hWnd) + CV_Error(Error::StsError, "Frame window can not be created"); #ifndef HAVE_OPENGL if (flags & CV_WINDOW_OPENGL) - CV_ERROR( CV_OpenGlNotSupported, "Library was built without OpenGL support" ); + CV_Error(Error::OpenGlNotSupported, "Library was built without OpenGL support"); #else useGl = false; hGLDC = 0; @@ -979,14 +1089,10 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) ShowWindow(hWnd, SW_SHOW); - len = (int)strlen(name); - CV_CALL( window = (CvWindow*)cvAlloc(sizeof(CvWindow) + len + 1)); + auto window = std::make_shared(name); - window->signature = CV_WINDOW_MAGIC_VAL; window->hwnd = hWnd; window->frame = mainhWnd; - window->name = (char*)(window + 1); - memcpy( window->name, name, len + 1 ); window->flags = flags; window->image = 0; @@ -1016,200 +1122,175 @@ CV_IMPL int cvNamedWindow( const char* name, int flags ) window->on_mouse = 0; window->on_mouse_param = 0; - memset( &window->toolbar, 0, sizeof(window->toolbar)); + icvSetWindowLongPtr(hWnd, CV_USERDATA, window.get()); + icvSetWindowLongPtr(mainhWnd, CV_USERDATA, window.get()); - window->next = hg_windows; - window->prev = 0; - if( hg_windows ) - hg_windows->prev = window; - hg_windows = window; - icvSetWindowLongPtr( hWnd, CV_USERDATA, window ); - icvSetWindowLongPtr( mainhWnd, CV_USERDATA, window ); + auto& g_windows = getWindowsList(); + g_windows.push_back(window); // Recalculate window pos - icvUpdateWindowPos( window ); + icvUpdateWindowPos(*window); - result = 1; - __END__; - - return result; + return window; } #ifdef HAVE_OPENGL CV_IMPL void cvSetOpenGlContext(const char* name) { - CV_FUNCNAME( "cvSetOpenGlContext" ); + CV_FUNCNAME("cvSetOpenGlContext"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - CV_ERROR( CV_StsNullPtr, "NULL window" ); + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); if (!window->useGl) - CV_ERROR( CV_OpenGlNotSupported, "Window doesn't support OpenGL" ); + CV_Error(Error::OpenGlNotSupported, "Window doesn't support OpenGL"); if (!wglMakeCurrent(window->dc, window->hGLRC)) - CV_ERROR( CV_OpenGlApiCallError, "Can't Activate The GL Rendering Context" ); - - __END__; + CV_Error(Error::OpenGlApiCallError, "Can't Activate The GL Rendering Context"); } CV_IMPL void cvUpdateWindow(const char* name) { - CV_FUNCNAME( "cvUpdateWindow" ); + CV_FUNCNAME("cvUpdateWindow"); - __BEGIN__; - - CvWindow* window; + AutoLock lock(getWindowMutex()); if (!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + CV_Error(Error::StsNullPtr, "NULL name string"); - window = icvFindWindowByName( name ); + auto window = icvFindWindowByName(name); if (!window) - EXIT; + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); InvalidateRect(window->hwnd, 0, 0); - - __END__; } CV_IMPL void cvSetOpenGlDrawCallback(const char* name, CvOpenGlDrawCallback callback, void* userdata) { - CV_FUNCNAME( "cvCreateOpenGLCallback" ); + CV_FUNCNAME("cvCreateOpenGLCallback"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); - - window = icvFindWindowByName( name ); - if( !window ) - EXIT; + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); if (!window->useGl) - CV_ERROR( CV_OpenGlNotSupported, "Window was created without OpenGL context" ); + CV_Error(Error::OpenGlNotSupported, "Window was created without OpenGL context"); window->glDrawCallback = callback; window->glDrawData = userdata; - - __END__; } #endif // HAVE_OPENGL -static void icvRemoveWindow( CvWindow* window ) +static void icvRemoveWindow(const std::shared_ptr& window_) { - CvTrackbar* trackbar = NULL; + CV_Assert(window_); + AutoLock lock(getWindowMutex()); + CvWindow& window = *window_; + RECT wrect={0,0,0,0}; + auto& g_windows = getWindowsList(); + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + const std::shared_ptr& w = *it; + if (w.get() == &window) + { + g_windows.erase(it); + break; + } + } + #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) releaseGlContext(window); #endif - if( window->frame ) - GetWindowRect( window->frame, &wrect ); - if( window->name ) - icvSaveWindowPos( window->name, cvRect(wrect.left, wrect.top, - wrect.right-wrect.left, wrect.bottom-wrect.top) ); + if (window.frame) + GetWindowRect(window.frame, &wrect); + icvSaveWindowPos(window.name.c_str(), cvRect(wrect.left, wrect.top, wrect.right-wrect.left, wrect.bottom-wrect.top)); - if( window->hwnd ) - icvSetWindowLongPtr( window->hwnd, CV_USERDATA, 0 ); - if( window->frame ) - icvSetWindowLongPtr( window->frame, CV_USERDATA, 0 ); + if (window.hwnd) + icvSetWindowLongPtr(window.hwnd, CV_USERDATA, 0); + if (window.frame) + icvSetWindowLongPtr(window.frame, CV_USERDATA, 0); - if( window->toolbar.toolbar ) - icvSetWindowLongPtr(window->toolbar.toolbar, CV_USERDATA, 0); + if (window.toolbar.toolbar) + icvSetWindowLongPtr(window.toolbar.toolbar, CV_USERDATA, 0); - if( window->prev ) - window->prev->next = window->next; - else - hg_windows = window->next; + if (window.dc && window.image) + DeleteObject(SelectObject(window.dc, window.image)); - if( window->next ) - window->next->prev = window->prev; + if (window.dc) + DeleteDC(window.dc); - window->prev = window->next = 0; - - if( window->dc && window->image ) - DeleteObject(SelectObject(window->dc,window->image)); - - if( window->dc ) - DeleteDC(window->dc); - - for( trackbar = window->toolbar.first; trackbar != 0; ) + for (auto it = window.toolbar.trackbars.begin(); it != window.toolbar.trackbars.end(); ++it) { - CvTrackbar* next = trackbar->next; - if( trackbar->hwnd ) + auto trackbar = (*it).get(); + if (trackbar && trackbar->hwnd) { - icvSetWindowLongPtr( trackbar->hwnd, CV_USERDATA, 0 ); - cvFree( &trackbar ); + icvSetWindowLongPtr(trackbar->hwnd, CV_USERDATA, 0); } - trackbar = next; } - - cvFree( &window ); } -CV_IMPL void cvDestroyWindow( const char* name ) +CV_IMPL void cvDestroyWindow(const char* name) { - CV_FUNCNAME( "cvDestroyWindow" ); + CV_FUNCNAME("cvDestroyWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - HWND mainhWnd; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name string"); - if(!name) - CV_ERROR( CV_StsNullPtr, "NULL name string" ); + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); - window = icvFindWindowByName( name ); - if( !window ) - EXIT; - - mainhWnd = window->frame; - - SendMessage(window->hwnd, WM_CLOSE, 0, 0); - SendMessage( mainhWnd, WM_CLOSE, 0, 0); - // Do NOT call _remove_window -- CvWindow list will be updated automatically ... - - __END__; + window->destroy(); } -static void icvScreenToClient( HWND hwnd, RECT* rect ) +void CvWindow::destroy() +{ + SendMessage(hwnd, WM_CLOSE, 0, 0); + SendMessage(frame, WM_CLOSE, 0, 0); + // Do NOT call _remove_window -- CvWindow list will be updated automatically ... +} + +static void icvScreenToClient(HWND hwnd, RECT* rect) { POINT p; p.x = rect->left; p.y = rect->top; ScreenToClient(hwnd, &p); - OffsetRect( rect, p.x - rect->left, p.y - rect->top ); + OffsetRect(rect, p.x - rect->left, p.y - rect->top); } /* Calculatess the window coordinates relative to the upper left corner of the mainhWnd window */ -static RECT icvCalcWindowRect( CvWindow* window ) +static RECT icvCalcWindowRect(CvWindow& window) { RECT crect = { 0 }, trect = { 0 }, rect = { 0 }; - assert(window); - - GetClientRect(window->frame, &crect); - if (window->toolbar.toolbar) + GetClientRect(window.frame, &crect); + if (window.toolbar.toolbar) { - GetWindowRect(window->toolbar.toolbar, &trect); - icvScreenToClient(window->frame, &trect); + GetWindowRect(window.toolbar.toolbar, &trect); + icvScreenToClient(window.frame, &trect); SubtractRect(&rect, &crect, &trect); } else @@ -1217,138 +1298,153 @@ static RECT icvCalcWindowRect( CvWindow* window ) return rect; } +static inline RECT icvCalcWindowRect(CvWindow* window) { CV_Assert(window); return icvCalcWindowRect(*window); } -// returns TRUE if there is a problem such as ERROR_IO_PENDING. -static bool icvGetBitmapData( CvWindow* window, SIZE* size, int* channels, void** data ) + +// returns FALSE if there is a problem such as ERROR_IO_PENDING. +static bool icvGetBitmapData(CvWindow& window, SIZE& size, int& channels, void*& data) { - BITMAP bmp; GdiFlush(); - HGDIOBJ h = GetCurrentObject( window->dc, OBJ_BITMAP ); - if( size ) - size->cx = size->cy = 0; - if( data ) - *data = 0; + + HGDIOBJ h = GetCurrentObject(window.dc, OBJ_BITMAP); + size.cx = size.cy = 0; + data = 0; if (h == NULL) - return true; + return false; + + BITMAP bmp = {}; if (GetObject(h, sizeof(bmp), &bmp) == 0) - return true; + return false; - if( size ) - { - size->cx = abs(bmp.bmWidth); - size->cy = abs(bmp.bmHeight); - } + size.cx = abs(bmp.bmWidth); + size.cy = abs(bmp.bmHeight); - if( channels ) - *channels = bmp.bmBitsPixel/8; + channels = bmp.bmBitsPixel/8; - if( data ) - *data = bmp.bmBits; + data = bmp.bmBits; - return false; + return true; +} +static bool icvGetBitmapData(CvWindow& window, SIZE& size) +{ + int channels = 0; + void* data = nullptr; + return icvGetBitmapData(window, size, channels, data); } -static void icvUpdateWindowPos( CvWindow* window ) +static void icvUpdateWindowPos(CvWindow& window) { RECT rect = { 0 }; - assert(window); - if( (window->flags & CV_WINDOW_AUTOSIZE) && window->image ) + if ((window.flags & CV_WINDOW_AUTOSIZE) && window.image) { int i; SIZE size = {0,0}; - icvGetBitmapData( window, &size, 0, 0 ); + icvGetBitmapData(window, size); // TODO check return value? // Repeat two times because after the first resizing of the mainhWnd window // toolbar may resize too - for(i = 0; i < (window->toolbar.toolbar ? 2 : 1); i++) + for(i = 0; i < (window.toolbar.toolbar ? 2 : 1); i++) { - RECT rmw = { 0 }, rw = icvCalcWindowRect(window ); - MoveWindow(window->hwnd, rw.left, rw.top, + RECT rmw = { 0 }, rw = icvCalcWindowRect(&window); + MoveWindow(window.hwnd, rw.left, rw.top, rw.right - rw.left, rw.bottom - rw.top, FALSE); - GetClientRect(window->hwnd, &rw); - GetWindowRect(window->frame, &rmw); + GetClientRect(window.hwnd, &rw); + GetWindowRect(window.frame, &rmw); // Resize the mainhWnd window in order to make the bitmap fit into the child window - MoveWindow(window->frame, rmw.left, rmw.top, + MoveWindow(window.frame, rmw.left, rmw.top, size.cx + (rmw.right - rmw.left) - (rw.right - rw.left), - size.cy + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE ); + size.cy + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); } } rect = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rect.left, rect.top, + MoveWindow(window.hwnd, rect.left, rect.top, rect.right - rect.left, - rect.bottom - rect.top, TRUE ); + rect.bottom - rect.top, TRUE); } +static void showImage_(CvWindow& window, const Mat& image); + CV_IMPL void -cvShowImage( const char* name, const CvArr* arr ) +cvShowImage(const char* name, const CvArr* arr) { - CV_FUNCNAME( "cvShowImage" ); + CV_FUNCNAME("cvShowImage"); - __BEGIN__; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); + + std::shared_ptr window; + { + AutoLock lock(getWindowMutex()); + + window = icvFindWindowByName(name); + if (!window) + { + cvNamedWindow(name, CV_WINDOW_AUTOSIZE); + window = icvFindWindowByName(name); + } + } + + if (!window || !arr) + return; // keep silence here. + + CvMat stub = {}; + CvMat* image_c = cvGetMat(arr, &stub); + Mat image = cv::cvarrToMat(image_c); +#ifdef HAVE_OPENGL + if (window->useGl) + { + cv::imshow(name, image); + return; + } +#endif + return showImage_(*window, image); +} + +static void showImage_(CvWindow& window, const Mat& image) +{ + AutoLock lock(window.mutex); - CvWindow* window; SIZE size = { 0, 0 }; int channels = 0; void* dst_ptr = 0; const int channels0 = 3; - CvMat stub, *image; bool changed_size = false; // philipg - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); - - window = icvFindWindowByName(name); - if(!window) + if (window.image) { - cvNamedWindow(name, CV_WINDOW_AUTOSIZE); - window = icvFindWindowByName(name); - } - - if( !window || !arr ) - EXIT; // keep silence here. - - CV_CALL( image = cvGetMat( arr, &stub )); - -#ifdef HAVE_OPENGL - if (window->useGl) - { - cv::imshow(name, cv::cvarrToMat(image)); - return; - } -#endif - - if (window->image) // if there is something wrong with these system calls, we cannot display image... - if (icvGetBitmapData( window, &size, &channels, &dst_ptr )) + if (!icvGetBitmapData(window, size, channels, dst_ptr)) return; + } - if( size.cx != image->width || size.cy != image->height || channels != channels0 ) + if (size.cx != image.cols || size.cy != image.rows || channels != channels0) { changed_size = true; uchar buffer[sizeof(BITMAPINFO) + 255*sizeof(RGBQUAD)]; BITMAPINFO* binfo = (BITMAPINFO*)buffer; - DeleteObject( SelectObject( window->dc, window->image )); - window->image = 0; + DeleteObject(SelectObject(window.dc, window.image)); + window.image = 0; - size.cx = image->width; - size.cy = image->height; + size.cx = image.cols; + size.cy = image.rows; channels = channels0; - FillBitmapInfo( binfo, size.cx, size.cy, channels*8, 1 ); + FillBitmapInfo(binfo, size.cx, size.cy, channels*8, 1); - window->image = SelectObject( window->dc, CreateDIBSection(window->dc, binfo, - DIB_RGB_COLORS, &dst_ptr, 0, 0)); + window.image = SelectObject(window.dc, + CreateDIBSection(window.dc, binfo, DIB_RGB_COLORS, &dst_ptr, 0, 0) + ); } { cv::Mat dst(size.cy, size.cx, CV_8UC3, dst_ptr, (size.cx * channels + 3) & -4); - convertToShow(cv::cvarrToMat(image), dst, false); + convertToShow(image, dst, false); CV_Assert(dst.data == (uchar*)dst_ptr); cv::flip(dst, dst, 0); } @@ -1356,98 +1452,103 @@ cvShowImage( const char* name, const CvArr* arr ) // only resize window if needed if (changed_size) icvUpdateWindowPos(window); - InvalidateRect(window->hwnd, 0, 0); + InvalidateRect(window.hwnd, 0, 0); // philipg: this is not needed and just slows things down // UpdateWindow(window->hwnd); - - __END__; } -CV_IMPL void cvResizeWindow(const char* name, int width, int height ) +static void resizeWindow_(CvWindow& window, const Size& size); + +CV_IMPL void cvResizeWindow(const char* name, int width, int height) { - CV_FUNCNAME( "cvResizeWindow" ); + CV_FUNCNAME("cvResizeWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - int i; - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); + + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); + + return resizeWindow_(*window, Size(width, height)); +} + +static void resizeWindow_(CvWindow& window, const Size& size) +{ RECT rmw = { 0 }, rw = { 0 }, rect = { 0 }; - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); - - window = icvFindWindowByName(name); - if(!window) - EXIT; - // Repeat two times because after the first resizing of the mainhWnd window // toolbar may resize too - for(i = 0; i < (window->toolbar.toolbar ? 2 : 1); i++) + for (int i = 0; i < (window.toolbar.toolbar ? 2 : 1); i++) { rw = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rw.left, rw.top, + MoveWindow(window.hwnd, rw.left, rw.top, rw.right - rw.left, rw.bottom - rw.top, FALSE); - GetClientRect(window->hwnd, &rw); - GetWindowRect(window->frame, &rmw); + GetClientRect(window.hwnd, &rw); + GetWindowRect(window.frame, &rmw); // Resize the mainhWnd window in order to make the bitmap fit into the child window - MoveWindow(window->frame, rmw.left, rmw.top, - width + (rmw.right - rmw.left) - (rw.right - rw.left), - height + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); + MoveWindow(window.frame, rmw.left, rmw.top, + size.width + (rmw.right - rmw.left) - (rw.right - rw.left), + size.height + (rmw.bottom - rmw.top) - (rw.bottom - rw.top), TRUE); } rect = icvCalcWindowRect(window); - MoveWindow(window->hwnd, rect.left, rect.top, + MoveWindow(window.hwnd, rect.left, rect.top, rect.right - rect.left, rect.bottom - rect.top, TRUE); - - __END__; } +static void moveWindow_(CvWindow& window, const Point& pt); -CV_IMPL void cvMoveWindow( const char* name, int x, int y ) +CV_IMPL void cvMoveWindow(const char* name, int x, int y) { - CV_FUNCNAME( "cvMoveWindow" ); + CV_FUNCNAME("cvMoveWindow"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; + if (!name) + CV_Error(Error::StsNullPtr, "NULL name"); + + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); + + (void)moveWindow_(*window, Point(x, y)); +} + +static void moveWindow_(CvWindow& window, const Point& pt) +{ RECT rect = { 0 }; - - if( !name ) - CV_ERROR( CV_StsNullPtr, "NULL name" ); - - window = icvFindWindowByName(name); - if(!window) - EXIT; - - GetWindowRect( window->frame, &rect ); - MoveWindow( window->frame, x, y, rect.right - rect.left, rect.bottom - rect.top, TRUE); - - __END__; + GetWindowRect(window.frame, &rect); // TODO check return value + MoveWindow(window.frame, pt.x, pt.y, rect.right - rect.left, rect.bottom - rect.top, TRUE); } static LRESULT CALLBACK -MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +MainWindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND( hwnd ); - if( !window ) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) return DefWindowProc(hwnd, uMsg, wParam, lParam); + CvWindow& window = *window_; + switch(uMsg) { case WM_COPY: - ::SendMessage(window->hwnd, uMsg, wParam, lParam); + ::SendMessage(window.hwnd, uMsg, wParam, lParam); break; case WM_DESTROY: - icvRemoveWindow(window); + icvRemoveWindow(window_); // Do nothing!!! //PostQuitMessage(0); break; case WM_GETMINMAXINFO: - if( !(window->flags & CV_WINDOW_AUTOSIZE) ) + if (!(window.flags & CV_WINDOW_AUTOSIZE)) { MINMAXINFO* minmax = (MINMAXINFO*)lParam; RECT rect = { 0 }; @@ -1456,10 +1557,10 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) minmax->ptMinTrackSize.y = 100; minmax->ptMinTrackSize.x = 100; - if( window->toolbar.first ) + if (!window.toolbar.trackbars.empty()) { - GetWindowRect( window->toolbar.first->hwnd, &rect ); - minmax->ptMinTrackSize.y += window->toolbar.rows*(rect.bottom - rect.top); + GetWindowRect(window.toolbar.trackbars[0]->hwnd, &rect); + minmax->ptMinTrackSize.y += window.toolbar.rows*(rect.bottom - rect.top); minmax->ptMinTrackSize.x = MAX(rect.right - rect.left + HG_BUDDY_WIDTH, HG_BUDDY_WIDTH*2); } return retval; @@ -1471,14 +1572,14 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) WINDOWPOS* pos = (WINDOWPOS*)lParam; // Update the toolbar pos/size - if(window->toolbar.toolbar) + if (window.toolbar.toolbar) { RECT rect = { 0 }; - GetWindowRect(window->toolbar.toolbar, &rect); - MoveWindow(window->toolbar.toolbar, 0, 0, pos->cx, rect.bottom - rect.top, TRUE); + GetWindowRect(window.toolbar.toolbar, &rect); + MoveWindow(window.toolbar.toolbar, 0, 0, pos->cx, rect.bottom - rect.top, TRUE); } - if(!(window->flags & CV_WINDOW_AUTOSIZE)) + if (!(window.flags & CV_WINDOW_AUTOSIZE)) icvUpdateWindowPos(window); break; @@ -1490,7 +1591,7 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) LPWINDOWPOS pos = (LPWINDOWPOS)lParam; RECT rect = { 0 }; - GetWindowRect(window->frame, &rect); + GetWindowRect(window.frame, &rect); HMONITOR hMonitor; hMonitor = MonitorFromRect(&rect, MONITOR_DEFAULTTONEAREST); @@ -1515,13 +1616,13 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) } case WM_ACTIVATE: - if(LOWORD(wParam) == WA_ACTIVE || LOWORD(wParam) == WA_CLICKACTIVE) - SetFocus(window->hwnd); + if (LOWORD(wParam) == WA_ACTIVE || LOWORD(wParam) == WA_CLICKACTIVE) + SetFocus(window.hwnd); break; case WM_MOUSEWHEEL: case WM_MOUSEHWHEEL: - if( window->on_mouse ) + if (window.on_mouse) { int flags = (wParam & MK_LBUTTON ? CV_EVENT_FLAG_LBUTTON : 0)| (wParam & MK_RBUTTON ? CV_EVENT_FLAG_RBUTTON : 0)| @@ -1536,32 +1637,32 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) flags |= (delta << 16); POINT pt; - pt.x = GET_X_LPARAM( lParam ); - pt.y = GET_Y_LPARAM( lParam ); + pt.x = GET_X_LPARAM(lParam); + pt.y = GET_Y_LPARAM(lParam); ::ScreenToClient(hwnd, &pt); // Convert screen coordinates to client coordinates. RECT rect = { 0 }; - GetClientRect( window->hwnd, &rect ); + GetClientRect(window.hwnd, &rect); SIZE size = {0,0}; #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) { - cv::ogl::Texture2D* texObj = static_cast(window->glDrawData); + cv::ogl::Texture2D* texObj = static_cast(window.glDrawData); size.cx = texObj->cols(); size.cy = texObj->rows(); } else { - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); } #else - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); #endif - window->on_mouse( event, pt.x*size.cx/MAX(rect.right - rect.left,1), - pt.y*size.cy/MAX(rect.bottom - rect.top,1), flags, - window->on_mouse_param ); + int x = cvRound((float)pt.x*size.cx/MAX(rect.right - rect.left,1)); + int y = cvRound((float)pt.y*size.cy/MAX(rect.bottom - rect.top,1)); + window.on_mouse(event, x, y, flags, window.on_mouse_param); } break; @@ -1571,17 +1672,17 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) HRGN rgn, rgn1, rgn2; int ret; HDC hdc = (HDC)wParam; - GetWindowRect(window->hwnd, &cr); - icvScreenToClient(window->frame, &cr); - if(window->toolbar.toolbar) + GetWindowRect(window.hwnd, &cr); + icvScreenToClient(window.frame, &cr); + if (window.toolbar.toolbar) { - GetWindowRect(window->toolbar.toolbar, &tr); - icvScreenToClient(window->frame, &tr); + GetWindowRect(window.toolbar.toolbar, &tr); + icvScreenToClient(window.frame, &tr); } else tr.left = tr.top = tr.right = tr.bottom = 0; - GetClientRect(window->frame, &wrc); + GetClientRect(window.frame, &wrc); rgn = CreateRectRgn(0, 0, wrc.right, wrc.bottom); rgn1 = CreateRectRgn(cr.left, cr.top, cr.right, cr.bottom); @@ -1591,7 +1692,7 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) ret = CombineRgn(rgn, rgn, rgn1, RGN_DIFF); ret = CombineRgn(rgn, rgn, rgn2, RGN_DIFF); - if(ret != NULLREGION && ret != ERROR) + if (ret != NULLREGION && ret != ERROR) FillRgn(hdc, rgn, (HBRUSH)icvGetClassLongPtr(hwnd, CV_HBRBACKGROUND)); DeleteObject(rgn); @@ -1605,20 +1706,24 @@ MainWindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) } -static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK HighGUIProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND(hwnd); - if( !window ) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) + { // This window is not mentioned in HighGUI storage // Actually, this should be error except for the case of calls to CreateWindow return DefWindowProc(hwnd, uMsg, wParam, lParam); + } + + CvWindow& window = *window_; // Process the message switch(uMsg) { case WM_COPY: { - if (!::OpenClipboard(hwnd) ) + if (!::OpenClipboard(hwnd)) break; HDC hDC = 0; @@ -1632,7 +1737,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM if (!::EmptyClipboard()) break; - if(!window->image) + if (!window.image) break; // Get window device context @@ -1640,19 +1745,20 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM break; // Create another DC compatible with hDC - if (0 == (memDC = ::CreateCompatibleDC( hDC ))) + if (0 == (memDC = ::CreateCompatibleDC(hDC))) break; // Determine the bitmap's dimensions - int nchannels = 3; SIZE size = {0,0}; - icvGetBitmapData( window, &size, &nchannels, 0 ); + int nchannels = 3; + void* data = NULL; // unused + icvGetBitmapData(window, size, nchannels, data); // Create bitmap to draw on and it in the new DC - if (0 == (memBM = ::CreateCompatibleBitmap ( hDC, size.cx, size.cy))) + if (0 == (memBM = ::CreateCompatibleBitmap(hDC, size.cx, size.cy))) break; - if (!::SelectObject( memDC, memBM )) + if (!::SelectObject(memDC, memBM)) break; // Begin drawing to DC @@ -1660,7 +1766,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM break; RGBQUAD table[256]; - if( 1 == nchannels ) + if (1 == nchannels) { for(int i = 0; i < 256; ++i) { @@ -1668,14 +1774,14 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM table[i].rgbGreen = (unsigned char)i; table[i].rgbRed = (unsigned char)i; } - if (!::SetDIBColorTable(window->dc, 0, 255, table)) + if (!::SetDIBColorTable(window.dc, 0, 255, table)) break; } // The image copied to the clipboard will be in its original size, regardless if the window itself was resized. // Render the image to the dc/bitmap (at original size). - if (!::BitBlt( memDC, 0, 0, size.cx, size.cy, window->dc, 0, 0, SRCCOPY )) + if (!::BitBlt(memDC, 0, 0, size.cx, size.cy, window.dc, 0, 0, SRCCOPY)) break; // Finally, set bitmap to clipboard @@ -1712,7 +1818,7 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM case WM_RBUTTONUP: case WM_MBUTTONUP: case WM_MOUSEMOVE: - if( window->on_mouse ) + if (window.on_mouse) { POINT pt; @@ -1732,50 +1838,50 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM uMsg == WM_RBUTTONDBLCLK ? CV_EVENT_RBUTTONDBLCLK : uMsg == WM_MBUTTONDBLCLK ? CV_EVENT_MBUTTONDBLCLK : CV_EVENT_MOUSEMOVE; - if( uMsg == WM_LBUTTONDOWN || uMsg == WM_RBUTTONDOWN || uMsg == WM_MBUTTONDOWN ) - SetCapture( hwnd ); - if( uMsg == WM_LBUTTONUP || uMsg == WM_RBUTTONUP || uMsg == WM_MBUTTONUP ) + if (uMsg == WM_LBUTTONDOWN || uMsg == WM_RBUTTONDOWN || uMsg == WM_MBUTTONDOWN) + SetCapture(hwnd); + if (uMsg == WM_LBUTTONUP || uMsg == WM_RBUTTONUP || uMsg == WM_MBUTTONUP) ReleaseCapture(); - pt.x = GET_X_LPARAM( lParam ); - pt.y = GET_Y_LPARAM( lParam ); + pt.x = GET_X_LPARAM(lParam); + pt.y = GET_Y_LPARAM(lParam); - if (window->flags & CV_WINDOW_AUTOSIZE) + if (window.flags & CV_WINDOW_AUTOSIZE) { // As user can't change window size, do not scale window coordinates. Underlying windowing system // may prevent full window from being displayed and in this case coordinates should not be scaled. - window->on_mouse( event, pt.x, pt.y, flags, window->on_mouse_param ); + window.on_mouse(event, pt.x, pt.y, flags, window.on_mouse_param); } else { // Full window is displayed using different size. Scale coordinates to match underlying positions. RECT rect = { 0 }; SIZE size = {0, 0}; - GetClientRect( window->hwnd, &rect ); + GetClientRect(window.hwnd, &rect); #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) { - cv::ogl::Texture2D* texObj = static_cast(window->glDrawData); + cv::ogl::Texture2D* texObj = static_cast(window.glDrawData); size.cx = texObj->cols(); size.cy = texObj->rows(); } else { - icvGetBitmapData(window, &size, 0, 0); + icvGetBitmapData(window, size); } #else - icvGetBitmapData( window, &size, 0, 0 ); + icvGetBitmapData(window, size); #endif - window->on_mouse( event, pt.x*size.cx/MAX(rect.right - rect.left,1), - pt.y*size.cy/MAX(rect.bottom - rect.top,1), flags, - window->on_mouse_param ); + int x = cvRound((float)pt.x*size.cx/MAX(rect.right - rect.left,1)); + int y = cvRound((float)pt.y*size.cy/MAX(rect.bottom - rect.top,1)); + window.on_mouse(event, x, y, flags, window.on_mouse_param); } } break; case WM_PAINT: - if(window->image != 0) + if (window.image != 0) { int nchannels = 3; SIZE size = {0,0}; @@ -1784,12 +1890,13 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM RGBQUAD table[256]; // Determine the bitmap's dimensions - icvGetBitmapData( window, &size, &nchannels, 0 ); + void* data = 0; // unused + icvGetBitmapData(window, size, nchannels, data); hdc = BeginPaint(hwnd, &paint); SetStretchBltMode(hdc, COLORONCOLOR); - if( nchannels == 1 ) + if (nchannels == 1) { int i; for(i = 0; i < 256; i++) @@ -1798,25 +1905,25 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM table[i].rgbGreen = (unsigned char)i; table[i].rgbRed = (unsigned char)i; } - SetDIBColorTable(window->dc, 0, 255, table); + SetDIBColorTable(window.dc, 0, 255, table); } - if(window->flags & CV_WINDOW_AUTOSIZE) + if (window.flags & CV_WINDOW_AUTOSIZE) { - BitBlt( hdc, 0, 0, size.cx, size.cy, window->dc, 0, 0, SRCCOPY ); + BitBlt(hdc, 0, 0, size.cx, size.cy, window.dc, 0, 0, SRCCOPY); } else { RECT rect = { 0 }; - GetClientRect(window->hwnd, &rect); - StretchBlt( hdc, 0, 0, rect.right - rect.left, rect.bottom - rect.top, - window->dc, 0, 0, size.cx, size.cy, SRCCOPY ); + GetClientRect(window.hwnd, &rect); + StretchBlt(hdc, 0, 0, rect.right - rect.left, rect.bottom - rect.top, + window.dc, 0, 0, size.cx, size.cy, SRCCOPY); } //DeleteDC(hdc); EndPaint(hwnd, &paint); } #ifdef HAVE_OPENGL - else if(window->useGl) + else if (window.useGl) { drawGl(window); return DefWindowProc(hwnd, uMsg, wParam, lParam); @@ -1829,13 +1936,13 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM return 0; case WM_ERASEBKGND: - if(window->image) + if (window.image) return 0; break; case WM_DESTROY: - icvRemoveWindow(window); + icvRemoveWindow(window_); // Do nothing!!! //PostQuitMessage(0); break; @@ -1845,15 +1952,15 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM return 0; case WM_KEYDOWN: - window->last_key = (int)wParam; + window.last_key = (int)wParam; return 0; case WM_SIZE: - window->width = LOWORD(lParam); - window->height = HIWORD(lParam); + window.width = LOWORD(lParam); + window.height = HIWORD(lParam); #ifdef HAVE_OPENGL - if (window->useGl) + if (window.useGl) resizeGl(window); #endif } @@ -1862,24 +1969,24 @@ static LRESULT CALLBACK HighGUIProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM } -static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK WindowProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { LRESULT ret; - if( hg_on_preprocess ) + if (hg_on_preprocess) { int was_processed = 0; int rethg = hg_on_preprocess(hwnd, uMsg, wParam, lParam, &was_processed); - if( was_processed ) + if (was_processed) return rethg; } ret = HighGUIProc(hwnd, uMsg, wParam, lParam); - if(hg_on_postprocess) + if (hg_on_postprocess) { int was_processed = 0; int rethg = hg_on_postprocess(hwnd, uMsg, wParam, lParam, &was_processed); - if( was_processed ) + if (was_processed) return rethg; } @@ -1887,51 +1994,56 @@ static LRESULT CALLBACK WindowProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM } -static void icvUpdateTrackbar( CvTrackbar* trackbar, int pos ) +static void icvUpdateTrackbar(CvTrackbar& trackbar, int pos) { const int max_name_len = 10; const char* suffix = ""; char pos_text[32]; int name_len; - if( trackbar->data ) - *trackbar->data = pos; + if (trackbar.data) + *trackbar.data = pos; - if( trackbar->pos != pos ) + if (trackbar.pos != pos) { - trackbar->pos = pos; - if( trackbar->notify2 ) - trackbar->notify2(pos, trackbar->userdata); - if( trackbar->notify ) - trackbar->notify(pos); + trackbar.pos = pos; + if (trackbar.onChangeCallback) + trackbar.onChangeCallback(pos, trackbar.userdata); + if (trackbar.notify2) + trackbar.notify2(pos, trackbar.userdata); + if (trackbar.notify) + trackbar.notify(pos); - name_len = (int)strlen(trackbar->name); + name_len = (int)trackbar.name.size(); - if( name_len > max_name_len ) + // TODO replace C strings manipulation + if (name_len > max_name_len) { int start_len = max_name_len*2/3; int end_len = max_name_len - start_len - 2; - memcpy( pos_text, trackbar->name, start_len ); - memcpy( pos_text + start_len, "...", 3 ); - memcpy( pos_text + start_len + 3, trackbar->name + name_len - end_len, end_len + 1 ); + memcpy(pos_text, trackbar.name.c_str(), start_len); + memcpy(pos_text + start_len, "...", 3); + memcpy(pos_text + start_len + 3, trackbar.name.c_str() + name_len - end_len, end_len + 1); } else { - memcpy( pos_text, trackbar->name, name_len + 1); + memcpy(pos_text, trackbar.name.c_str(), name_len + 1); } - sprintf( pos_text + strlen(pos_text), "%s: %d\n", suffix, pos ); - SetWindowText( trackbar->buddy, pos_text ); + sprintf(pos_text + strlen(pos_text), "%s: %d\n", suffix, pos); + SetWindowText(trackbar.buddy, pos_text); } } -static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam ) +static LRESULT CALLBACK HGToolbarProc(HWND hwnd, UINT uMsg, WPARAM wParam, LPARAM lParam) { - CvWindow* window = icvWindowByHWND( hwnd ); - if(!window) + auto window_ = icvWindowByHWND(hwnd); + if (!window_) return DefWindowProc(hwnd, uMsg, wParam, lParam); + CvWindow& window = *window_; + // Control messages processing switch(uMsg) { @@ -1940,32 +2052,34 @@ static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPAR { HWND slider = (HWND)lParam; int pos = (int)SendMessage(slider, TBM_GETPOS, 0, 0); - CvTrackbar* trackbar = icvTrackbarByHWND( slider ); + auto trackbar = icvTrackbarByHWND(slider); - if( trackbar ) + if (trackbar) { - if( trackbar->pos != pos ) - icvUpdateTrackbar( trackbar, pos ); + if (trackbar->pos != pos) + icvUpdateTrackbar(*trackbar, pos); } - SetFocus( window->hwnd ); + SetFocus(window.hwnd); return 0; } case WM_NCCALCSIZE: { - LRESULT ret = CallWindowProc(window->toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); + LRESULT ret = CallWindowProc(window.toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); int rows = (int)SendMessage(hwnd, TB_GETROWS, 0, 0); - if(window->toolbar.rows != rows) + if (window.toolbar.rows != rows) { - SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - CvTrackbar* trackbar = window->toolbar.first; + SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); + auto& trakbars = window.toolbar.trackbars; - for( ; trackbar != 0; trackbar = trackbar->next ) + for (auto it = trakbars.begin(); it != trakbars.end(); ++it) { + auto trackbar = *it; + CV_Assert(trackbar); RECT rect = { 0 }; - SendMessage(window->toolbar.toolbar, TB_GETITEMRECT, + SendMessage(window.toolbar.toolbar, TB_GETITEMRECT, (WPARAM)trackbar->id, (LPARAM)&rect); MoveWindow(trackbar->hwnd, rect.left + HG_BUDDY_WIDTH, rect.top, rect.right - rect.left - HG_BUDDY_WIDTH, @@ -1973,46 +2087,63 @@ static LRESULT CALLBACK HGToolbarProc( HWND hwnd, UINT uMsg, WPARAM wParam, LPAR MoveWindow(trackbar->buddy, rect.left, rect.top, HG_BUDDY_WIDTH, rect.bottom - rect.top, FALSE); } - window->toolbar.rows = rows; + window.toolbar.rows = rows; } return ret; } } - return CallWindowProc(window->toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); + return CallWindowProc(window.toolbar.toolBarProc, hwnd, uMsg, wParam, lParam); } CV_IMPL void cvDestroyAllWindows(void) { - CvWindow* window = hg_windows; - - while( window ) + std::vector< std::shared_ptr > g_windows; { - HWND mainhWnd = window->frame; - HWND hwnd = window->hwnd; - window = window->next; + AutoLock lock(getWindowMutex()); + g_windows = getWindowsList(); // copy + } + for (auto it = g_windows.begin(); it != g_windows.end(); ++it) + { + auto window_ = *it; + if (!window_) + continue; - SendMessage( hwnd, WM_CLOSE, 0, 0 ); - SendMessage( mainhWnd, WM_CLOSE, 0, 0 ); + { + CvWindow& window = *window_; + + HWND mainhWnd = window.frame; + HWND hwnd = window.hwnd; + + SendMessage(hwnd, WM_CLOSE, 0, 0); + SendMessage(mainhWnd, WM_CLOSE, 0, 0); + } + + window_.reset(); + } + // TODO needed? + { + AutoLock lock(getWindowMutex()); + getWindowsList().clear(); } } -static void showSaveDialog(CvWindow* window) +static void showSaveDialog(CvWindow& window) { - if (!window || !window->image) + if (!window.image) return; SIZE sz; int channels; void* data; - if (icvGetBitmapData(window, &sz, &channels, &data)) + if (!icvGetBitmapData(window, sz, channels, data)) return; // nothing to save char szFileName[MAX_PATH] = ""; // try to use window title as file name - GetWindowText(window->frame, szFileName, MAX_PATH); + GetWindowText(window.frame, szFileName, MAX_PATH); OPENFILENAME ofn; ZeroMemory(&ofn, sizeof(ofn)); @@ -2022,7 +2153,7 @@ static void showSaveDialog(CvWindow* window) #else ofn.lStructSize = sizeof(ofn); #endif - ofn.hwndOwner = window->hwnd; + ofn.hwndOwner = window.hwnd; ofn.lpstrFilter = #ifdef HAVE_PNG "Portable Network Graphics files (*.png)\0*.png\0" @@ -2071,19 +2202,19 @@ static void showSaveDialog(CvWindow* window) */ static bool handleMessage(MSG& message, int& keyCode) { - // whether we have to call translate and dispatch yet - // otherwise the message was handled specifically - bool is_processed = false; - - for (CvWindow* window = hg_windows; window != 0 && is_processed == 0; window = window->next) + std::shared_ptr window_; { - if (!(window->hwnd == message.hwnd || window->frame == message.hwnd)) - continue; + AutoLock lock(getWindowMutex()); + window_ = icvFindWindowByHandle(message.hwnd); + } + if (window_) + { + CvWindow& window = *window_; - is_processed = true; switch (message.message) { case WM_DESTROY: + // fallthru case WM_CHAR: DispatchMessage(&message); keyCode = (int)message.wParam; @@ -2092,13 +2223,26 @@ static bool handleMessage(MSG& message, int& keyCode) case WM_SYSKEYDOWN: if (message.wParam == VK_F10) { - is_processed = true; keyCode = (int)(message.wParam << 16); return true; } break; case WM_KEYDOWN: + // Intercept Ctrl+C for copy to clipboard + if ('C' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) + { + ::SendMessage(message.hwnd, WM_COPY, 0, 0); + return false; + } + + // Intercept Ctrl+S for "save as" dialog + if ('S' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) + { + showSaveDialog(window); + return false; + } + TranslateMessage(&message); if ((message.wParam >= VK_F1 && message.wParam <= VK_F24) || message.wParam == VK_HOME || message.wParam == VK_END || @@ -2108,39 +2252,30 @@ static bool handleMessage(MSG& message, int& keyCode) message.wParam == VK_PRIOR || message.wParam == VK_NEXT) { DispatchMessage(&message); - is_processed = true; keyCode = (int)(message.wParam << 16); return true; } - // Intercept Ctrl+C for copy to clipboard - if ('C' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) - ::SendMessage(message.hwnd, WM_COPY, 0, 0); - - // Intercept Ctrl+S for "save as" dialog - if ('S' == message.wParam && (::GetKeyState(VK_CONTROL) >> 15)) - showSaveDialog(window); + // fallthru default: DispatchMessage(&message); - is_processed = true; break; } } - - if (!is_processed) + else { TranslateMessage(&message); DispatchMessage(&message); } - return false; // no value to return, keep processing + return false; // no keyCode to return, keep processing } /* * process until queue is empty but don't wait. */ -int cv::pollKey() +int pollKey_W32() { CV_TRACE_FUNCTION(); for(;;) @@ -2156,7 +2291,7 @@ int cv::pollKey() } CV_IMPL int -cvWaitKey( int delay ) +cvWaitKey(int delay) { int64 time0 = cv::getTickCount(); int64 timeEnd = time0 + (int64)(delay * 0.001f * cv::getTickFrequency()); @@ -2165,9 +2300,9 @@ cvWaitKey( int delay ) { MSG message; - if( (delay <= 0) && hg_windows) + if ((delay <= 0) && !getWindowsList().empty()) GetMessage(&message, 0, 0, 0); - else if( PeekMessage(&message, 0, 0, 0, PM_REMOVE) == FALSE ) + else if (PeekMessage(&message, 0, 0, 0, PM_REMOVE) == FALSE) { int64 t = cv::getTickCount(); if (t - timeEnd >= 0) @@ -2183,110 +2318,135 @@ cvWaitKey( int delay ) } -static CvTrackbar* -icvFindTrackbarByName( const CvWindow* window, const char* name ) +static +std::shared_ptr icvFindTrackbarByName(CvWindow& window, const std::string& name) { - CvTrackbar* trackbar = window->toolbar.first; - - for( ; trackbar != 0 && strcmp( trackbar->name, name ) != 0; trackbar = trackbar->next ) - ; - - return trackbar; + auto trackbars = window.toolbar.trackbars; + for (auto it = trackbars.begin(); it != trackbars.end(); ++it) + { + auto& trackbar = *it; + CV_Assert(trackbar); + if (trackbar->name == name) + return trackbar; + } + return std::shared_ptr(); +} +static inline +std::shared_ptr icvFindTrackbarByName(const std::shared_ptr& window, const std::string& name) +{ + CV_Assert(window); + return icvFindTrackbarByName(window, name); } +static +std::shared_ptr createTrackbar_(CvWindow& window, const std::string& trackbar_name, + int count, + TrackbarCallback onChange, void* userdata); static int -icvCreateTrackbar( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback on_notify, - CvTrackbarCallback2 on_notify2, void* userdata ) +icvCreateTrackbar(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback on_notify, + CvTrackbarCallback2 on_notify2, void* userdata) { - int result = 0; + CV_FUNCNAME("icvCreateTrackbar"); - CV_FUNCNAME( "icvCreateTrackbar" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (!window_name || !trackbar_name) + CV_Error(Error::StsNullPtr, "NULL window or trackbar name"); - char slider_name[32]; - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - int pos = 0; + if (count < 0) + CV_Error(Error::StsOutOfRange, "Bad trackbar maximal value"); - if( !window_name || !trackbar_name ) - CV_ERROR( CV_StsNullPtr, "NULL window or trackbar name" ); + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - if( count < 0 ) - CV_ERROR( CV_StsOutOfRange, "Bad trackbar maximal value" ); + auto trackbar = icvFindTrackbarByName(*window, trackbar_name); + if (!trackbar) + trackbar = createTrackbar_(*window, trackbar_name, count, nullptr, userdata); + CV_Assert(trackbar); - window = icvFindWindowByName(window_name); - if( !window ) - EXIT; + trackbar->notify = on_notify; + trackbar->notify2 = on_notify2; + trackbar->userdata = userdata; + trackbar->data = val; - trackbar = icvFindTrackbarByName(window,trackbar_name); - if( !trackbar ) + return 1; +} + +static void createToolbar_(CvWindow& window) +{ + CV_Assert(!window.toolbar.toolbar); + + const int default_height = 30; + + // CreateToolbarEx is deprecated and forces linking against Comctl32.lib. + window.toolbar.toolbar = CreateWindowEx(0, TOOLBARCLASSNAME, NULL, + WS_CHILD | CCS_TOP | TBSTYLE_WRAPABLE | BTNS_AUTOSIZE | BTNS_BUTTON, + 0, 0, 0, 0, + window.frame, NULL, GetModuleHandle(NULL), NULL); + // CreateToolbarEx automatically sends this but CreateWindowEx doesn't. + SendMessage(window.toolbar.toolbar, TB_BUTTONSTRUCTSIZE, (WPARAM)sizeof(TBBUTTON), 0); + + RECT rect; + GetClientRect(window.frame, &rect); + MoveWindow(window.toolbar.toolbar, 0, 0, + rect.right - rect.left, default_height, TRUE); + SendMessage(window.toolbar.toolbar, TB_AUTOSIZE, 0, 0); + ShowWindow(window.toolbar.toolbar, SW_SHOW); + + window.toolbar.pos = 0; + window.toolbar.rows = 0; + window.toolbar.toolBarProc = + (WNDPROC)icvGetWindowLongPtr(window.toolbar.toolbar, CV_WNDPROC); + + icvUpdateWindowPos(window); + + // Subclassing from toolbar + icvSetWindowLongPtr(window.toolbar.toolbar, CV_WNDPROC, HGToolbarProc); + icvSetWindowLongPtr(window.toolbar.toolbar, CV_USERDATA, (void*)&window); + +} + +static +std::shared_ptr createTrackbar_(CvWindow& window, const std::string& trackbar_name, + int count, + TrackbarCallback onChange, void* userdata) +{ + // create toolbar if it is not created yet + if (!window.toolbar.toolbar) { - TBBUTTON tbs = {}; - TBBUTTONINFO tbis = {}; - RECT rect = { 0 }; - int bcount; - int len = (int)strlen( trackbar_name ); + createToolbar_(window); + } - // create toolbar if it is not created yet - if( !window->toolbar.toolbar ) - { - const int default_height = 30; + TBBUTTON tbs = {}; - // CreateToolbarEx is deprecated and forces linking against Comctl32.lib. - window->toolbar.toolbar = CreateWindowEx(0, TOOLBARCLASSNAME, NULL, - WS_CHILD | CCS_TOP | TBSTYLE_WRAPABLE | BTNS_AUTOSIZE | BTNS_BUTTON, - 0, 0, 0, 0, - window->frame, NULL, GetModuleHandle(NULL), NULL); - // CreateToolbarEx automatically sends this but CreateWindowEx doesn't. - SendMessage(window->toolbar.toolbar, TB_BUTTONSTRUCTSIZE, (WPARAM)sizeof(TBBUTTON), 0); + /* Retrieve current buttons count */ + int bcount = (int)SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - GetClientRect(window->frame, &rect); - MoveWindow( window->toolbar.toolbar, 0, 0, - rect.right - rect.left, default_height, TRUE); - SendMessage(window->toolbar.toolbar, TB_AUTOSIZE, 0, 0); - ShowWindow(window->toolbar.toolbar, SW_SHOW); - - window->toolbar.first = 0; - window->toolbar.pos = 0; - window->toolbar.rows = 0; - window->toolbar.toolBarProc = - (WNDPROC)icvGetWindowLongPtr(window->toolbar.toolbar, CV_WNDPROC); - - icvUpdateWindowPos(window); - - // Subclassing from toolbar - icvSetWindowLongPtr(window->toolbar.toolbar, CV_WNDPROC, HGToolbarProc); - icvSetWindowLongPtr(window->toolbar.toolbar, CV_USERDATA, window); - } - - /* Retrieve current buttons count */ - bcount = (int)SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - - if(bcount > 1) - { - /* If this is not the first button then we need to - separate it from the previous one */ - tbs.iBitmap = 0; - tbs.idCommand = bcount; // Set button id to it's number - tbs.iString = 0; - tbs.fsStyle = TBSTYLE_SEP; - tbs.fsState = TBSTATE_ENABLED; - SendMessage(window->toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); - - // Retrieve current buttons count - bcount = (int)SendMessage(window->toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); - } - - /* Add a button which we're going to cover with the slider */ + if (bcount > 1) + { + /* If this is not the first button then we need to + separate it from the previous one */ tbs.iBitmap = 0; tbs.idCommand = bcount; // Set button id to it's number - tbs.fsState = TBSTATE_ENABLED; -#if 0/*!defined WIN64 && !defined EM64T*/ - tbs.fsStyle = 0; tbs.iString = 0; + tbs.fsStyle = TBSTYLE_SEP; + tbs.fsState = TBSTATE_ENABLED; + SendMessage(window.toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); + + // Retrieve current buttons count + bcount = (int)SendMessage(window.toolbar.toolbar, TB_BUTTONCOUNT, 0, 0); + } + + /* Add a button which we're going to cover with the slider */ + tbs.iBitmap = 0; + tbs.idCommand = bcount; // Set button id to it's number + tbs.fsState = TBSTATE_ENABLED; +#if 0/*!defined WIN64 && !defined EM64T*/ + tbs.fsStyle = 0; + tbs.iString = 0; #else #ifndef TBSTYLE_AUTOSIZE @@ -2296,307 +2456,257 @@ icvCreateTrackbar( const char* trackbar_name, const char* window_name, #ifndef TBSTYLE_GROUP #define TBSTYLE_GROUP 0x0004 #endif - //tbs.fsStyle = TBSTYLE_AUTOSIZE; - tbs.fsStyle = TBSTYLE_GROUP; - tbs.iString = (INT_PTR)trackbar_text; + //tbs.fsStyle = TBSTYLE_AUTOSIZE; + tbs.fsStyle = TBSTYLE_GROUP; + tbs.iString = (INT_PTR)trackbar_text; #endif - SendMessage(window->toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); + SendMessage(window.toolbar.toolbar, TB_ADDBUTTONS, 1, (LPARAM)&tbs); - /* Adjust button size to the slider */ - tbis.cbSize = sizeof(tbis); - tbis.dwMask = TBIF_SIZE; + TBBUTTONINFO tbis = {}; - GetClientRect(window->hwnd, &rect); - tbis.cx = (unsigned short)(rect.right - rect.left); + /* Adjust button size to the slider */ + tbis.cbSize = sizeof(tbis); + tbis.dwMask = TBIF_SIZE; - SendMessage(window->toolbar.toolbar, TB_SETBUTTONINFO, - (WPARAM)tbs.idCommand, (LPARAM)&tbis); + RECT rect = { 0 }; + GetClientRect(window.hwnd, &rect); + tbis.cx = (unsigned short)(rect.right - rect.left); - /* Get button pos */ - SendMessage(window->toolbar.toolbar, TB_GETITEMRECT, - (WPARAM)tbs.idCommand, (LPARAM)&rect); + SendMessage(window.toolbar.toolbar, TB_SETBUTTONINFO, + (WPARAM)tbs.idCommand, (LPARAM)&tbis); - /* Create a slider */ - trackbar = (CvTrackbar*)cvAlloc( sizeof(CvTrackbar) + len + 1 ); - trackbar->signature = CV_TRACKBAR_MAGIC_VAL; - trackbar->notify = 0; - trackbar->notify2 = 0; - trackbar->parent = window; - trackbar->pos = 0; - trackbar->data = 0; - trackbar->id = bcount; - trackbar->next = window->toolbar.first; - trackbar->name = (char*)(trackbar + 1); - memcpy( trackbar->name, trackbar_name, len + 1 ); - window->toolbar.first = trackbar; + /* Get button pos */ + SendMessage(window.toolbar.toolbar, TB_GETITEMRECT, + (WPARAM)tbs.idCommand, (LPARAM)&rect); - sprintf(slider_name, "Trackbar%p", val); - trackbar->hwnd = CreateWindowEx(0, TRACKBAR_CLASS, slider_name, - WS_CHILD | WS_VISIBLE | TBS_AUTOTICKS | - TBS_FIXEDLENGTH | TBS_HORZ | TBS_BOTTOM, - rect.left + HG_BUDDY_WIDTH, rect.top, - rect.right - rect.left - HG_BUDDY_WIDTH, - rect.bottom - rect.top, window->toolbar.toolbar, - (HMENU)(size_t)bcount, hg_hinstance, 0); + /* Create a slider */ + auto trackbar = std::make_shared(window, trackbar_name); + trackbar->id = bcount; + window.toolbar.trackbars.push_back(trackbar); - sprintf(slider_name,"Buddy%p", val); - trackbar->buddy = CreateWindowEx(0, "STATIC", slider_name, - WS_CHILD | SS_RIGHT, - rect.left, rect.top, - HG_BUDDY_WIDTH, rect.bottom - rect.top, - window->toolbar.toolbar, 0, hg_hinstance, 0); + auto slider_name = cv::format("Trackbar%p", trackbar.get()); + trackbar->hwnd = CreateWindowEx(0, TRACKBAR_CLASS, slider_name.c_str(), + WS_CHILD | WS_VISIBLE | TBS_AUTOTICKS | + TBS_FIXEDLENGTH | TBS_HORZ | TBS_BOTTOM, + rect.left + HG_BUDDY_WIDTH, rect.top, + rect.right - rect.left - HG_BUDDY_WIDTH, + rect.bottom - rect.top, window.toolbar.toolbar, + (HMENU)(size_t)bcount, hg_hinstance, 0); - icvSetWindowLongPtr( trackbar->hwnd, CV_USERDATA, trackbar ); + slider_name = cv::format("Buddy%p", trackbar.get()); + trackbar->buddy = CreateWindowEx(0, "STATIC", slider_name.c_str(), + WS_CHILD | SS_RIGHT, + rect.left, rect.top, + HG_BUDDY_WIDTH, rect.bottom - rect.top, + window.toolbar.toolbar, 0, hg_hinstance, 0); - /* Minimize the number of rows */ - SendMessage( window->toolbar.toolbar, TB_SETROWS, - MAKEWPARAM(1, FALSE), (LPARAM)&rect ); - } - else - { - trackbar->data = 0; - trackbar->notify = 0; - trackbar->notify2 = 0; - } + icvSetWindowLongPtr(trackbar->hwnd, CV_USERDATA, (void*)trackbar.get()); + + /* Minimize the number of rows */ + SendMessage(window.toolbar.toolbar, TB_SETROWS, + MAKEWPARAM(1, FALSE), (LPARAM)&rect); trackbar->maxval = count; /* Adjust slider parameters */ SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)0); SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)count); - SendMessage(trackbar->hwnd, TBM_SETTICFREQ, (WPARAM)1, (LPARAM)0 ); - if( val ) - pos = *val; + SendMessage(trackbar->hwnd, TBM_SETTICFREQ, (WPARAM)1, (LPARAM)0); - SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos ); - SendMessage(window->toolbar.toolbar, TB_AUTOSIZE, 0, 0); + int pos = 0; + SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + SendMessage(window.toolbar.toolbar, TB_AUTOSIZE, 0, 0); trackbar->pos = -1; - icvUpdateTrackbar( trackbar, pos ); - ShowWindow( trackbar->buddy, SW_SHOW ); - ShowWindow( trackbar->hwnd, SW_SHOW ); - - trackbar->notify = on_notify; - trackbar->notify2 = on_notify2; - trackbar->userdata = userdata; - trackbar->data = val; + icvUpdateTrackbar(*trackbar, pos); + ShowWindow(trackbar->buddy, SW_SHOW); + ShowWindow(trackbar->hwnd, SW_SHOW); /* Resize the window to reflect the toolbar resizing*/ icvUpdateWindowPos(window); - result = 1; + trackbar->onChangeCallback = onChange; + trackbar->userdata = userdata; - __END__; - - return result; + return trackbar; } CV_IMPL int -cvCreateTrackbar( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback on_notify ) +cvCreateTrackbar(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback on_notify) { - return icvCreateTrackbar( trackbar_name, window_name, val, count, - on_notify, 0, 0 ); + return icvCreateTrackbar(trackbar_name, window_name, val, count, + on_notify, 0, 0); } CV_IMPL int -cvCreateTrackbar2( const char* trackbar_name, const char* window_name, - int* val, int count, CvTrackbarCallback2 on_notify2, - void* userdata ) +cvCreateTrackbar2(const char* trackbar_name, const char* window_name, + int* val, int count, CvTrackbarCallback2 on_notify2, + void* userdata) { - return icvCreateTrackbar( trackbar_name, window_name, val, count, - 0, on_notify2, userdata ); + return icvCreateTrackbar(trackbar_name, window_name, val, count, + 0, on_notify2, userdata); } CV_IMPL void -cvSetMouseCallback( const char* window_name, CvMouseCallback on_mouse, void* param ) +cvSetMouseCallback(const char* name, CvMouseCallback on_mouse, void* param) { - CV_FUNCNAME( "cvSetMouseCallback" ); + CV_FUNCNAME("cvSetMouseCallback"); - __BEGIN__; + if (!name) + CV_Error(Error::StsNullPtr, "NULL window name"); - CvWindow* window = 0; + AutoLock lock(getWindowMutex()); - if( !window_name ) - CV_ERROR( CV_StsNullPtr, "NULL window name" ); - - window = icvFindWindowByName(window_name); - if( !window ) - EXIT; + auto window = icvFindWindowByName(name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", name)); window->on_mouse = on_mouse; window->on_mouse_param = param; - - __END__; } -CV_IMPL int cvGetTrackbarPos( const char* trackbar_name, const char* window_name ) +CV_IMPL int cvGetTrackbarPos(const char* trackbar_name, const char* window_name) { - int pos = -1; + CV_FUNCNAME("cvGetTrackbarPos"); - CV_FUNCNAME( "cvGetTrackbarPos" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (trackbar_name == 0 || window_name == 0) + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); - CvWindow* window; - CvTrackbar* trackbar = 0; + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - if( trackbar_name == 0 || window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL trackbar or window name" ); + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); - window = icvFindWindowByName( window_name ); - if( window ) - trackbar = icvFindTrackbarByName( window, trackbar_name ); - - if( trackbar ) - pos = trackbar->pos; - - __END__; - - return pos; + return trackbar->pos; } -CV_IMPL void cvSetTrackbarPos( const char* trackbar_name, const char* window_name, int pos ) +CV_IMPL void cvSetTrackbarPos(const char* trackbar_name, const char* window_name, int pos) { - CV_FUNCNAME( "cvSetTrackbarPos" ); + CV_FUNCNAME("cvSetTrackbarPos"); - __BEGIN__; + AutoLock lock(getWindowMutex()); - CvWindow* window; - CvTrackbar* trackbar = 0; + if (trackbar_name == 0 || window_name == 0) + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); - if( trackbar_name == 0 || window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL trackbar or window name" ); + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - window = icvFindWindowByName( window_name ); - if( window ) - trackbar = icvFindTrackbarByName( window, trackbar_name ); + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); - if( trackbar ) { - if( pos < 0 ) + if (pos < 0) pos = 0; - if( pos > trackbar->maxval ) + if (pos > trackbar->maxval) pos = trackbar->maxval; - SendMessage( trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos ); - icvUpdateTrackbar( trackbar, pos ); + SendMessage(trackbar->hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + icvUpdateTrackbar(*trackbar, pos); } - - __END__; } CV_IMPL void cvSetTrackbarMax(const char* trackbar_name, const char* window_name, int maxval) { - CV_FUNCNAME( "cvSetTrackbarMax" ); + CV_FUNCNAME("cvSetTrackbarMax"); - __BEGIN__; - - if (maxval >= 0) + if (trackbar_name == 0 || window_name == 0) { - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - if (trackbar_name == 0 || window_name == 0) - { - CV_ERROR(CV_StsNullPtr, "NULL trackbar or window name"); - } - - window = icvFindWindowByName(window_name); - if (window) - { - trackbar = icvFindTrackbarByName(window, trackbar_name); - if (trackbar) - { - // The position will be min(pos, maxval). - trackbar->maxval = (trackbar->minval>maxval)?trackbar->minval:maxval; - SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)maxval); - } - } + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); } - __END__; + AutoLock lock(getWindowMutex()); + + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); + + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); + + // FIXIT + if (maxval >= 0) + { + // The position will be min(pos, maxval). + trackbar->maxval = (trackbar->minval>maxval)?trackbar->minval:maxval; + SendMessage(trackbar->hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)maxval); + } } CV_IMPL void cvSetTrackbarMin(const char* trackbar_name, const char* window_name, int minval) { - CV_FUNCNAME( "cvSetTrackbarMin" ); + CV_FUNCNAME("cvSetTrackbarMin"); - __BEGIN__; - - if (minval >= 0) + if (trackbar_name == 0 || window_name == 0) { - CvWindow* window = 0; - CvTrackbar* trackbar = 0; - if (trackbar_name == 0 || window_name == 0) - { - CV_ERROR(CV_StsNullPtr, "NULL trackbar or window name"); - } - - window = icvFindWindowByName(window_name); - if (window) - { - trackbar = icvFindTrackbarByName(window, trackbar_name); - if (trackbar) - { - // The position will be min(pos, maxval). - trackbar->minval = (minvalmaxval)?minval:trackbar->maxval; - SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)minval); - } - } + CV_Error(Error::StsNullPtr, "NULL trackbar or window name"); } - __END__; + AutoLock lock(getWindowMutex()); + + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); + + auto trackbar = icvFindTrackbarByName(window, trackbar_name); + if (!trackbar) + CV_Error_(Error::StsNullPtr, ("NULL trackbar: '%s'", trackbar_name)); + + // FIXIT + if (minval >= 0) + { + // The position will be min(pos, maxval). + trackbar->minval = (minvalmaxval)?minval:trackbar->maxval; + SendMessage(trackbar->hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)minval); + } } -CV_IMPL void* cvGetWindowHandle( const char* window_name ) +CV_IMPL void* cvGetWindowHandle(const char* window_name) { - void* hwnd = 0; + CV_FUNCNAME("cvGetWindowHandle"); - CV_FUNCNAME( "cvGetWindowHandle" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (window_name == 0) + CV_Error(Error::StsNullPtr, "NULL window name"); - CvWindow* window; + auto window = icvFindWindowByName(window_name); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%s'", window_name)); - if( window_name == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL window name" ); - - window = icvFindWindowByName( window_name ); - if( window ) - hwnd = (void*)window->hwnd; - - __END__; - - return hwnd; + return (void*)window->hwnd; } - -CV_IMPL const char* cvGetWindowName( void* window_handle ) +// FIXIT: result is not safe to use +CV_IMPL const char* cvGetWindowName(void* window_handle) { - const char* window_name = ""; + CV_FUNCNAME("cvGetWindowName"); - CV_FUNCNAME( "cvGetWindowName" ); + AutoLock lock(getWindowMutex()); - __BEGIN__; + if (window_handle == 0) + CV_Error(Error::StsNullPtr, "NULL window handle"); - CvWindow* window; + auto window = icvWindowByHWND((HWND)window_handle); + if (!window) + CV_Error_(Error::StsNullPtr, ("NULL window: '%p'", window_handle)); - if( window_handle == 0 ) - CV_ERROR( CV_StsNullPtr, "NULL window" ); - - window = icvWindowByHWND( (HWND)window_handle ); - if( window ) - window_name = window->name; - - __END__; - - return window_name; + return window->name.c_str(); } @@ -2612,4 +2722,374 @@ cvSetPostprocessFuncWin32_(const void* callback) hg_on_postprocess = (CvWin32WindowCallback)callback; } -#endif //_WIN32 + + +namespace cv { namespace impl { + +using namespace cv::highgui_backend; + +class Win32UITrackbar; + +class Win32UIWindow + : public UIWindow + , public std::enable_shared_from_this +{ +protected: + const std::string name_; + std::weak_ptr window_; + std::map > trackbars_; +public: + Win32UIWindow(const std::string& name, const std::shared_ptr& window) + : name_(name) + , window_(window) + { + // nothing + } + + ~Win32UIWindow() CV_OVERRIDE + { + if (!window_.expired()) + destroy(); + CV_LOG_DEBUG(NULL, "OpenCV/UI/Win32UI: Win32UIWindow(" << name_ << ") is disposed"); + } + + const std::string& getID() const CV_OVERRIDE { return name_; } + + bool isActive() const CV_OVERRIDE { return !window_.expired(); } + + void destroy() CV_OVERRIDE + { + cv::AutoLock lock(getWindowMutex()); + if (!window_.expired()) + { + auto window = window_.lock(); + if (window) + window->destroy(); + window_.reset(); + } + } + + void imshow(InputArray image) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + Mat image_mat = image.getMat(); + showImage_(window, image_mat); + } + + double getProperty(int prop) const CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + // see cvGetWindowProperty + switch ((WindowPropertyFlags)prop) + { + case WND_PROP_FULLSCREEN: + return (double)window.status; + + case WND_PROP_AUTOSIZE: + return (window.flags & WINDOW_AUTOSIZE) ? 1.0 : 0.0; + + case WND_PROP_ASPECT_RATIO: + return static_cast(window.width) / window.height; + +#ifdef HAVE_OPENGL + case WND_PROP_OPENGL: + return window.useGl ? 1.0 : 0.0; +#endif + + case WND_PROP_VISIBLE: + return 1.0; + + case WND_PROP_TOPMOST: + return getPropTopmost_(window); + + case WND_PROP_VSYNC: + return getPropVsync_(window); + + // don't use default, add unsupported cases below: + // case WND_PROP_UNSUPPORTED: // fallthru + // break; + } + return std::numeric_limits::quiet_NaN(); + } + + bool setProperty(int prop, double value) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + // see cvSetWindowProperty + switch ((WindowPropertyFlags)prop) + { + case WND_PROP_FULLSCREEN: + if (value != WINDOW_NORMAL && value != WINDOW_FULLSCREEN) // bad arg + break; + setModeWindow_(window, (int)value); + return true; + + case WND_PROP_TOPMOST: + return setPropTopmost_(window, value != 0.0); + + case WND_PROP_VSYNC: + return setPropVsync_(window, value != 0.0); + + // don't use default, add unsupported cases below: + // case WND_PROP_UNSUPPORTED: // fallthru + case WND_PROP_AUTOSIZE: // fallthru + case WND_PROP_ASPECT_RATIO: // fallthru + case WND_PROP_OPENGL: // fallthru + case WND_PROP_VISIBLE: // fallthru + break; + } + return false; + } + + void resize(int width, int height) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + resizeWindow_(window, Size(width, height)); + } + + void move(int x, int y) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + moveWindow_(window, Point(x, y)); + } + + Rect getImageRect() const CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + return getImageRect_(window); + } + + void setTitle(const std::string& title) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + if (!SetWindowText(window.frame, title.c_str())) + CV_Error_(Error::StsError, ("Failed to set \"%s\" window title to \"%s\"", window.name.c_str(), title.c_str())); + } + + void setMouseCallback(MouseCallback onMouse, void* userdata /*= 0*/) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + window.on_mouse = onMouse; + window.on_mouse_param = userdata; + } + + std::shared_ptr createTrackbar( + const std::string& name, + int count, + TrackbarCallback onChange /*= 0*/, + void* userdata /*= 0*/ + ) CV_OVERRIDE + { + auto window_ptr = window_.lock(); + CV_Assert(window_ptr); + CvWindow& window = *window_ptr; + CV_LOG_INFO(NULL, "OpenCV/UI: Creating Win32UI trackbar at '" << name_ << "': '" << name << "'"); + auto trackbar = createTrackbar_(window, name, count, onChange, userdata); + auto ui_trackbar = std::make_shared(name, trackbar, shared_from_this()); + { + cv::AutoLock lock(getWindowMutex()); + trackbars_.emplace(name, ui_trackbar); + } + return std::static_pointer_cast(ui_trackbar); + } + + std::shared_ptr findTrackbar(const std::string& name) CV_OVERRIDE + { + cv::AutoLock lock(getWindowMutex()); + auto i = trackbars_.find(name); + if (i != trackbars_.end()) + { + return std::static_pointer_cast(i->second); + } + return std::shared_ptr(); + } +}; // Win32UIWindow + + +class Win32UITrackbar : public UITrackbar +{ +protected: + /*const*/ std::string name_; + std::weak_ptr trackbar_; + std::weak_ptr parent_; + std::map > trackbars_; +public: + Win32UITrackbar(const std::string& name, const std::shared_ptr& trackbar, const std::shared_ptr& parent) + : trackbar_(trackbar) + , parent_(parent) + { + name_ = std::string("<") + name + ">@" + parent->getID(); + } + + ~Win32UITrackbar() CV_OVERRIDE + { + if (!trackbar_.expired()) + destroy(); + CV_LOG_DEBUG(NULL, "OpenCV/UI/Win32UI: Win32UITrackbar(" << name_ << ") is disposed"); + } + + const std::string& getID() const CV_OVERRIDE { return name_; } + + bool isActive() const CV_OVERRIDE { return !trackbar_.expired(); } + + void destroy() CV_OVERRIDE + { + // nothing (destroyed with parent window, dedicated trackbar removal is not supported) + } + + int getPos() const CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + return trackbar.pos; + } + void setPos(int pos) CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + SendMessage(trackbar.hwnd, TBM_SETPOS, (WPARAM)TRUE, (LPARAM)pos); + icvUpdateTrackbar(trackbar, pos); + } + + cv::Range getRange() const CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + return cv::Range(trackbar.minval, trackbar.maxval); + } + + void setRange(const cv::Range& range) CV_OVERRIDE + { + auto trackbar_ptr = trackbar_.lock(); + CV_Assert(trackbar_ptr); + CvTrackbar& trackbar = *trackbar_ptr; + CV_CheckLE(range.start, range.end, "Invalid trackbar range"); + trackbar.minval = range.start; + trackbar.maxval = range.start; + SendMessage(trackbar.hwnd, TBM_SETRANGEMIN, (WPARAM)TRUE, (LPARAM)trackbar.minval); + SendMessage(trackbar.hwnd, TBM_SETRANGEMAX, (WPARAM)TRUE, (LPARAM)trackbar.maxval); + } +}; // Win32UITrackbar + + +class Win32BackendUI : public UIBackend +{ +public: + ~Win32BackendUI() CV_OVERRIDE + { + destroyAllWindows(); + } + + void destroyAllWindows() CV_OVERRIDE + { + cvDestroyAllWindows(); + } + + // namedWindow + virtual std::shared_ptr createWindow( + const std::string& winname, + int flags + ) CV_OVERRIDE + { + CV_LOG_INFO(NULL, "OpenCV/UI: Creating Win32UI window: " << winname << " (" << flags << ")"); + auto window = namedWindow_(winname, flags); + auto ui_window = std::make_shared(winname, window); + return ui_window; + } + + int waitKeyEx(int delay) CV_OVERRIDE + { + return cvWaitKey(delay); + } + int pollKey() CV_OVERRIDE + { + return pollKey_W32(); + } +}; // Win32BackendUI + +static +std::shared_ptr& getInstance() +{ + static std::shared_ptr g_instance = std::make_shared(); + return g_instance; +} + +} // namespace impl + +#ifndef BUILD_PLUGIN +namespace highgui_backend { + +std::shared_ptr createUIBackendWin32UI() +{ + return impl::getInstance(); +} + +} // namespace highgui_backend +#endif + +} // namespace + + +#ifdef BUILD_PLUGIN + +#define ABI_VERSION 0 +#define API_VERSION 0 +#include "plugin_api.hpp" + +static +CvResult cv_getInstance(CV_OUT CvPluginUIBackend* handle) CV_NOEXCEPT +{ + try + { + if (!handle) + return CV_ERROR_FAIL; + *handle = cv::impl::getInstance().get(); + return CV_ERROR_OK; + } + catch (...) + { + return CV_ERROR_FAIL; + } +} + +static const OpenCV_UI_Plugin_API plugin_api = +{ + { + sizeof(OpenCV_UI_Plugin_API), ABI_VERSION, API_VERSION, + CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS, + "Win32 OpenCV UI plugin" + }, + { + /* 1*/cv_getInstance + } +}; + +const OpenCV_UI_Plugin_API* CV_API_CALL opencv_ui_plugin_init_v0(int requested_abi_version, int requested_api_version, void* /*reserved=NULL*/) CV_NOEXCEPT +{ + if (requested_abi_version == ABI_VERSION && requested_api_version <= API_VERSION) + return &plugin_api; + return NULL; +} + +#endif // BUILD_PLUGIN + +#endif // HAVE_WIN32UI diff --git a/modules/imgcodecs/CMakeLists.txt b/modules/imgcodecs/CMakeLists.txt index 24f89e6e38..cc512f3e19 100644 --- a/modules/imgcodecs/CMakeLists.txt +++ b/modules/imgcodecs/CMakeLists.txt @@ -152,7 +152,7 @@ if(APPLE) add_apple_compiler_options(${the_module}) endif() -if(MSVC) +if(MSVC AND NOT BUILD_SHARED_LIBS AND BUILD_WITH_STATIC_CRT) set_target_properties(${the_module} PROPERTIES LINK_FLAGS "/NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:atlsd.lib /NODEFAULTLIB:libcmt.lib /DEBUG") endif() diff --git a/modules/imgcodecs/misc/objc/ios/Mat+Converters.h b/modules/imgcodecs/misc/objc/ios/Mat+Converters.h index a3ee005c18..0f74bb2f5d 100644 --- a/modules/imgcodecs/misc/objc/ios/Mat+Converters.h +++ b/modules/imgcodecs/misc/objc/ios/Mat+Converters.h @@ -1,5 +1,5 @@ // -// Mat+UIImage.h +// Mat+Converters.h // // Created by Giles Payne on 2020/03/03. // diff --git a/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm b/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm index 69250eb994..79358cb6de 100644 --- a/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm +++ b/modules/imgcodecs/misc/objc/ios/Mat+Converters.mm @@ -1,5 +1,5 @@ // -// Mat+UIImage.mm +// Mat+Converters.mm // // Created by Giles Payne on 2020/03/03. // diff --git a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h new file mode 100644 index 0000000000..341172798e --- /dev/null +++ b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.h @@ -0,0 +1,27 @@ +// +// Mat+QuickLook.h +// +// Created by Giles Payne on 2021/07/18. +// + +#pragma once + +#ifdef __cplusplus +#import "opencv2/core.hpp" +#else +#define CV_EXPORTS +#endif + +#import "Mat.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +CV_EXPORTS @interface Mat (QuickLook) + +- (id)debugQuickLookObject; + +@end + +NS_ASSUME_NONNULL_END diff --git a/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm new file mode 100644 index 0000000000..84cf25879b --- /dev/null +++ b/modules/imgcodecs/misc/objc/ios/Mat+QuickLook.mm @@ -0,0 +1,159 @@ +// +// Mat+QuickLook.mm +// +// Created by Giles Payne on 2021/07/18. +// + +#import "Mat+QuickLook.h" +#import "Mat+Converters.h" +#import "Rect2i.h" +#import "Core.h" +#import "Imgproc.h" +#import + +#define SIZE 20 + +static UIFont* getCMU() { + return [UIFont fontWithName:@"CMU Serif" size:SIZE]; +} + +static UIFont* getBodoni72() { + return [UIFont fontWithName:@"Bodoni 72" size:SIZE]; +} + +static UIFont* getAnySerif() { +#if defined(__IPHONE_OS_VERSION_MAX_ALLOWED) && __IPHONE_OS_VERSION_MAX_ALLOWED >= 130000 + if (@available(iOS 13.0, *)) { + return [UIFont fontWithDescriptor:[[UIFontDescriptor preferredFontDescriptorWithTextStyle:UIFontTextStyleBody] fontDescriptorWithDesign:UIFontDescriptorSystemDesignSerif] size:SIZE]; + } else { + return nil; + } +#else + return nil; +#endif +} + +static UIFont* getSystemFont() { + return [UIFont systemFontOfSize:SIZE]; +} + +typedef UIFont* (*FontGetter)(); + +@implementation Mat (QuickLook) + +- (NSString*)makeLabel:(BOOL)isIntType val:(NSNumber*)num { + if (isIntType) { + return [NSString stringWithFormat:@"%d", num.intValue]; + } else { + int exponent = 1 + (int)log10(abs(num.doubleValue)); + if (num.doubleValue == (double)num.intValue && num.doubleValue < 10000 && num.doubleValue > -10000) { + return [NSString stringWithFormat:@"%d", num.intValue];; + } else if (exponent <= 5 && exponent >= -1) { + return [NSString stringWithFormat:[NSString stringWithFormat:@"%%%d.%df", 6, MIN(5 - exponent, 4)], num.doubleValue]; + } else { + return [[[NSString stringWithFormat:@"%.2e", num.doubleValue] stringByReplacingOccurrencesOfString:@"e+0" withString:@"e"] stringByReplacingOccurrencesOfString:@"e-0" withString:@"e-"]; + } + } +} + +- (void)relativeLine:(UIBezierPath*)path relX:(CGFloat)x relY:(CGFloat)y { + CGPoint curr = path.currentPoint; + [path addLineToPoint:CGPointMake(curr.x + x, curr.y + y)]; +} + +- (id)debugQuickLookObject { + if ([self dims] == 2 && [self rows] <= 10 && [self cols] <= 10) { + FontGetter fontGetters[] = { getCMU, getBodoni72, getAnySerif, getSystemFont }; + UIFont* font = nil; + for (int fontGetterIndex = 0; font==nil && fontGetterIndex < (sizeof(fontGetters)) / (sizeof(fontGetters[0])); fontGetterIndex++) { + font = fontGetters[fontGetterIndex](); + } + int elements = [self rows] * [self cols]; + NSDictionary* textFontAttributes = @{ NSFontAttributeName: font, NSForegroundColorAttributeName: UIColor.blackColor }; + NSMutableArray* rawData = [NSMutableArray new]; + for (int dataIndex = 0; dataIndex < elements; dataIndex++) { + [rawData addObject:[NSNumber numberWithDouble:0]]; + } + [self get:0 col: 0 data: rawData]; + BOOL isIntType = [self depth] <= CV_32S; + NSMutableArray* labels = [NSMutableArray new]; + NSMutableDictionary* boundingRects = [NSMutableDictionary dictionaryWithCapacity:elements]; + int maxWidth = 0, maxHeight = 0; + for (NSNumber* number in rawData) { + NSString* label = [self makeLabel:isIntType val:number]; + [labels addObject:label]; + CGRect boundingRect = [label boundingRectWithSize:CGSizeMake(CGFLOAT_MAX, CGFLOAT_MAX) options:NSStringDrawingUsesLineFragmentOrigin attributes:textFontAttributes context:nil]; + if (boundingRect.size.width > maxWidth) { + maxWidth = boundingRect.size.width; + } + if (boundingRect.size.height > maxHeight) { + maxHeight = boundingRect.size.height; + } + boundingRects[label] = [NSValue valueWithCGRect:boundingRect]; + } + + int rowGap = 6; + int colGap = 6; + int borderGap = 8; + int lineThickness = 3; + int lipWidth = 6; + int imageWidth = 2 * (borderGap + lipWidth) + maxWidth * [self cols] + colGap * ([self cols] - 1); + int imageHeight = 2 * (borderGap + lipWidth) + maxHeight * [self rows] + rowGap * ([self rows] - 1); + + UIBezierPath* leftBracket = [UIBezierPath new]; + [leftBracket moveToPoint:CGPointMake(borderGap, borderGap)]; + [self relativeLine:leftBracket relX:0 relY:imageHeight - 2 * borderGap]; + [self relativeLine:leftBracket relX:lineThickness + lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-lineThickness]; + [self relativeLine:leftBracket relX:-lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-(imageHeight - 2 * (borderGap + lineThickness))]; + [self relativeLine:leftBracket relX:lipWidth relY:0]; + [self relativeLine:leftBracket relX:0 relY:-lineThickness]; + [leftBracket closePath]; + CGAffineTransform reflect = CGAffineTransformConcat(CGAffineTransformMakeTranslation(-imageWidth, 0), CGAffineTransformMakeScale(-1, 1)); + UIBezierPath* rightBracket = [leftBracket copy]; + [rightBracket applyTransform:reflect]; + + CGRect rect = CGRectMake(0, 0, imageWidth, imageHeight); + UIGraphicsBeginImageContextWithOptions(rect.size, false, 0.0); + [UIColor.whiteColor setFill]; + UIRectFill(rect); + [UIColor.blackColor setFill]; + [leftBracket fill]; + [rightBracket fill]; + [labels enumerateObjectsUsingBlock:^(id label, NSUInteger index, BOOL *stop) + { + CGRect boundingRect = boundingRects[label].CGRectValue; + int row = (int)index / [self cols]; + int col = (int)index % [self cols]; + int x = borderGap + lipWidth + col * (maxWidth + colGap) + (maxWidth - boundingRect.size.width) / 2; + int y = borderGap + lipWidth + row * (maxHeight + rowGap) + (maxHeight - boundingRect.size.height) / 2; + CGRect textRect = CGRectMake(x, y, boundingRect.size.width, boundingRect.size.height); + [label drawInRect:textRect withAttributes:textFontAttributes]; + }]; + UIImage* image = UIGraphicsGetImageFromCurrentImageContext(); + UIGraphicsEndImageContext(); + return image; + } else if (([self dims] == 2) && ([self type] == CV_8U || [self type] == CV_8UC3 || [self type] == CV_8UC4)) { + return [self toUIImage]; + } else if ([self dims] == 2 && [self channels] == 1) { + Mat* normalized = [Mat new]; + [Core normalize:self dst:normalized alpha:0 beta:255 norm_type:NORM_MINMAX dtype:CV_8U]; + Mat* normalizedKey = [[Mat alloc] initWithRows:[self rows] + 10 cols:[self cols] type:CV_8U]; + std::vector key; + for (int index = 0; index < [self cols]; index++) { + key.push_back((char)(index * 256 / [self cols])); + } + for (int index = 0; index < 10; index++) { + [normalizedKey put:@[[NSNumber numberWithInt:index], [NSNumber numberWithInt:0]] count:[self cols] byteBuffer:key.data()]; + } + [normalized copyTo:[normalizedKey submatRoi:[[Rect2i alloc] initWithX:0 y:10 width:[self cols] height:[self rows]]]]; + Mat* colorMap = [Mat new]; + [Imgproc applyColorMap:normalizedKey dst:colorMap colormap:COLORMAP_JET]; + [Imgproc cvtColor:colorMap dst:colorMap code:COLOR_BGR2RGB]; + return [colorMap toUIImage]; + } + return [self description]; +} + +@end diff --git a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h new file mode 100644 index 0000000000..9fa31aba39 --- /dev/null +++ b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.h @@ -0,0 +1,27 @@ +// +// Mat+QuickLook.h +// +// Created by Giles Payne on 2021/07/18. +// + +#pragma once + +#ifdef __cplusplus +#import "opencv2/core.hpp" +#else +#define CV_EXPORTS +#endif + +#import "Mat.h" +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +CV_EXPORTS @interface Mat (QuickLook) + +- (id)debugQuickLookObject; + +@end + +NS_ASSUME_NONNULL_END diff --git a/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm new file mode 100644 index 0000000000..bdc5eda811 --- /dev/null +++ b/modules/imgcodecs/misc/objc/macosx/Mat+QuickLook.mm @@ -0,0 +1,158 @@ +// +// Mat+QuickLook.mm +// +// Created by Giles Payne on 2021/07/18. +// + +#import "Mat+QuickLook.h" +#import "Mat+Converters.h" +#import "Rect2i.h" +#import "Core.h" +#import "Imgproc.h" +#import + +#define SIZE 20 + +static NSFont* getCMU() { + return [NSFont fontWithName:@"CMU Serif" size:SIZE]; +} + +static NSFont* getBodoni72() { + return [NSFont fontWithName:@"Bodoni 72" size:SIZE]; +} + +static NSFont* getAnySerif() { +#if defined(__MAC_OS_X_VERSION_MAX_ALLOWED) && __MAC_OS_X_VERSION_MAX_ALLOWED >= 110000 + if (@available(macOS 11.0, *)) { + return [NSFont fontWithDescriptor:[[NSFontDescriptor preferredFontDescriptorForTextStyle:NSFontTextStyleBody options:@{}] fontDescriptorWithDesign:NSFontDescriptorSystemDesignSerif] size:SIZE]; + } else { + return nil; + } +#else + return nil; +#endif +} + +static NSFont* getSystemFont() { + return [NSFont systemFontOfSize:SIZE]; +} + +typedef NSFont* (*FontGetter)(); + +@implementation Mat (QuickLook) + +- (NSString*)makeLabel:(BOOL)isIntType val:(NSNumber*)num { + if (isIntType) { + return [NSString stringWithFormat:@"%d", num.intValue]; + } else { + int exponent = 1 + (int)log10(abs(num.doubleValue)); + if (num.doubleValue == (double)num.intValue && num.doubleValue < 10000 && num.doubleValue > -10000) { + return [NSString stringWithFormat:@"%d", num.intValue];; + } else if (exponent <= 5 && exponent >= -1) { + return [NSString stringWithFormat:[NSString stringWithFormat:@"%%%d.%df", 6, MIN(5 - exponent, 4)], num.doubleValue]; + } else { + return [[[NSString stringWithFormat:@"%.2e", num.doubleValue] stringByReplacingOccurrencesOfString:@"e+0" withString:@"e"] stringByReplacingOccurrencesOfString:@"e-0" withString:@"e-"]; + } + } +} + +- (id)debugQuickLookObject { + // for smallish Mat objects display as a matrix + if ([self dims] == 2 && [self rows] <= 10 && [self cols] <= 10) { + FontGetter fontGetters[] = { getCMU, getBodoni72, getAnySerif, getSystemFont }; + NSFont* font = nil; + for (int fontGetterIndex = 0; font==nil && fontGetterIndex < (sizeof(fontGetters)) / (sizeof(fontGetters[0])); fontGetterIndex++) { + font = fontGetters[fontGetterIndex](); + } + int elements = [self rows] * [self cols]; + NSDictionary* textFontAttributes = @{ NSFontAttributeName: font, NSForegroundColorAttributeName: NSColor.blackColor }; + NSMutableArray* rawData = [NSMutableArray new]; + for (int dataIndex = 0; dataIndex < elements; dataIndex++) { + [rawData addObject:[NSNumber numberWithDouble:0]]; + } + [self get:0 col: 0 data: rawData]; + BOOL isIntType = [self depth] <= CV_32S; + NSMutableArray* labels = [NSMutableArray new]; + NSMutableDictionary* boundingRects = [NSMutableDictionary dictionaryWithCapacity:elements]; + int maxWidth = 0, maxHeight = 0; + for (NSNumber* number in rawData) { + NSString* label = [self makeLabel:isIntType val:number]; + [labels addObject:label]; + NSRect boundingRect = [label boundingRectWithSize:NSMakeSize(CGFLOAT_MAX, CGFLOAT_MAX) options:NSStringDrawingUsesLineFragmentOrigin attributes:textFontAttributes]; + if (boundingRect.size.width > maxWidth) { + maxWidth = boundingRect.size.width; + } + if (boundingRect.size.height > maxHeight) { + maxHeight = boundingRect.size.height; + } + boundingRects[label] = [NSValue valueWithRect:boundingRect]; + } + + int rowGap = 8; + int colGap = 8; + int borderGap = 9; + int lineThickness = 4; + int lipWidth = 8; + int imageWidth = 2 * (borderGap + lipWidth) + maxWidth * [self cols] + colGap * ([self cols] - 1); + int imageHeight = 2 * (borderGap + lipWidth) + maxHeight * [self rows] + rowGap * ([self rows] - 1); + NSImage* image = [[NSImage alloc] initWithSize:NSMakeSize(imageWidth, imageHeight)]; + NSBezierPath* leftBracket = [NSBezierPath new]; + [leftBracket moveToPoint:NSMakePoint(borderGap, borderGap)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, imageHeight - 2 * borderGap)]; + [leftBracket relativeLineToPoint:NSMakePoint(lineThickness + lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -lineThickness)]; + [leftBracket relativeLineToPoint:NSMakePoint(-lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -(imageHeight - 2 * (borderGap + lineThickness)))]; + [leftBracket relativeLineToPoint:NSMakePoint(lipWidth, 0)]; + [leftBracket relativeLineToPoint:NSMakePoint(0, -lineThickness)]; + [leftBracket relativeLineToPoint:NSMakePoint(-(lineThickness + lipWidth), 0)]; + NSAffineTransform* reflect = [NSAffineTransform new]; + [reflect scaleXBy:-1 yBy:1]; + [reflect translateXBy:-imageWidth yBy:0]; + NSBezierPath* rightBracket = [leftBracket copy]; + [rightBracket transformUsingAffineTransform:reflect]; + + [image lockFocus]; + [NSColor.whiteColor drawSwatchInRect:NSMakeRect(0, 0, imageWidth, imageHeight)]; + [NSColor.blackColor set]; + [leftBracket fill]; + [rightBracket fill]; + + [labels enumerateObjectsUsingBlock:^(id label, NSUInteger index, BOOL *stop) + { + NSRect boundingRect = boundingRects[label].rectValue; + int row = [self rows] - 1 - ((int)index / [self cols]); + int col = (int)index % [self cols]; + int x = borderGap + lipWidth + col * (maxWidth + colGap) + (maxWidth - boundingRect.size.width) / 2; + int y = borderGap + lipWidth + row * (maxHeight + rowGap) + (maxHeight - boundingRect.size.height) / 2; + NSRect textRect = NSMakeRect(x, y, boundingRect.size.width, boundingRect.size.height); + [label drawInRect:textRect withAttributes:textFontAttributes]; + }]; + [image unlockFocus]; + return image; + } else if (([self dims] == 2) && ([self type] == CV_8U || [self type] == CV_8UC3 || [self type] == CV_8UC4)) { + // convert to NSImage if the Mats has 2 dimensions and a type and number of channels consistent with it being a image + return [self toNSImage]; + } else if ([self dims] == 2 && [self channels] == 1) { + // for other Mats with 2 dimensions and one channel - generate heat map + Mat* normalized = [Mat new]; + [Core normalize:self dst:normalized alpha:0 beta:255 norm_type:NORM_MINMAX dtype:CV_8U]; + Mat* normalizedKey = [[Mat alloc] initWithRows:[self rows] + 10 cols:[self cols] type:CV_8U]; + std::vector key; + for (int index = 0; index < [self cols]; index++) { + key.push_back((char)(index * 256 / [self cols])); + } + for (int index = 0; index < 10; index++) { + [normalizedKey put:@[[NSNumber numberWithInt:index], [NSNumber numberWithInt:0]] count:[self cols] byteBuffer:key.data()]; + } + [normalized copyTo:[normalizedKey submatRoi:[[Rect2i alloc] initWithX:0 y:10 width:[self cols] height:[self rows]]]]; + Mat* colorMap = [Mat new]; + [Imgproc applyColorMap:normalizedKey dst:colorMap colormap:COLORMAP_JET]; + [Imgproc cvtColor:colorMap dst:colorMap code:COLOR_BGR2RGB]; + return [colorMap toNSImage]; + } + //everything just return the Mat description + return [self description]; +} + +@end diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index 28d8ff285b..f5ebb8a7bc 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -377,10 +377,6 @@ static void ApplyExifOrientation(ExifEntry_t orientationTag, Mat& img) * * @param[in] filename File to load * @param[in] flags Flags - * @param[in] hdrtype { LOAD_CVMAT=0, - * LOAD_IMAGE=1, - * LOAD_MAT=2 - * } * @param[in] mat Reference to C++ Mat object (If LOAD_MAT) * */ diff --git a/modules/imgproc/include/opencv2/imgproc.hpp b/modules/imgproc/include/opencv2/imgproc.hpp index 89c97085a0..f10b6af15e 100644 --- a/modules/imgproc/include/opencv2/imgproc.hpp +++ b/modules/imgproc/include/opencv2/imgproc.hpp @@ -485,6 +485,14 @@ enum HoughModes { HOUGH_GRADIENT_ALT = 4, //!< variation of HOUGH_GRADIENT to get better accuracy }; +//! Variants of Line Segment %Detector +enum LineSegmentDetectorModes { + LSD_REFINE_NONE = 0, //!< No refinement applied + LSD_REFINE_STD = 1, //!< Standard refinement is applied. E.g. breaking arches into smaller straighter line approximations. + LSD_REFINE_ADV = 2 //!< Advanced refinement. Number of false alarms is calculated, lines are + //!< refined through increase of precision, decrement in size, etc. +}; + //! @} imgproc_feature /** Histogram comparison methods @@ -1248,6 +1256,92 @@ protected: //! @} imgproc_subdiv2d + +//! @addtogroup imgproc_feature +//! @{ + +/** @example samples/cpp/lsd_lines.cpp +An example using the LineSegmentDetector +\image html building_lsd.png "Sample output image" width=434 height=300 +*/ + +/** @brief Line segment detector class + +following the algorithm described at @cite Rafael12 . + +@note Implementation has been removed from OpenCV version 3.4.6 to 3.4.15 and version 4.1.0 to 4.5.3 due original code license conflict. +restored again after [Computation of a NFA](https://github.com/rafael-grompone-von-gioi/binomial_nfa) code published under the MIT license. +*/ +class CV_EXPORTS_W LineSegmentDetector : public Algorithm +{ +public: + + /** @brief Finds lines in the input image. + + This is the output of the default parameters of the algorithm on the above shown image. + + ![image](pics/building_lsd.png) + + @param image A grayscale (CV_8UC1) input image. If only a roi needs to be selected, use: + `lsd_ptr-\>detect(image(roi), lines, ...); lines += Scalar(roi.x, roi.y, roi.x, roi.y);` + @param lines A vector of Vec4f elements specifying the beginning and ending point of a line. Where + Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. Returned lines are strictly + oriented depending on the gradient. + @param width Vector of widths of the regions, where the lines are found. E.g. Width of line. + @param prec Vector of precisions with which the lines are found. + @param nfa Vector containing number of false alarms in the line region, with precision of 10%. The + bigger the value, logarithmically better the detection. + - -1 corresponds to 10 mean false alarms + - 0 corresponds to 1 mean false alarm + - 1 corresponds to 0.1 mean false alarms + This vector will be calculated only when the objects type is #LSD_REFINE_ADV. + */ + CV_WRAP virtual void detect(InputArray image, OutputArray lines, + OutputArray width = noArray(), OutputArray prec = noArray(), + OutputArray nfa = noArray()) = 0; + + /** @brief Draws the line segments on a given image. + @param image The image, where the lines will be drawn. Should be bigger or equal to the image, + where the lines were found. + @param lines A vector of the lines that needed to be drawn. + */ + CV_WRAP virtual void drawSegments(InputOutputArray image, InputArray lines) = 0; + + /** @brief Draws two groups of lines in blue and red, counting the non overlapping (mismatching) pixels. + + @param size The size of the image, where lines1 and lines2 were found. + @param lines1 The first group of lines that needs to be drawn. It is visualized in blue color. + @param lines2 The second group of lines. They visualized in red color. + @param image Optional image, where the lines will be drawn. The image should be color(3-channel) + in order for lines1 and lines2 to be drawn in the above mentioned colors. + */ + CV_WRAP virtual int compareSegments(const Size& size, InputArray lines1, InputArray lines2, InputOutputArray image = noArray()) = 0; + + virtual ~LineSegmentDetector() { } +}; + +/** @brief Creates a smart pointer to a LineSegmentDetector object and initializes it. + +The LineSegmentDetector algorithm is defined using the standard values. Only advanced users may want +to edit those, as to tailor it for their own application. + +@param refine The way found lines will be refined, see #LineSegmentDetectorModes +@param scale The scale of the image that will be used to find the lines. Range (0..1]. +@param sigma_scale Sigma for Gaussian filter. It is computed as sigma = sigma_scale/scale. +@param quant Bound to the quantization error on the gradient norm. +@param ang_th Gradient angle tolerance in degrees. +@param log_eps Detection threshold: -log10(NFA) \> log_eps. Used only when advance refinement is chosen. +@param density_th Minimal density of aligned region points in the enclosing rectangle. +@param n_bins Number of bins in pseudo-ordering of gradient modulus. + */ +CV_EXPORTS_W Ptr createLineSegmentDetector( + LineSegmentDetectorModes refine = LSD_REFINE_STD, double scale = 0.8, + double sigma_scale = 0.6, double quant = 2.0, double ang_th = 22.5, + double log_eps = 0, double density_th = 0.7, int n_bins = 1024); + +//! @} imgproc_feature + + //! @addtogroup imgproc_filter //! @{ @@ -2229,7 +2323,7 @@ enlarge an image, it will generally look best with c#INTER_CUBIC (slow) or #INTE @param src input image. @param dst output image; it has the size dsize (when it is non-zero) or the size computed from src.size(), fx, and fy; the type of dst is the same as of src. -@param dsize output image size; if it equals zero, it is computed as: +@param dsize output image size; if it equals zero (`None` in Python), it is computed as: \f[\texttt{dsize = Size(round(fx*src.cols), round(fy*src.rows))}\f] Either dsize or both fx and fy must be non-zero. @param fx scale factor along the horizontal axis; when it equals 0, it is computed as @@ -2668,13 +2762,6 @@ CV_EXPORTS_W void warpPolar(InputArray src, OutputArray dst, Size dsize, //! @addtogroup imgproc_misc //! @{ -/** @overload */ -CV_EXPORTS_W void integral( InputArray src, OutputArray sum, int sdepth = -1 ); - -/** @overload */ -CV_EXPORTS_AS(integral2) void integral( InputArray src, OutputArray sum, - OutputArray sqsum, int sdepth = -1, int sqdepth = -1 ); - /** @brief Calculates the integral of an image. The function calculates one or more integral images for the source image as follows: @@ -2713,6 +2800,13 @@ CV_EXPORTS_AS(integral3) void integral( InputArray src, OutputArray sum, OutputArray sqsum, OutputArray tilted, int sdepth = -1, int sqdepth = -1 ); +/** @overload */ +CV_EXPORTS_W void integral( InputArray src, OutputArray sum, int sdepth = -1 ); + +/** @overload */ +CV_EXPORTS_AS(integral2) void integral( InputArray src, OutputArray sum, + OutputArray sqsum, int sdepth = -1, int sqdepth = -1 ); + //! @} imgproc_misc //! @addtogroup imgproc_motion @@ -3395,19 +3489,6 @@ the first variant of the function and distanceType == #DIST_L1. CV_EXPORTS_W void distanceTransform( InputArray src, OutputArray dst, int distanceType, int maskSize, int dstType=CV_32F); -/** @example samples/cpp/ffilldemo.cpp -An example using the FloodFill technique -*/ - -/** @overload - -variant without `mask` parameter -*/ -CV_EXPORTS int floodFill( InputOutputArray image, - Point seedPoint, Scalar newVal, CV_OUT Rect* rect = 0, - Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), - int flags = 4 ); - /** @brief Fills a connected component with the given color. The function cv::floodFill fills a connected component starting from the seed point with the specified @@ -3484,6 +3565,19 @@ CV_EXPORTS_W int floodFill( InputOutputArray image, InputOutputArray mask, Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), int flags = 4 ); +/** @example samples/cpp/ffilldemo.cpp +An example using the FloodFill technique +*/ + +/** @overload + +variant without `mask` parameter +*/ +CV_EXPORTS int floodFill( InputOutputArray image, + Point seedPoint, Scalar newVal, CV_OUT Rect* rect = 0, + Scalar loDiff = Scalar(), Scalar upDiff = Scalar(), + int flags = 4 ); + //! Performs linear blending of two images: //! \f[ \texttt{dst}(i,j) = \texttt{weights1}(i,j)*\texttt{src1}(i,j) + \texttt{weights2}(i,j)*\texttt{src2}(i,j) \f] //! @param src1 It has a type of CV_8UC(n) or CV_32FC(n), where n is a positive integer. @@ -3823,6 +3917,7 @@ hierarchy[i][0] , hierarchy[i][1] , hierarchy[i][2] , and hierarchy[i][3] are se in contours of the next and previous contours at the same hierarchical level, the first child contour and the parent contour, respectively. If for the contour i there are no next, previous, parent, or nested contours, the corresponding elements of hierarchy[i] will be negative. +@note In Python, hierarchy is nested inside a top level array. Use hierarchy[0][i] to access hierarchical elements of i-th contour. @param mode Contour retrieval mode, see #RetrievalModes @param method Contour approximation method, see #ContourApproximationModes @param offset Optional offset by which every contour point is shifted. This is useful if the @@ -4321,7 +4416,7 @@ lines are drawn using Gaussian filtering. CV_EXPORTS_W void line(InputOutputArray img, Point pt1, Point pt2, const Scalar& color, int thickness = 1, int lineType = LINE_8, int shift = 0); -/** @brief Draws a arrow segment pointing from the first point to the second one. +/** @brief Draws an arrow segment pointing from the first point to the second one. The function cv::arrowedLine draws an arrow between pt1 and pt2 points in the image. See also #line. @@ -4451,11 +4546,6 @@ CV_EXPORTS_W void drawMarker(InputOutputArray img, Point position, const Scalar& /* END OF MARKER SECTION */ /* ----------------------------------------------------------------------------------------- */ -/** @overload */ -CV_EXPORTS void fillConvexPoly(InputOutputArray img, const Point* pts, int npts, - const Scalar& color, int lineType = LINE_8, - int shift = 0); - /** @brief Fills a convex polygon. The function cv::fillConvexPoly draws a filled convex polygon. This function is much faster than the @@ -4474,10 +4564,9 @@ CV_EXPORTS_W void fillConvexPoly(InputOutputArray img, InputArray points, int shift = 0); /** @overload */ -CV_EXPORTS void fillPoly(InputOutputArray img, const Point** pts, - const int* npts, int ncontours, - const Scalar& color, int lineType = LINE_8, int shift = 0, - Point offset = Point() ); +CV_EXPORTS void fillConvexPoly(InputOutputArray img, const Point* pts, int npts, + const Scalar& color, int lineType = LINE_8, + int shift = 0); /** @example samples/cpp/tutorial_code/ImgProc/basic_drawing/Drawing_1.cpp An example using drawing functions @@ -4502,9 +4591,10 @@ CV_EXPORTS_W void fillPoly(InputOutputArray img, InputArrayOfArrays pts, Point offset = Point() ); /** @overload */ -CV_EXPORTS void polylines(InputOutputArray img, const Point* const* pts, const int* npts, - int ncontours, bool isClosed, const Scalar& color, - int thickness = 1, int lineType = LINE_8, int shift = 0 ); +CV_EXPORTS void fillPoly(InputOutputArray img, const Point** pts, + const int* npts, int ncontours, + const Scalar& color, int lineType = LINE_8, int shift = 0, + Point offset = Point() ); /** @brief Draws several polygonal curves. @@ -4523,6 +4613,11 @@ CV_EXPORTS_W void polylines(InputOutputArray img, InputArrayOfArrays pts, bool isClosed, const Scalar& color, int thickness = 1, int lineType = LINE_8, int shift = 0 ); +/** @overload */ +CV_EXPORTS void polylines(InputOutputArray img, const Point* const* pts, const int* npts, + int ncontours, bool isClosed, const Scalar& color, + int thickness = 1, int lineType = LINE_8, int shift = 0 ); + /** @example samples/cpp/contours2.cpp An example program illustrates the use of cv::findContours and cv::drawContours \image html WindowsQtContoursOutput.png "Screenshot of the program" @@ -4555,7 +4650,7 @@ parameter is only taken into account when there is hierarchy available. @param offset Optional contour shift parameter. Shift all the drawn contours by the specified \f$\texttt{offset}=(dx,dy)\f$ . @note When thickness=#FILLED, the function is designed to handle connected components with holes correctly -even when no hierarchy date is provided. This is done by analyzing all the outlines together +even when no hierarchy data is provided. This is done by analyzing all the outlines together using even-odd rule. This may give incorrect results if you have a joint collection of separately retrieved contours. In order to solve this problem, you need to call #drawContours separately for each sub-group of contours, or iterate over the collection using contourIdx parameter. @@ -4569,7 +4664,7 @@ CV_EXPORTS_W void drawContours( InputOutputArray image, InputArrayOfArrays conto /** @brief Clips the line against the image rectangle. The function cv::clipLine calculates a part of the line segment that is entirely within the specified -rectangle. it returns false if the line segment is completely outside the rectangle. Otherwise, +rectangle. It returns false if the line segment is completely outside the rectangle. Otherwise, it returns true . @param imgSize Image size. The image rectangle is Rect(0, 0, imgSize.width, imgSize.height) . @param pt1 First line point. diff --git a/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp b/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp index ac20725e63..f129012ba6 100644 --- a/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp +++ b/modules/imgproc/include/opencv2/imgproc/hal/hal.hpp @@ -198,6 +198,11 @@ CV_EXPORTS void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx); +CV_EXPORTS void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, + int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx); + CV_EXPORTS void cvtThreePlaneYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, diff --git a/modules/imgproc/src/color_yuv.dispatch.cpp b/modules/imgproc/src/color_yuv.dispatch.cpp index 18b2096680..cac4fa1b41 100644 --- a/modules/imgproc/src/color_yuv.dispatch.cpp +++ b/modules/imgproc/src/color_yuv.dispatch.cpp @@ -124,8 +124,9 @@ void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, CALL_HAL(cvtTwoPlaneYUVtoBGR, cv_hal_cvtTwoPlaneYUVtoBGR, src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); - CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (src_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), - CV_CPU_DISPATCH_MODES_ALL); + cvtTwoPlaneYUVtoBGR( + src_data, src_step, src_data + src_step * dst_height, src_step, dst_data, dst_step, + dst_width, dst_height, dcn, swapBlue, uIdx); } void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, @@ -135,7 +136,20 @@ void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src { CV_INSTRUMENT_REGION(); - CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (y_data, uv_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), + cvtTwoPlaneYUVtoBGR(y_data, src_step, uv_data, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); +} + +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, + int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx) +{ + CV_INSTRUMENT_REGION(); + + CALL_HAL(cvtTwoPlaneYUVtoBGREx, cv_hal_cvtTwoPlaneYUVtoBGREx, + y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); + + CV_CPU_DISPATCH(cvtTwoPlaneYUVtoBGR, (y_data, y_step, uv_data, uv_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx), CV_CPU_DISPATCH_MODES_ALL); } @@ -172,7 +186,8 @@ void cvtBGRtoTwoPlaneYUV(const uchar * src_data, size_t src_step, { CV_INSTRUMENT_REGION(); - // TODO: add hal replacement method + CALL_HAL(cvtBGRtoTwoPlaneYUV, cv_hal_cvtBGRtoTwoPlaneYUV, + src_data, src_step, y_data, dst_step, uv_data, dst_step, width, height, scn, swapBlue, uIdx); CV_CPU_DISPATCH(cvtBGRtoTwoPlaneYUV, (src_data, src_step, y_data, uv_data, dst_step, width, height, scn, swapBlue, uIdx), CV_CPU_DISPATCH_MODES_ALL); @@ -406,14 +421,21 @@ void cvtColorTwoPlaneYUV2BGRpair( InputArray _ysrc, InputArray _uvsrc, OutputArr Mat ysrc = _ysrc.getMat(), uvsrc = _uvsrc.getMat(); - CV_CheckEQ(ysrc.step, uvsrc.step, ""); - _dst.create( ysz, CV_MAKETYPE(depth, dcn)); Mat dst = _dst.getMat(); - hal::cvtTwoPlaneYUVtoBGR(ysrc.data, uvsrc.data, ysrc.step, - dst.data, dst.step, dst.cols, dst.rows, - dcn, swapb, uidx); + if(ysrc.step == uvsrc.step) + { + hal::cvtTwoPlaneYUVtoBGR(ysrc.data, uvsrc.data, ysrc.step, + dst.data, dst.step, dst.cols, dst.rows, + dcn, swapb, uidx); + } + else + { + hal::cvtTwoPlaneYUVtoBGR(ysrc.data, ysrc.step, uvsrc.data, uvsrc.step, + dst.data, dst.step, dst.cols, dst.rows, + dcn, swapb, uidx); + } } } // namespace cv diff --git a/modules/imgproc/src/color_yuv.simd.hpp b/modules/imgproc/src/color_yuv.simd.hpp index 076d1a4bd5..196a03b995 100644 --- a/modules/imgproc/src/color_yuv.simd.hpp +++ b/modules/imgproc/src/color_yuv.simd.hpp @@ -17,11 +17,7 @@ void cvtYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int width, int height, int depth, int dcn, bool swapBlue, bool isCbCr); -void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, - uchar * dst_data, size_t dst_step, - int dst_width, int dst_height, - int dcn, bool swapBlue, int uIdx); -void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx); @@ -1177,24 +1173,28 @@ struct YUV420sp2RGB8Invoker : ParallelLoopBody uchar * dst_data; size_t dst_step; int width; - const uchar* my1, *muv; - size_t stride; + const uchar* my1; + size_t my1_step; + const uchar* muv; + size_t muv_step; - YUV420sp2RGB8Invoker(uchar * _dst_data, size_t _dst_step, int _dst_width, size_t _stride, const uchar* _y1, const uchar* _uv) - : dst_data(_dst_data), dst_step(_dst_step), width(_dst_width), my1(_y1), muv(_uv), stride(_stride) {} + YUV420sp2RGB8Invoker(uchar * _dst_data, size_t _dst_step, int _dst_width, + const uchar* _y1, size_t _y1_step, const uchar* _uv, size_t _uv_step) : + dst_data(_dst_data), dst_step(_dst_step), width(_dst_width), + my1(_y1), my1_step(_y1_step), muv(_uv), muv_step(_uv_step) {} void operator()(const Range& range) const CV_OVERRIDE { const int rangeBegin = range.start * 2; const int rangeEnd = range.end * 2; - const uchar* y1 = my1 + rangeBegin * stride, *uv = muv + rangeBegin * stride / 2; + const uchar* y1 = my1 + rangeBegin * my1_step, *uv = muv + rangeBegin * muv_step / 2; - for (int j = rangeBegin; j < rangeEnd; j += 2, y1 += stride * 2, uv += stride) + for (int j = rangeBegin; j < rangeEnd; j += 2, y1 += my1_step * 2, uv += muv_step) { uchar* row1 = dst_data + dst_step * j; uchar* row2 = dst_data + dst_step * (j + 1); - const uchar* y2 = y1 + stride; + const uchar* y2 = y1 + my1_step; int i = 0; #if CV_SIMD @@ -1395,9 +1395,10 @@ struct YUV420p2RGB8Invoker : ParallelLoopBody #define MIN_SIZE_FOR_PARALLEL_YUV420_CONVERSION (320*240) template -inline void cvtYUV420sp2RGB(uchar * dst_data, size_t dst_step, int dst_width, int dst_height, size_t _stride, const uchar* _y1, const uchar* _uv) +inline void cvtYUV420sp2RGB(uchar * dst_data, size_t dst_step, int dst_width, int dst_height, + const uchar* _y1, size_t _y1_step, const uchar* _uv, size_t _uv_step) { - YUV420sp2RGB8Invoker converter(dst_data, dst_step, dst_width, _stride, _y1, _uv); + YUV420sp2RGB8Invoker converter(dst_data, dst_step, dst_width, _y1, _y1_step, _uv, _uv_step); if (dst_width * dst_height >= MIN_SIZE_FOR_PARALLEL_YUV420_CONVERSION) parallel_for_(Range(0, dst_height/2), converter); else @@ -1817,26 +1818,16 @@ void cvtYUVtoBGR(const uchar * src_data, size_t src_step, CvtColorLoop(src_data, src_step, dst_data, dst_step, width, height, YCrCb2RGB_f(dcn, blueIdx, isCbCr)); } -void cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, - uchar * dst_data, size_t dst_step, - int dst_width, int dst_height, - int dcn, bool swapBlue, int uIdx) -{ - CV_INSTRUMENT_REGION(); - - const uchar* uv = src_data + src_step * static_cast(dst_height); - cvtTwoPlaneYUVtoBGR(src_data, uv, src_step, dst_data, dst_step, dst_width, dst_height, dcn, swapBlue, uIdx); -} - typedef void (*cvt_2plane_yuv_ptr_t)(uchar * /* dst_data*/, size_t /* dst_step */, int /* dst_width */, int /* dst_height */, - size_t /* _stride */, const uchar* /* _y1 */, - const uchar* /* _uv */); + size_t /* _y1_step */, + const uchar* /* _uv */, + size_t /* _uv_step */); -void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src_step, +void cvtTwoPlaneYUVtoBGR(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx) @@ -1859,7 +1850,7 @@ void cvtTwoPlaneYUVtoBGR(const uchar * y_data, const uchar * uv_data, size_t src default: CV_Error( CV_StsBadFlag, "Unknown/unsupported color conversion code" ); break; }; - cvtPtr(dst_data, dst_step, dst_width, dst_height, src_step, y_data, uv_data); + cvtPtr(dst_data, dst_step, dst_width, dst_height, y_data, y_step, uv_data, uv_step); } typedef void (*cvt_3plane_yuv_ptr_t)(uchar * /* dst_data */, diff --git a/modules/imgproc/src/filter.simd.hpp b/modules/imgproc/src/filter.simd.hpp index bed6f83071..966de94c0b 100644 --- a/modules/imgproc/src/filter.simd.hpp +++ b/modules/imgproc/src/filter.simd.hpp @@ -465,6 +465,49 @@ struct RowVec_8u32s bool smallValues; }; +struct RowVec_8u32f +{ + RowVec_8u32f() {} + RowVec_8u32f( const Mat& _kernel ) : kernel(_kernel) {} + + int operator()(const uchar* _src, uchar* _dst, int width, int cn) const + { + CV_INSTRUMENT_REGION(); + + int i = 0, k, _ksize = kernel.rows + kernel.cols - 1; + float* dst = (float*)_dst; + const float* _kx = kernel.ptr(); + width *= cn; + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 s0 = vx_setzero_f32(); + v_float32 s1 = vx_setzero_f32(); + v_float32 s2 = vx_setzero_f32(); + v_float32 s3 = vx_setzero_f32(); + k = 0; + for( ; k < _ksize ; k++ ) + { + v_float32 f = vx_setall_f32(_kx[k]); + const uchar* src = (const uchar*)_src + i + k * cn; + v_float32 vs_ll = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src))); + v_float32 vs_lh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + v_float32::nlanes))); + v_float32 vs_hl = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 2*v_float32::nlanes))); + v_float32 vs_hh = v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(src + 3*v_float32::nlanes))); + s0 = v_muladd(vs_ll, f, s0); + s1 = v_muladd(vs_lh, f, s1); + s2 = v_muladd(vs_hl, f, s2); + s3 = v_muladd(vs_hh, f, s3); + } + v_store(dst + i, s0); + v_store(dst + i + v_float32::nlanes, s1); + v_store(dst + i + 2*v_float32::nlanes, s2); + v_store(dst + i + 3*v_float32::nlanes, s3); + } + return i; + } + + Mat kernel; +}; struct SymmRowSmallVec_8u32s { @@ -1117,6 +1160,81 @@ struct SymmColumnVec_32s8u Mat kernel; }; +struct SymmColumnVec_32f8u +{ + SymmColumnVec_32f8u() { symmetryType = 0; delta = 0; } + SymmColumnVec_32f8u(const Mat& _kernel, int _symmetryType, int, double _delta) + { + symmetryType = _symmetryType; + kernel = _kernel; + delta = (float)_delta; + CV_Assert( (symmetryType & (KERNEL_SYMMETRICAL | KERNEL_ASYMMETRICAL)) != 0 ); + } + + int operator()(const uchar** _src, uchar* _dst, int width) const + { + CV_INSTRUMENT_REGION(); + + int _ksize = kernel.rows + kernel.cols - 1; + if( _ksize == 1 ) return 0; + const int ksize2 = _ksize / 2; + const float* ky = kernel.ptr() + ksize2; + int i = 0, k; + bool symmetrical = (symmetryType & KERNEL_SYMMETRICAL) != 0; + const float** src = (const float**)_src; + + if( symmetrical ) + { + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 v_ky0 = vx_setall_f32(ky[0]); + v_float32 v32_delta = vx_setall_f32(delta); + const float* S = src[0] + i; + v_float32 s0 = v_muladd(v_ky0, vx_load(S), v32_delta); + v_float32 s1 = v_muladd(v_ky0, vx_load(S + v_float32::nlanes), v32_delta); + v_float32 s2 = v_muladd(v_ky0, vx_load(S + 2*v_float32::nlanes), v32_delta); + v_float32 s3 = v_muladd(v_ky0, vx_load(S + 3*v_float32::nlanes), v32_delta); + for( k = 1; k <= ksize2; k++ ) + { + v_float32 v_kyk = vx_setall_f32(ky[k]); + const float* S0 = src[k] + i; + const float* S1 = src[-k] + i; + s0 = v_muladd(v_kyk, vx_load(S0) + vx_load(S1), s0); + s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) + vx_load(S1 + v_float32::nlanes), s1); + s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) + vx_load(S1 + 2*v_float32::nlanes), s2); + s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) + vx_load(S1 + 3*v_float32::nlanes), s3); + } + v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); + } + } + else + { + for( ; i <= width - v_uint8::nlanes; i += v_uint8::nlanes ) + { + v_float32 s0 = vx_setall_f32(delta); + v_float32 s1 = vx_setall_f32(delta); + v_float32 s2 = vx_setall_f32(delta); + v_float32 s3 = vx_setall_f32(delta); + for( k = 1; k <= ksize2; k++ ) + { + v_float32 v_kyk = vx_setall_f32(ky[k]); + const float* S0 = src[k] + i; + const float* S1 = src[-k] + i; + s0 = v_muladd(v_kyk, vx_load(S0) - vx_load(S1), s0); + s1 = v_muladd(v_kyk, vx_load(S0 + v_float32::nlanes) - vx_load(S1 + v_float32::nlanes), s1); + s2 = v_muladd(v_kyk, vx_load(S0 + 2*v_float32::nlanes) - vx_load(S1 + 2*v_float32::nlanes), s2); + s3 = v_muladd(v_kyk, vx_load(S0 + 3*v_float32::nlanes) - vx_load(S1 + 3*v_float32::nlanes), s3); + } + v_store(_dst + i, v_pack_u(v_pack(v_round(s0), v_round(s1)), v_pack(v_round(s2), v_round(s3)))); + } + } + return i; + } + int symmetryType; + float delta; + Mat kernel; +}; + struct SymmColumnSmallVec_32s16s { @@ -2292,11 +2410,13 @@ struct FilterVec_32f #else typedef RowNoVec RowVec_8u32s; +typedef RowNoVec RowVec_8u32f; typedef RowNoVec RowVec_16s32f; typedef RowNoVec RowVec_32f; typedef SymmRowSmallNoVec SymmRowSmallVec_8u32s; typedef SymmRowSmallNoVec SymmRowSmallVec_32f; typedef ColumnNoVec SymmColumnVec_32s8u; +typedef ColumnNoVec SymmColumnVec_32f8u; typedef ColumnNoVec SymmColumnVec_32f16s; typedef ColumnNoVec SymmColumnVec_32f; typedef SymmColumnSmallNoVec SymmColumnSmallVec_32s16s; @@ -2899,7 +3019,8 @@ Ptr getLinearRowFilter( return makePtr > (kernel, anchor, RowVec_8u32s(kernel)); if( sdepth == CV_8U && ddepth == CV_32F ) - return makePtr >(kernel, anchor); + return makePtr > + (kernel, anchor, RowVec_8u32f(kernel)); if( sdepth == CV_8U && ddepth == CV_64F ) return makePtr >(kernel, anchor); if( sdepth == CV_16U && ddepth == CV_32F ) @@ -2986,8 +3107,9 @@ Ptr getLinearColumnFilter( (kernel, anchor, delta, symmetryType, FixedPtCastEx(bits), SymmColumnVec_32s8u(kernel, symmetryType, bits, delta)); if( ddepth == CV_8U && sdepth == CV_32F ) - return makePtr, ColumnNoVec> > - (kernel, anchor, delta, symmetryType); + return makePtr, SymmColumnVec_32f8u> > + (kernel, anchor, delta, symmetryType, Cast(), + SymmColumnVec_32f8u(kernel, symmetryType, 0, delta)); if( ddepth == CV_8U && sdepth == CV_64F ) return makePtr, ColumnNoVec> > (kernel, anchor, delta, symmetryType); diff --git a/modules/imgproc/src/hal_replacement.hpp b/modules/imgproc/src/hal_replacement.hpp index 40f9d302df..ff4a41923d 100644 --- a/modules/imgproc/src/hal_replacement.hpp +++ b/modules/imgproc/src/hal_replacement.hpp @@ -498,6 +498,39 @@ inline int hal_ni_cvtLabtoBGR(const uchar * src_data, size_t src_step, uchar * d */ inline int hal_ni_cvtTwoPlaneYUVtoBGR(const uchar * src_data, size_t src_step, uchar * dst_data, size_t dst_step, int dst_width, int dst_height, int dcn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } +/** + @brief Extended version of hal_cvtTwoPlaneYUVtoBGR. + @param y_data,y_step source image data and step (Y-plane) + @param uv_data,uv_step source image data and step (UV-plane) + @param dst_data,dst_step destination image data and step + @param dst_width,dst_height destination image size + @param dcn destination image channels (3 or 4) + @param swapBlue if set to true B and R destination channels will be swapped (write RGB) + @param uIdx U-channel index in the interleaved U/V plane (0 or 1) + Convert from YUV (YUV420sp (or NV12/NV21) - Y plane followed by interleaved U/V plane) to BGR, RGB, BGRA or RGBA. + Only for CV_8U. + */ +inline int hal_ni_cvtTwoPlaneYUVtoBGREx(const uchar * y_data, size_t y_step, const uchar * uv_data, size_t uv_step, + uchar * dst_data, size_t dst_step, int dst_width, int dst_height, + int dcn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } + +/** + @brief hal_cvtBGRtoTwoPlaneYUV + @param src_data,src_step source image data and step + @param y_data,y_step destination image data and step (Y-plane) + @param uv_data,uv_step destination image data and step (UV-plane) + @param width,height image size + @param scn source image channels (3 or 4) + @param swapBlue if set to true B and R source channels will be swapped (treat as RGB) + @param uIdx U-channel plane index (0 or 1) + Convert from BGR, RGB, BGRA or RGBA to YUV (YUV420sp (or NV12/NV21) - Y plane followed by interleaved U/V plane). + Only for CV_8U. + */ +inline int hal_ni_cvtBGRtoTwoPlaneYUV(const uchar * src_data, size_t src_step, + uchar * y_data, size_t y_step, uchar * uv_data, size_t uv_step, + int width, int height, + int scn, bool swapBlue, int uIdx) { return CV_HAL_ERROR_NOT_IMPLEMENTED; } + /** @brief hal_cvtThreePlaneYUVtoBGR @param src_data,src_step source image data and step @@ -576,6 +609,8 @@ inline int hal_ni_cvtMultipliedRGBAtoRGBA(const uchar * src_data, size_t src_ste #define cv_hal_cvtBGRtoLab hal_ni_cvtBGRtoLab #define cv_hal_cvtLabtoBGR hal_ni_cvtLabtoBGR #define cv_hal_cvtTwoPlaneYUVtoBGR hal_ni_cvtTwoPlaneYUVtoBGR +#define cv_hal_cvtTwoPlaneYUVtoBGREx hal_ni_cvtTwoPlaneYUVtoBGREx +#define cv_hal_cvtBGRtoTwoPlaneYUV hal_ni_cvtBGRtoTwoPlaneYUV #define cv_hal_cvtThreePlaneYUVtoBGR hal_ni_cvtThreePlaneYUVtoBGR #define cv_hal_cvtBGRtoThreePlaneYUV hal_ni_cvtBGRtoThreePlaneYUV #define cv_hal_cvtOnePlaneYUVtoBGR hal_ni_cvtOnePlaneYUVtoBGR diff --git a/modules/imgproc/src/hough.cpp b/modules/imgproc/src/hough.cpp index 1743e9a9db..38441f4230 100644 --- a/modules/imgproc/src/hough.cpp +++ b/modules/imgproc/src/hough.cpp @@ -435,12 +435,14 @@ HoughLinesSDiv( InputArray image, OutputArray lines, int type, } } + int pos = (int)(lst.size() - 1); + if( pos >= 0 && lst[pos].rho < 0 ) + lst.pop_back(); + lines.create((int)lst.size(), 1, type); Mat _lines = lines.getMat(); for( size_t idx = 0; idx < lst.size(); idx++ ) { - if( lst[idx].rho < 0 ) - continue; if (type == CV_32FC2) { _lines.at((int)idx) = Vec2f(lst[idx].rho, lst[idx].theta); @@ -2301,14 +2303,16 @@ static void HoughCircles( InputArray _image, OutputArray _circles, std::vector cw(ncircles); for( i = 0; i < ncircles; i++ ) cw[i] = GetCircle4f(circles[i]); - Mat(cw).copyTo(_circles); + if (ncircles > 0) + Mat(1, (int)ncircles, cv::traits::Type::value, &cw[0]).copyTo(_circles); } else if( type == CV_32FC3 ) { std::vector cwow(ncircles); for( i = 0; i < ncircles; i++ ) cwow[i] = GetCircle(circles[i]); - Mat(cwow).copyTo(_circles); + if (ncircles > 0) + Mat(1, (int)ncircles, cv::traits::Type::value, &cwow[0]).copyTo(_circles); } else CV_Error(Error::StsError, "Internal error"); diff --git a/modules/imgproc/src/intersection.cpp b/modules/imgproc/src/intersection.cpp index 3f749896a4..47d3f3f457 100644 --- a/modules/imgproc/src/intersection.cpp +++ b/modules/imgproc/src/intersection.cpp @@ -47,24 +47,16 @@ namespace cv { -int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, OutputArray intersectingRegion ) +static int _rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, std::vector &intersection ) { CV_INSTRUMENT_REGION(); // L2 metric const float samePointEps = std::max(1e-16f, 1e-6f * (float)std::max(rect1.size.area(), rect2.size.area())); - if (rect1.size.empty() || rect2.size.empty()) - { - intersectingRegion.release(); - return INTERSECT_NONE; - } - Point2f vec1[4], vec2[4]; Point2f pts1[4], pts2[4]; - std::vector intersection; intersection.reserve(24); - rect1.points(pts1); rect2.points(pts2); @@ -92,8 +84,6 @@ int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& r intersection[i] = pts1[i]; } - Mat(intersection).copyTo(intersectingRegion); - return INTERSECT_FULL; } } @@ -300,7 +290,50 @@ int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& r } intersection.resize(N); - Mat(intersection).copyTo(intersectingRegion); + + return ret; +} + +int rotatedRectangleIntersection( const RotatedRect& rect1, const RotatedRect& rect2, OutputArray intersectingRegion ) +{ + CV_INSTRUMENT_REGION(); + + if (rect1.size.empty() || rect2.size.empty()) + { + intersectingRegion.release(); + return INTERSECT_NONE; + } + + // Shift rectangles closer to origin (0, 0) to improve the calculation of the intesection region + // To do that, the average center of the rectangles is moved to the origin + const Point2f averageCenter = (rect1.center + rect2.center) / 2.0f; + + RotatedRect shiftedRect1(rect1); + RotatedRect shiftedRect2(rect2); + + // Move rectangles closer to origin + shiftedRect1.center -= averageCenter; + shiftedRect2.center -= averageCenter; + + std::vector intersection; intersection.reserve(24); + + const int ret = _rotatedRectangleIntersection(shiftedRect1, shiftedRect2, intersection); + + // If return is not None, the intersection Points are shifted back to the original position + // and copied to the interesectingRegion + if (ret != INTERSECT_NONE) + { + for (size_t i = 0; i < intersection.size(); ++i) + { + intersection[i] += averageCenter; + } + + Mat(intersection).copyTo(intersectingRegion); + } + else + { + intersectingRegion.release(); + } return ret; } diff --git a/modules/imgproc/src/lsd.cpp b/modules/imgproc/src/lsd.cpp new file mode 100644 index 0000000000..63e10d8fdc --- /dev/null +++ b/modules/imgproc/src/lsd.cpp @@ -0,0 +1,1231 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "precomp.hpp" +#include + +///////////////////////////////////////////////////////////////////////////////////////// +// Default LSD parameters +// SIGMA_SCALE 0.6 - Sigma for Gaussian filter is computed as sigma = sigma_scale/scale. +// QUANT 2.0 - Bound to the quantization error on the gradient norm. +// ANG_TH 22.5 - Gradient angle tolerance in degrees. +// LOG_EPS 0.0 - Detection threshold: -log10(NFA) > log_eps +// DENSITY_TH 0.7 - Minimal density of region points in rectangle. +// N_BINS 1024 - Number of bins in pseudo-ordering of gradient modulus. + +#define M_3_2_PI (3 * CV_PI) / 2 // 3/2 pi +#define M_2__PI (2 * CV_PI) // 2 pi + +#ifndef M_LN10 +#define M_LN10 2.30258509299404568402 +#endif + +#define NOTDEF double(-1024.0) // Label for pixels with undefined gradient. + +#define NOTUSED 0 // Label for pixels not used in yet. +#define USED 1 // Label for pixels already used in detection. + +#define RELATIVE_ERROR_FACTOR 100.0 + +const double DEG_TO_RADS = CV_PI / 180; + +#define log_gamma(x) ((x)>15.0?log_gamma_windschitl(x):log_gamma_lanczos(x)) + +struct edge +{ + cv::Point p; + bool taken; +}; + +///////////////////////////////////////////////////////////////////////////////////////// + +inline double distSq(const double x1, const double y1, + const double x2, const double y2) +{ + return (x2 - x1)*(x2 - x1) + (y2 - y1)*(y2 - y1); +} + +inline double dist(const double x1, const double y1, + const double x2, const double y2) +{ + return sqrt(distSq(x1, y1, x2, y2)); +} + +// Signed angle difference +inline double angle_diff_signed(const double& a, const double& b) +{ + double diff = a - b; + while(diff <= -CV_PI) diff += M_2__PI; + while(diff > CV_PI) diff -= M_2__PI; + return diff; +} + +// Absolute value angle difference +inline double angle_diff(const double& a, const double& b) +{ + return std::fabs(angle_diff_signed(a, b)); +} + +// Compare doubles by relative error. +inline bool double_equal(const double& a, const double& b) +{ + // trivial case + if(a == b) return true; + + double abs_diff = fabs(a - b); + double aa = fabs(a); + double bb = fabs(b); + double abs_max = (aa > bb)? aa : bb; + + if(abs_max < DBL_MIN) abs_max = DBL_MIN; + + return (abs_diff / abs_max) <= (RELATIVE_ERROR_FACTOR * DBL_EPSILON); +} + +inline bool AsmallerB_XoverY(const edge& a, const edge& b) +{ + if (a.p.x == b.p.x) return a.p.y < b.p.y; + else return a.p.x < b.p.x; +} + +/** + * Computes the natural logarithm of the absolute value of + * the gamma function of x using Windschitl method. + * See http://www.rskey.org/gamma.htm + */ +inline double log_gamma_windschitl(const double& x) +{ + return 0.918938533204673 + (x-0.5)*log(x) - x + + 0.5*x*log(x*sinh(1/x) + 1/(810.0*pow(x, 6.0))); +} + +/** + * Computes the natural logarithm of the absolute value of + * the gamma function of x using the Lanczos approximation. + * See http://www.rskey.org/gamma.htm + */ +inline double log_gamma_lanczos(const double& x) +{ + static double q[7] = { 75122.6331530, 80916.6278952, 36308.2951477, + 8687.24529705, 1168.92649479, 83.8676043424, + 2.50662827511 }; + double a = (x + 0.5) * log(x + 5.5) - (x + 5.5); + double b = 0; + for(int n = 0; n < 7; ++n) + { + a -= log(x + double(n)); + b += q[n] * pow(x, double(n)); + } + return a + log(b); +} +/////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +namespace cv{ + +class LineSegmentDetectorImpl CV_FINAL : public LineSegmentDetector +{ +public: + +/** + * Create a LineSegmentDetectorImpl object. Specifying scale, number of subdivisions for the image, should the lines be refined and other constants as follows: + * + * @param _refine How should the lines found be refined? + * LSD_REFINE_NONE - No refinement applied. + * LSD_REFINE_STD - Standard refinement is applied. E.g. breaking arches into smaller line approximations. + * LSD_REFINE_ADV - Advanced refinement. Number of false alarms is calculated, + * lines are refined through increase of precision, decrement in size, etc. + * @param _scale The scale of the image that will be used to find the lines. Range (0..1]. + * @param _sigma_scale Sigma for Gaussian filter is computed as sigma = _sigma_scale/_scale. + * @param _quant Bound to the quantization error on the gradient norm. + * @param _ang_th Gradient angle tolerance in degrees. + * @param _log_eps Detection threshold: -log10(NFA) > _log_eps + * @param _density_th Minimal density of aligned region points in rectangle. + * @param _n_bins Number of bins in pseudo-ordering of gradient modulus. + */ + LineSegmentDetectorImpl(LineSegmentDetectorModes _refine = LSD_REFINE_STD, double _scale = 0.8, + double _sigma_scale = 0.6, double _quant = 2.0, double _ang_th = 22.5, + double _log_eps = 0, double _density_th = 0.7, int _n_bins = 1024); + +/** + * Detect lines in the input image. + * + * @param _image A grayscale(CV_8UC1) input image. + * If only a roi needs to be selected, use + * lsd_ptr->detect(image(roi), ..., lines); + * lines += Scalar(roi.x, roi.y, roi.x, roi.y); + * @param _lines Return: A vector of Vec4f elements specifying the beginning and ending point of a line. + * Where Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. + * Returned lines are strictly oriented depending on the gradient. + * @param width Return: Vector of widths of the regions, where the lines are found. E.g. Width of line. + * @param prec Return: Vector of precisions with which the lines are found. + * @param nfa Return: Vector containing number of false alarms in the line region, with precision of 10%. + * The bigger the value, logarithmically better the detection. + * * -1 corresponds to 10 mean false alarms + * * 0 corresponds to 1 mean false alarm + * * 1 corresponds to 0.1 mean false alarms + * This vector will be calculated _only_ when the objects type is REFINE_ADV + */ + void detect(InputArray _image, OutputArray _lines, + OutputArray width = noArray(), OutputArray prec = noArray(), + OutputArray nfa = noArray()) CV_OVERRIDE; + +/** + * Draw lines on the given canvas. + * + * @param image The image, where lines will be drawn. + * Should have the size of the image, where the lines were found + * @param lines The lines that need to be drawn + */ + void drawSegments(InputOutputArray _image, InputArray lines) CV_OVERRIDE; + +/** + * Draw both vectors on the image canvas. Uses blue for lines 1 and red for lines 2. + * + * @param size The size of the image, where lines1 and lines2 were found. + * @param lines1 The first lines that need to be drawn. Color - Blue. + * @param lines2 The second lines that need to be drawn. Color - Red. + * @param image An optional image, where lines will be drawn. + * Should have the size of the image, where the lines were found + * @return The number of mismatching pixels between lines1 and lines2. + */ + int compareSegments(const Size& size, InputArray lines1, InputArray lines2, InputOutputArray _image = noArray()) CV_OVERRIDE; + +private: + Mat image; + Mat scaled_image; + Mat_ angles; // in rads + Mat_ modgrad; + Mat_ used; + + int img_width; + int img_height; + double LOG_NT; + + bool w_needed; + bool p_needed; + bool n_needed; + + const double SCALE; + const LineSegmentDetectorModes doRefine; + const double SIGMA_SCALE; + const double QUANT; + const double ANG_TH; + const double LOG_EPS; + const double DENSITY_TH; + const int N_BINS; + + struct RegionPoint { + int x; + int y; + uchar* used; + double angle; + double modgrad; + }; + + struct normPoint + { + Point2i p; + int norm; + }; + + std::vector ordered_points; + + struct rect + { + double x1, y1, x2, y2; // first and second point of the line segment + double width; // rectangle width + double x, y; // center of the rectangle + double theta; // angle + double dx,dy; // (dx,dy) is vector oriented as the line segment + double prec; // tolerance angle + double p; // probability of a point with angle within 'prec' + }; + + LineSegmentDetectorImpl& operator= (const LineSegmentDetectorImpl&); // to quiet MSVC + +/** + * Detect lines in the whole input image. + * + * @param lines Return: A vector of Vec4f elements specifying the beginning and ending point of a line. + * Where Vec4f is (x1, y1, x2, y2), point 1 is the start, point 2 - end. + * Returned lines are strictly oriented depending on the gradient. + * @param widths Return: Vector of widths of the regions, where the lines are found. E.g. Width of line. + * @param precisions Return: Vector of precisions with which the lines are found. + * @param nfas Return: Vector containing number of false alarms in the line region, with precision of 10%. + * The bigger the value, logarithmically better the detection. + * * -1 corresponds to 10 mean false alarms + * * 0 corresponds to 1 mean false alarm + * * 1 corresponds to 0.1 mean false alarms + */ + void flsd(std::vector& lines, + std::vector& widths, std::vector& precisions, + std::vector& nfas); + +/** + * Finds the angles and the gradients of the image. Generates a list of pseudo ordered points. + * + * @param threshold The minimum value of the angle that is considered defined, otherwise NOTDEF + * @param n_bins The number of bins with which gradients are ordered by, using bucket sort. + * @param ordered_points Return: Vector of coordinate points that are pseudo ordered by magnitude. + * Pixels would be ordered by norm value, up to a precision given by max_grad/n_bins. + */ + void ll_angle(const double& threshold, const unsigned int& n_bins); + +/** + * Grow a region starting from point s with a defined precision, + * returning the containing points size and the angle of the gradients. + * + * @param s Starting point for the region. + * @param reg Return: Vector of points, that are part of the region + * @param reg_angle Return: The mean angle of the region. + * @param prec The precision by which each region angle should be aligned to the mean. + */ + void region_grow(const Point2i& s, std::vector& reg, + double& reg_angle, const double& prec); + +/** + * Finds the bounding rotated rectangle of a region. + * + * @param reg The region of points, from which the rectangle to be constructed from. + * @param reg_angle The mean angle of the region. + * @param prec The precision by which points were found. + * @param p Probability of a point with angle within 'prec'. + * @param rec Return: The generated rectangle. + */ + void region2rect(const std::vector& reg, const double reg_angle, + const double prec, const double p, rect& rec) const; + +/** + * Compute region's angle as the principal inertia axis of the region. + * @return Regions angle. + */ + double get_theta(const std::vector& reg, const double& x, + const double& y, const double& reg_angle, const double& prec) const; + +/** + * An estimation of the angle tolerance is performed by the standard deviation of the angle at points + * near the region's starting point. Then, a new region is grown starting from the same point, but using the + * estimated angle tolerance. If this fails to produce a rectangle with the right density of region points, + * 'reduce_region_radius' is called to try to satisfy this condition. + */ + bool refine(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, const double& density_th); + +/** + * Reduce the region size, by elimination the points far from the starting point, until that leads to + * rectangle with the right density of region points or to discard the region if too small. + */ + bool reduce_region_radius(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, double density, const double& density_th); + +/** + * Try some rectangles variations to improve NFA value. Only if the rectangle is not meaningful (i.e., log_nfa <= log_eps). + * @return The new NFA value. + */ + double rect_improve(rect& rec) const; + +/** + * Calculates the number of correctly aligned points within the rectangle. + * @return The new NFA value. + */ + double rect_nfa(const rect& rec) const; + +/** + * Computes the NFA values based on the total number of points, points that agree. + * n, k, p are the binomial parameters. + * @return The new NFA value. + */ + double nfa(const int& n, const int& k, const double& p) const; + +/** + * Is the point at place 'address' aligned to angle theta, up to precision 'prec'? + * @return Whether the point is aligned. + */ + bool isAligned(int x, int y, const double& theta, const double& prec) const; + +public: + // Compare norm + static inline bool compare_norm( const normPoint& n1, const normPoint& n2 ) + { + return (n1.norm > n2.norm); + } +}; + +///////////////////////////////////////////////////////////////////////////////////////// + +CV_EXPORTS Ptr createLineSegmentDetector( + LineSegmentDetectorModes _refine, double _scale, double _sigma_scale, double _quant, double _ang_th, + double _log_eps, double _density_th, int _n_bins) +{ + return makePtr( + _refine, _scale, _sigma_scale, _quant, _ang_th, + _log_eps, _density_th, _n_bins); +} + +///////////////////////////////////////////////////////////////////////////////////////// + +LineSegmentDetectorImpl::LineSegmentDetectorImpl(LineSegmentDetectorModes _refine, double _scale, double _sigma_scale, double _quant, + double _ang_th, double _log_eps, double _density_th, int _n_bins) + : img_width(0), img_height(0), LOG_NT(0), w_needed(false), p_needed(false), n_needed(false), + SCALE(_scale), doRefine(_refine), SIGMA_SCALE(_sigma_scale), QUANT(_quant), + ANG_TH(_ang_th), LOG_EPS(_log_eps), DENSITY_TH(_density_th), N_BINS(_n_bins) +{ + CV_Assert(_scale > 0 && _sigma_scale > 0 && _quant >= 0 && + _ang_th > 0 && _ang_th < 180 && _density_th >= 0 && _density_th < 1 && + _n_bins > 0); +} + +void LineSegmentDetectorImpl::detect(InputArray _image, OutputArray _lines, + OutputArray _width, OutputArray _prec, OutputArray _nfa) +{ + CV_INSTRUMENT_REGION(); + + image = _image.getMat(); + CV_Assert(!image.empty() && image.type() == CV_8UC1); + + std::vector lines; + std::vector w, p, n; + w_needed = _width.needed(); + p_needed = _prec.needed(); + if (doRefine < LSD_REFINE_ADV) + n_needed = false; + else + n_needed = _nfa.needed(); + + flsd(lines, w, p, n); + + Mat(lines).copyTo(_lines); + if(w_needed) Mat(w).copyTo(_width); + if(p_needed) Mat(p).copyTo(_prec); + if(n_needed) Mat(n).copyTo(_nfa); + + // Clear used structures + ordered_points.clear(); +} + +void LineSegmentDetectorImpl::flsd(std::vector& lines, + std::vector& widths, std::vector& precisions, + std::vector& nfas) +{ + // Angle tolerance + const double prec = CV_PI * ANG_TH / 180; + const double p = ANG_TH / 180; + const double rho = QUANT / sin(prec); // gradient magnitude threshold + + if(SCALE != 1) + { + Mat gaussian_img; + const double sigma = (SCALE < 1)?(SIGMA_SCALE / SCALE):(SIGMA_SCALE); + const double sprec = 3; + const unsigned int h = (unsigned int)(ceil(sigma * sqrt(2 * sprec * log(10.0)))); + Size ksize(1 + 2 * h, 1 + 2 * h); // kernel size + GaussianBlur(image, gaussian_img, ksize, sigma); + // Scale image to needed size + resize(gaussian_img, scaled_image, Size(), SCALE, SCALE, INTER_LINEAR_EXACT); + ll_angle(rho, N_BINS); + } + else + { + scaled_image = image; + ll_angle(rho, N_BINS); + } + + LOG_NT = 5 * (log10(double(img_width)) + log10(double(img_height))) / 2 + log10(11.0); + const size_t min_reg_size = size_t(-LOG_NT/log10(p)); // minimal number of points in region that can give a meaningful event + + // // Initialize region only when needed + // Mat region = Mat::zeros(scaled_image.size(), CV_8UC1); + used = Mat_::zeros(scaled_image.size()); // zeros = NOTUSED + std::vector reg; + + // Search for line segments + for(size_t i = 0, points_size = ordered_points.size(); i < points_size; ++i) + { + const Point2i& point = ordered_points[i].p; + if((used.at(point) == NOTUSED) && (angles.at(point) != NOTDEF)) + { + double reg_angle; + region_grow(ordered_points[i].p, reg, reg_angle, prec); + + // Ignore small regions + if(reg.size() < min_reg_size) { continue; } + + // Construct rectangular approximation for the region + rect rec; + region2rect(reg, reg_angle, prec, p, rec); + + double log_nfa = -1; + if(doRefine > LSD_REFINE_NONE) + { + // At least REFINE_STANDARD lvl. + if(!refine(reg, reg_angle, prec, p, rec, DENSITY_TH)) { continue; } + + if(doRefine >= LSD_REFINE_ADV) + { + // Compute NFA + log_nfa = rect_improve(rec); + if(log_nfa <= LOG_EPS) { continue; } + } + } + // Found new line + + // Add the offset + rec.x1 += 0.5; rec.y1 += 0.5; + rec.x2 += 0.5; rec.y2 += 0.5; + + // scale the result values if a sub-sampling was performed + if(SCALE != 1) + { + rec.x1 /= SCALE; rec.y1 /= SCALE; + rec.x2 /= SCALE; rec.y2 /= SCALE; + rec.width /= SCALE; + } + + //Store the relevant data + lines.push_back(Vec4f(float(rec.x1), float(rec.y1), float(rec.x2), float(rec.y2))); + if(w_needed) widths.push_back(rec.width); + if(p_needed) precisions.push_back(rec.p); + if(n_needed && doRefine >= LSD_REFINE_ADV) nfas.push_back(log_nfa); + } + } +} + +void LineSegmentDetectorImpl::ll_angle(const double& threshold, + const unsigned int& n_bins) +{ + //Initialize data + angles = Mat_(scaled_image.size()); + modgrad = Mat_(scaled_image.size()); + + img_width = scaled_image.cols; + img_height = scaled_image.rows; + + // Undefined the down and right boundaries + angles.row(img_height - 1).setTo(NOTDEF); + angles.col(img_width - 1).setTo(NOTDEF); + + // Computing gradient for remaining pixels + double max_grad = -1; + for(int y = 0; y < img_height - 1; ++y) + { + const uchar* scaled_image_row = scaled_image.ptr(y); + const uchar* next_scaled_image_row = scaled_image.ptr(y+1); + double* angles_row = angles.ptr(y); + double* modgrad_row = modgrad.ptr(y); + for(int x = 0; x < img_width-1; ++x) + { + int DA = next_scaled_image_row[x + 1] - scaled_image_row[x]; + int BC = scaled_image_row[x + 1] - next_scaled_image_row[x]; + int gx = DA + BC; // gradient x component + int gy = DA - BC; // gradient y component + double norm = std::sqrt((gx * gx + gy * gy) / 4.0); // gradient norm + + modgrad_row[x] = norm; // store gradient + + if (norm <= threshold) // norm too small, gradient no defined + { + angles_row[x] = NOTDEF; + } + else + { + angles_row[x] = fastAtan2(float(gx), float(-gy)) * DEG_TO_RADS; // gradient angle computation + if (norm > max_grad) { max_grad = norm; } + } + + } + } + + // Compute histogram of gradient values + double bin_coef = (max_grad > 0) ? double(n_bins - 1) / max_grad : 0; // If all image is smooth, max_grad <= 0 + for(int y = 0; y < img_height - 1; ++y) + { + const double* modgrad_row = modgrad.ptr(y); + for(int x = 0; x < img_width - 1; ++x) + { + normPoint _point; + int i = int(modgrad_row[x] * bin_coef); + _point.p = Point(x, y); + _point.norm = i; + ordered_points.push_back(_point); + } + } + + // Sort + std::sort(ordered_points.begin(), ordered_points.end(), compare_norm); +} + +void LineSegmentDetectorImpl::region_grow(const Point2i& s, std::vector& reg, + double& reg_angle, const double& prec) +{ + reg.clear(); + + // Point to this region + RegionPoint seed; + seed.x = s.x; + seed.y = s.y; + seed.used = &used.at(s); + reg_angle = angles.at(s); + seed.angle = reg_angle; + seed.modgrad = modgrad.at(s); + reg.push_back(seed); + + float sumdx = float(std::cos(reg_angle)); + float sumdy = float(std::sin(reg_angle)); + *seed.used = USED; + + //Try neighboring regions + for (size_t i = 0;i(yy); + const double* angles_row = angles.ptr(yy); + const double* modgrad_row = modgrad.ptr(yy); + for(int xx = xx_min; xx <= xx_max; ++xx) + { + uchar& is_used = used_row[xx]; + if(is_used != USED && + (isAligned(xx, yy, reg_angle, prec))) + { + const double& angle = angles_row[xx]; + // Add point + is_used = USED; + RegionPoint region_point; + region_point.x = xx; + region_point.y = yy; + region_point.used = &is_used; + region_point.modgrad = modgrad_row[xx]; + region_point.angle = angle; + reg.push_back(region_point); + + // Update region's angle + sumdx += cos(float(angle)); + sumdy += sin(float(angle)); + // reg_angle is used in the isAligned, so it needs to be updates? + reg_angle = fastAtan2(sumdy, sumdx) * DEG_TO_RADS; + } + } + } + } +} + +void LineSegmentDetectorImpl::region2rect(const std::vector& reg, + const double reg_angle, const double prec, const double p, rect& rec) const +{ + double x = 0, y = 0, sum = 0; + for(size_t i = 0; i < reg.size(); ++i) + { + const RegionPoint& pnt = reg[i]; + const double& weight = pnt.modgrad; + x += double(pnt.x) * weight; + y += double(pnt.y) * weight; + sum += weight; + } + + // Weighted sum must differ from 0 + CV_Assert(sum > 0); + + x /= sum; + y /= sum; + + double theta = get_theta(reg, x, y, reg_angle, prec); + + // Find length and width + double dx = cos(theta); + double dy = sin(theta); + double l_min = 0, l_max = 0, w_min = 0, w_max = 0; + + for(size_t i = 0; i < reg.size(); ++i) + { + double regdx = double(reg[i].x) - x; + double regdy = double(reg[i].y) - y; + + double l = regdx * dx + regdy * dy; + double w = -regdx * dy + regdy * dx; + + if(l > l_max) l_max = l; + else if(l < l_min) l_min = l; + if(w > w_max) w_max = w; + else if(w < w_min) w_min = w; + } + + // Store values + rec.x1 = x + l_min * dx; + rec.y1 = y + l_min * dy; + rec.x2 = x + l_max * dx; + rec.y2 = y + l_max * dy; + rec.width = w_max - w_min; + rec.x = x; + rec.y = y; + rec.theta = theta; + rec.dx = dx; + rec.dy = dy; + rec.prec = prec; + rec.p = p; + + // Min width of 1 pixel + if(rec.width < 1.0) rec.width = 1.0; +} + +double LineSegmentDetectorImpl::get_theta(const std::vector& reg, const double& x, + const double& y, const double& reg_angle, const double& prec) const +{ + double Ixx = 0.0; + double Iyy = 0.0; + double Ixy = 0.0; + + // Compute inertia matrix + for(size_t i = 0; i < reg.size(); ++i) + { + const double& regx = reg[i].x; + const double& regy = reg[i].y; + const double& weight = reg[i].modgrad; + double dx = regx - x; + double dy = regy - y; + Ixx += dy * dy * weight; + Iyy += dx * dx * weight; + Ixy -= dx * dy * weight; + } + + // Check if inertia matrix is null + CV_Assert(!(double_equal(Ixx, 0) && double_equal(Iyy, 0) && double_equal(Ixy, 0))); + + // Compute smallest eigenvalue + double lambda = 0.5 * (Ixx + Iyy - sqrt((Ixx - Iyy) * (Ixx - Iyy) + 4.0 * Ixy * Ixy)); + + // Compute angle + double theta = (fabs(Ixx)>fabs(Iyy))? + double(fastAtan2(float(lambda - Ixx), float(Ixy))): + double(fastAtan2(float(Ixy), float(lambda - Iyy))); // in degs + theta *= DEG_TO_RADS; + + // Correct angle by 180 deg if necessary + if(angle_diff(theta, reg_angle) > prec) { theta += CV_PI; } + + return theta; +} + +bool LineSegmentDetectorImpl::refine(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, const double& density_th) +{ + double density = double(reg.size()) / (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + + if (density >= density_th) { return true; } + + // Try to reduce angle tolerance + double xc = double(reg[0].x); + double yc = double(reg[0].y); + const double& ang_c = reg[0].angle; + double sum = 0, s_sum = 0; + int n = 0; + + for (size_t i = 0; i < reg.size(); ++i) + { + *(reg[i].used) = NOTUSED; + if (dist(xc, yc, reg[i].x, reg[i].y) < rec.width) + { + const double& angle = reg[i].angle; + double ang_d = angle_diff_signed(angle, ang_c); + sum += ang_d; + s_sum += ang_d * ang_d; + ++n; + } + } + CV_Assert(n > 0); + double mean_angle = sum / double(n); + // 2 * standard deviation + double tau = 2.0 * sqrt((s_sum - 2.0 * mean_angle * sum) / double(n) + mean_angle * mean_angle); + + // Try new region + region_grow(Point(reg[0].x, reg[0].y), reg, reg_angle, tau); + + if (reg.size() < 2) { return false; } + + region2rect(reg, reg_angle, prec, p, rec); + density = double(reg.size()) / (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + + if (density < density_th) + { + return reduce_region_radius(reg, reg_angle, prec, p, rec, density, density_th); + } + else + { + return true; + } +} + +bool LineSegmentDetectorImpl::reduce_region_radius(std::vector& reg, double reg_angle, + const double prec, double p, rect& rec, double density, const double& density_th) +{ + // Compute region's radius + double xc = double(reg[0].x); + double yc = double(reg[0].y); + double radSq1 = distSq(xc, yc, rec.x1, rec.y1); + double radSq2 = distSq(xc, yc, rec.x2, rec.y2); + double radSq = radSq1 > radSq2 ? radSq1 : radSq2; + + while(density < density_th) + { + radSq *= 0.75*0.75; // Reduce region's radius to 75% of its value + // Remove points from the region and update 'used' map + for (size_t i = 0; i < reg.size(); ++i) + { + if(distSq(xc, yc, double(reg[i].x), double(reg[i].y)) > radSq) + { + // Remove point from the region + *(reg[i].used) = NOTUSED; + std::swap(reg[i], reg[reg.size() - 1]); + reg.pop_back(); + --i; // To avoid skipping one point + } + } + + if(reg.size() < 2) { return false; } + + // Re-compute rectangle + region2rect(reg ,reg_angle, prec, p, rec); + + // Re-compute region points density + density = double(reg.size()) / + (dist(rec.x1, rec.y1, rec.x2, rec.y2) * rec.width); + } + + return true; +} + +double LineSegmentDetectorImpl::rect_improve(rect& rec) const +{ + double delta = 0.5; + double delta_2 = delta / 2.0; + + double log_nfa = rect_nfa(rec); + + if(log_nfa > LOG_EPS) return log_nfa; // Good rectangle + + // Try to improve + // Finer precision + rect r = rect(rec); // Copy + for(int n = 0; n < 5; ++n) + { + r.p /= 2; + r.prec = r.p * CV_PI; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + log_nfa = log_nfa_new; + rec = rect(r); + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce width + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce one side of rectangle + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.x1 += -r.dy * delta_2; + r.y1 += r.dx * delta_2; + r.x2 += -r.dy * delta_2; + r.y2 += r.dx * delta_2; + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try to reduce other side of rectangle + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.x1 -= -r.dy * delta_2; + r.y1 -= r.dx * delta_2; + r.x2 -= -r.dy * delta_2; + r.y2 -= r.dx * delta_2; + r.width -= delta; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + if(log_nfa > LOG_EPS) return log_nfa; + + // Try finer precision + r = rect(rec); + for(unsigned int n = 0; n < 5; ++n) + { + if((r.width - delta) >= 0.5) + { + r.p /= 2; + r.prec = r.p * CV_PI; + double log_nfa_new = rect_nfa(r); + if(log_nfa_new > log_nfa) + { + rec = rect(r); + log_nfa = log_nfa_new; + } + } + } + + return log_nfa; +} + +double LineSegmentDetectorImpl::rect_nfa(const rect& rec) const +{ + int total_pts = 0, alg_pts = 0; + double half_width = rec.width / 2.0; + double dyhw = rec.dy * half_width; + double dxhw = rec.dx * half_width; + + edge ordered_x[4]; + edge* min_y = &ordered_x[0]; + edge* max_y = &ordered_x[0]; // Will be used for loop range + + ordered_x[0].p.x = int(rec.x1 - dyhw); ordered_x[0].p.y = int(rec.y1 + dxhw); ordered_x[0].taken = false; + ordered_x[1].p.x = int(rec.x2 - dyhw); ordered_x[1].p.y = int(rec.y2 + dxhw); ordered_x[1].taken = false; + ordered_x[2].p.x = int(rec.x2 + dyhw); ordered_x[2].p.y = int(rec.y2 - dxhw); ordered_x[2].taken = false; + ordered_x[3].p.x = int(rec.x1 + dyhw); ordered_x[3].p.y = int(rec.y1 - dxhw); ordered_x[3].taken = false; + + std::sort(ordered_x, ordered_x + 4, AsmallerB_XoverY); + + // Find min y. And mark as taken. find max y. + for(unsigned int i = 1; i < 4; ++i) + { + if(min_y->p.y > ordered_x[i].p.y) {min_y = &ordered_x[i]; } + if(max_y->p.y < ordered_x[i].p.y) {max_y = &ordered_x[i]; } + } + min_y->taken = true; + + // Find leftmost untaken point; + edge* leftmost = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!leftmost) // if uninitialized + { + leftmost = &ordered_x[i]; + } + else if (leftmost->p.x > ordered_x[i].p.x) + { + leftmost = &ordered_x[i]; + } + } + } + CV_Assert(leftmost != NULL); + leftmost->taken = true; + + // Find rightmost untaken point; + edge* rightmost = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!rightmost) // if uninitialized + { + rightmost = &ordered_x[i]; + } + else if (rightmost->p.x < ordered_x[i].p.x) + { + rightmost = &ordered_x[i]; + } + } + } + CV_Assert(rightmost != NULL); + rightmost->taken = true; + + // Find last untaken point; + edge* tailp = 0; + for(unsigned int i = 0; i < 4; ++i) + { + if(!ordered_x[i].taken) + { + if(!tailp) // if uninitialized + { + tailp = &ordered_x[i]; + } + else if (tailp->p.x > ordered_x[i].p.x) + { + tailp = &ordered_x[i]; + } + } + } + CV_Assert(tailp != NULL); + tailp->taken = true; + + double flstep = (min_y->p.y != leftmost->p.y) ? + (min_y->p.x - leftmost->p.x) / (min_y->p.y - leftmost->p.y) : 0; //first left step + double slstep = (leftmost->p.y != tailp->p.x) ? + (leftmost->p.x - tailp->p.x) / (leftmost->p.y - tailp->p.x) : 0; //second left step + + double frstep = (min_y->p.y != rightmost->p.y) ? + (min_y->p.x - rightmost->p.x) / (min_y->p.y - rightmost->p.y) : 0; //first right step + double srstep = (rightmost->p.y != tailp->p.x) ? + (rightmost->p.x - tailp->p.x) / (rightmost->p.y - tailp->p.x) : 0; //second right step + + double lstep = flstep, rstep = frstep; + + double left_x = min_y->p.x, right_x = min_y->p.x; + + // Loop around all points in the region and count those that are aligned. + int min_iter = min_y->p.y; + int max_iter = max_y->p.y; + for(int y = min_iter; y <= max_iter; ++y) + { + if (y < 0 || y >= img_height) continue; + + for(int x = int(left_x); x <= int(right_x); ++x) + { + if (x < 0 || x >= img_width) continue; + + ++total_pts; + if(isAligned(x, y, rec.theta, rec.prec)) + { + ++alg_pts; + } + } + + if(y >= leftmost->p.y) { lstep = slstep; } + if(y >= rightmost->p.y) { rstep = srstep; } + + left_x += lstep; + right_x += rstep; + } + + return nfa(total_pts, alg_pts, rec.p); +} + +double LineSegmentDetectorImpl::nfa(const int& n, const int& k, const double& p) const +{ + // Trivial cases + if(n == 0 || k == 0) { return -LOG_NT; } + if(n == k) { return -LOG_NT - double(n) * log10(p); } + + double p_term = p / (1 - p); + + double log1term = (double(n) + 1) - log_gamma(double(k) + 1) + - log_gamma(double(n-k) + 1) + + double(k) * log(p) + double(n-k) * log(1.0 - p); + double term = exp(log1term); + + if(double_equal(term, 0)) + { + if(k > n * p) return -log1term / M_LN10 - LOG_NT; + else return -LOG_NT; + } + + // Compute more terms if needed + double bin_tail = term; + double tolerance = 0.1; // an error of 10% in the result is accepted + for(int i = k + 1; i <= n; ++i) + { + double bin_term = double(n - i + 1) / double(i); + double mult_term = bin_term * p_term; + term *= mult_term; + bin_tail += term; + if(bin_term < 1) + { + double err = term * ((1 - pow(mult_term, double(n-i+1))) / (1 - mult_term) - 1); + if(err < tolerance * fabs(-log10(bin_tail) - LOG_NT) * bin_tail) break; + } + + } + return -log10(bin_tail) - LOG_NT; +} + +inline bool LineSegmentDetectorImpl::isAligned(int x, int y, const double& theta, const double& prec) const +{ + if(x < 0 || y < 0 || x >= angles.cols || y >= angles.rows) { return false; } + const double& a = angles.at(y, x); + if(a == NOTDEF) { return false; } + + // It is assumed that 'theta' and 'a' are in the range [-pi,pi] + double n_theta = theta - a; + if(n_theta < 0) { n_theta = -n_theta; } + if(n_theta > M_3_2_PI) + { + n_theta -= M_2__PI; + if(n_theta < 0) n_theta = -n_theta; + } + + return n_theta <= prec; +} + + +void LineSegmentDetectorImpl::drawSegments(InputOutputArray _image, InputArray lines) +{ + CV_INSTRUMENT_REGION(); + + CV_Assert(!_image.empty() && (_image.channels() == 1 || _image.channels() == 3)); + + if (_image.channels() == 1) + { + cvtColor(_image, _image, COLOR_GRAY2BGR); + } + + Mat _lines = lines.getMat(); + const int N = _lines.checkVector(4); + + CV_Assert(_lines.depth() == CV_32F || _lines.depth() == CV_32S); + + // Draw segments + if (_lines.depth() == CV_32F) + { + for (int i = 0; i < N; ++i) + { + const Vec4f& v = _lines.at(i); + const Point2f b(v[0], v[1]); + const Point2f e(v[2], v[3]); + line(_image, b, e, Scalar(0, 0, 255), 1); + } + } + else + { + for (int i = 0; i < N; ++i) + { + const Vec4i& v = _lines.at(i); + const Point2i b(v[0], v[1]); + const Point2i e(v[2], v[3]); + line(_image, b, e, Scalar(0, 0, 255), 1); + } + } +} + + +int LineSegmentDetectorImpl::compareSegments(const Size& size, InputArray lines1, InputArray lines2, InputOutputArray _image) +{ + CV_INSTRUMENT_REGION(); + + Size sz = size; + if (_image.needed() && _image.size() != size) sz = _image.size(); + CV_Assert(!sz.empty()); + + Mat_ I1 = Mat_::zeros(sz); + Mat_ I2 = Mat_::zeros(sz); + + Mat _lines1 = lines1.getMat(); + Mat _lines2 = lines2.getMat(); + const int N1 = _lines1.checkVector(4); + const int N2 = _lines2.checkVector(4); + + CV_Assert(_lines1.depth() == CV_32F || _lines1.depth() == CV_32S); + CV_Assert(_lines2.depth() == CV_32F || _lines2.depth() == CV_32S); + + if (_lines1.depth() == CV_32S) + _lines1.convertTo(_lines1, CV_32F); + if (_lines2.depth() == CV_32S) + _lines2.convertTo(_lines2, CV_32F); + + // Draw segments + for(int i = 0; i < N1; ++i) + { + const Point2f b(_lines1.at(i)[0], _lines1.at(i)[1]); + const Point2f e(_lines1.at(i)[2], _lines1.at(i)[3]); + line(I1, b, e, Scalar::all(255), 1); + } + for(int i = 0; i < N2; ++i) + { + const Point2f b(_lines2.at(i)[0], _lines2.at(i)[1]); + const Point2f e(_lines2.at(i)[2], _lines2.at(i)[3]); + line(I2, b, e, Scalar::all(255), 1); + } + + // Count the pixels that don't agree + Mat Ixor; + bitwise_xor(I1, I2, Ixor); + int N = countNonZero(Ixor); + + if (_image.needed()) + { + CV_Assert(_image.channels() == 3); + Mat img = _image.getMatRef(); + CV_Assert(img.isContinuous() && I1.isContinuous() && I2.isContinuous()); + + for (unsigned int i = 0; i < I1.total(); ++i) + { + uchar i1 = I1.ptr()[i]; + uchar i2 = I2.ptr()[i]; + if (i1 || i2) + { + unsigned int base_idx = i * 3; + if (i1) img.ptr()[base_idx] = 255; + else img.ptr()[base_idx] = 0; + img.ptr()[base_idx + 1] = 0; + if (i2) img.ptr()[base_idx + 2] = 255; + else img.ptr()[base_idx + 2] = 0; + } + } + } + + return N; +} + +} // namespace cv diff --git a/modules/imgproc/src/min_enclosing_triangle.cpp b/modules/imgproc/src/min_enclosing_triangle.cpp index 4853a755d9..2651871412 100644 --- a/modules/imgproc/src/min_enclosing_triangle.cpp +++ b/modules/imgproc/src/min_enclosing_triangle.cpp @@ -129,7 +129,6 @@ static bool areOnTheSameSideOfLine(const cv::Point2f &p1, const cv::Point2f &p2, static double areaOfTriangle(const cv::Point2f &a, const cv::Point2f &b, const cv::Point2f &c); -static void createConvexHull(cv::InputArray points, std::vector &polygon); static double distanceBtwPoints(const cv::Point2f &a, const cv::Point2f &b); @@ -319,29 +318,12 @@ namespace minEnclosingTriangle { static void findMinEnclosingTriangle(cv::InputArray points, CV_OUT cv::OutputArray triangle, CV_OUT double &area) { std::vector resultingTriangle, polygon; - - createConvexHull(points, polygon); + CV_Assert(!points.empty()); + convexHull(points, polygon, true, true); findMinEnclosingTriangle(polygon, resultingTriangle, area); cv::Mat(resultingTriangle).copyTo(triangle); } -//! Create the convex hull of the given set of points -/*! -* @param points The provided set of points -* @param polygon The polygon representing the convex hull of the points -*/ -static void createConvexHull(cv::InputArray points, std::vector &polygon) { - cv::Mat pointsMat = points.getMat(); - std::vector pointsVector; - - CV_Assert((pointsMat.checkVector(2) > 0) && - ((pointsMat.depth() == CV_32F) || (pointsMat.depth() == CV_32S))); - - pointsMat.convertTo(pointsVector, CV_32F); - - convexHull(pointsVector, polygon, true, true); -} - //! Find the minimum enclosing triangle and its area /*! * The overall complexity of the algorithm is theta(n) where "n" represents the number diff --git a/modules/imgproc/src/rotcalipers.cpp b/modules/imgproc/src/rotcalipers.cpp index 527f71a247..e3d81c7e0c 100644 --- a/modules/imgproc/src/rotcalipers.cpp +++ b/modules/imgproc/src/rotcalipers.cpp @@ -88,6 +88,32 @@ enum { CALIPERS_MAXHEIGHT=0, CALIPERS_MINAREARECT=1, CALIPERS_MAXDIST=2 }; // Notes: //F*/ +static void rotate90CCW(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = -in.y; + out.y = in.x; +} + +static void rotate90CW(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = in.y; + out.y = -in.x; +} + +static void rotate180(const cv::Point2f& in, cv::Point2f &out) +{ + out.x = -in.x; + out.y = -in.y; +} + +/* return true if first vector is to the right (clockwise) of the second */ +static bool firstVecIsRight(const cv::Point2f& vec1, const cv::Point2f &vec2) +{ + cv::Point2f tmp; + rotate90CW(vec1, tmp); + return tmp.x * vec2.x + tmp.y * vec2.y < 0; +} + /* we will use usual cartesian coordinates */ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out ) { @@ -100,6 +126,7 @@ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out Point2f* vect = (Point2f*)(inv_vect_length + n); int left = 0, bottom = 0, right = 0, top = 0; int seq[4] = { -1, -1, -1, -1 }; + Point2f rot_vect[4]; /* rotating calipers sides will always have coordinates (a,b) (-b,a) (-a,-b) (b, -a) @@ -179,32 +206,18 @@ static void rotatingCalipers( const Point2f* points, int n, int mode, float* out /* all of edges will be checked while rotating calipers by 90 degrees */ for( k = 0; k < n; k++ ) { - /* sinus of minimal angle */ - /*float sinus;*/ - - /* compute cosine of angle between calipers side and polygon edge */ - /* dp - dot product */ - float dp[4] = { - +base_a * vect[seq[0]].x + base_b * vect[seq[0]].y, - -base_b * vect[seq[1]].x + base_a * vect[seq[1]].y, - -base_a * vect[seq[2]].x - base_b * vect[seq[2]].y, - +base_b * vect[seq[3]].x - base_a * vect[seq[3]].y, - }; - - float maxcos = dp[0] * inv_vect_length[seq[0]]; - /* number of calipers edges, that has minimal angle with edge */ int main_element = 0; - /* choose minimal angle */ - for ( i = 1; i < 4; ++i ) + /* choose minimum angle between calipers side and polygon edge by dot product sign */ + rot_vect[0] = vect[seq[0]]; + rotate90CW(vect[seq[1]], rot_vect[1]); + rotate180(vect[seq[2]], rot_vect[2]); + rotate90CCW(vect[seq[3]], rot_vect[3]); + for (i = 1; i < 4; i++) { - float cosalpha = dp[i] * inv_vect_length[seq[i]]; - if (cosalpha > maxcos) - { + if (firstVecIsRight(rot_vect[i], rot_vect[main_element])) main_element = i; - maxcos = cosalpha; - } } /*rotate calipers*/ diff --git a/modules/imgproc/src/smooth.simd.hpp b/modules/imgproc/src/smooth.simd.hpp index 3a39765c71..62ff31ac94 100644 --- a/modules/imgproc/src/smooth.simd.hpp +++ b/modules/imgproc/src/smooth.simd.hpp @@ -1958,7 +1958,10 @@ public: } else if (kxlen % 2 == 1) { - hlineSmoothFunc = hlineSmoothONa_yzy_a; + if (kx[(kxlen - 1)/ 2] == FT::one()) + hlineSmoothFunc = hlineSmooth1N1; + else + hlineSmoothFunc = hlineSmoothONa_yzy_a; for (int i = 0; i < kxlen / 2; i++) if (!(kx[i] == kx[kxlen - 1 - i])) { diff --git a/modules/imgproc/test/test_color.cpp b/modules/imgproc/test/test_color.cpp index 450cf57923..204203d053 100644 --- a/modules/imgproc/test/test_color.cpp +++ b/modules/imgproc/test/test_color.cpp @@ -3072,20 +3072,34 @@ TEST(ImgProc_RGB2YUV, regression_13668) EXPECT_EQ(res, ref); } -TEST(ImgProc_cvtColorTwoPlane, missing_check_17036) // test can be removed if required feature is implemented +TEST(ImgProc_cvtColorTwoPlane, y_plane_padding_differs_from_uv_plane_padding_17036) { - std::vector y_data(700 * 480); - std::vector uv_data(640 * 240); + RNG &rng = theRNG(); - Mat y_plane_padding(480, 640, CV_8UC1, y_data.data(), 700); // with stride - Mat uv_plane(240, 320, CV_8UC2, uv_data.data()); + std::vector y_reference(640 * 480); + std::vector uv_reference(640 * 240); + std::vector y_padded(700 * 480); + std::vector uv_padded(700 * 240); - Mat result; + Mat y_reference_mat(480, 640, CV_8UC1, y_reference.data()); + Mat uv_reference_mat(240, 320, CV_8UC2, uv_reference.data()); + Mat y_padded_mat(480, 640, CV_8UC1, y_padded.data(), 700); + Mat uv_padded_mat(240, 320, CV_8UC2, uv_padded.data(), 700); - EXPECT_THROW( - cvtColorTwoPlane(y_plane_padding, uv_plane, result, COLOR_YUV2RGB_NV21); - , cv::Exception - ); + rng.fill(y_reference_mat, RNG::UNIFORM, 16, 235 + 1); + rng.fill(uv_reference_mat, RNG::UNIFORM, 16, 240 + 1); + + y_reference_mat.copyTo(y_padded_mat(Rect(0, 0, y_reference_mat.cols, y_reference_mat.rows))); + uv_reference_mat.copyTo(uv_padded_mat(Rect(0, 0, uv_reference_mat.cols, uv_reference_mat.rows))); + + Mat rgb_reference_mat, rgb_y_padded_mat, rgb_uv_padded_mat; + + cvtColorTwoPlane(y_reference_mat, uv_reference_mat, rgb_reference_mat, COLOR_YUV2RGB_NV21); + cvtColorTwoPlane(y_padded_mat, uv_reference_mat, rgb_y_padded_mat, COLOR_YUV2RGB_NV21); + cvtColorTwoPlane(y_reference_mat, uv_padded_mat, rgb_uv_padded_mat, COLOR_YUV2RGB_NV21); + + EXPECT_DOUBLE_EQ(cvtest::norm(rgb_reference_mat, rgb_y_padded_mat, NORM_INF), .0); + EXPECT_DOUBLE_EQ(cvtest::norm(rgb_reference_mat, rgb_uv_padded_mat, NORM_INF), .0); } diff --git a/modules/imgproc/test/test_convhull.cpp b/modules/imgproc/test/test_convhull.cpp index 9c3f060e60..0f379cd02b 100644 --- a/modules/imgproc/test/test_convhull.cpp +++ b/modules/imgproc/test/test_convhull.cpp @@ -2384,5 +2384,78 @@ TEST(Imgproc_minAreaRect, reproducer_18157) EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; } +TEST(Imgproc_minAreaRect, reproducer_19769_lightweight) +{ + const int N = 23; + float pts_[N][2] = { + {1325, 732}, {1248, 808}, {582, 1510}, {586, 1524}, + {595, 1541}, {599, 1547}, {789, 1745}, {829, 1786}, + {997, 1958}, {1116, 2074}, {1207, 2066}, {1216, 2058}, + {1231, 2044}, {1265, 2011}, {2036, 1254}, {2100, 1191}, + {2169, 1123}, {2315, 979}, {2395, 900}, {2438, 787}, + {2434, 782}, {2416, 762}, {2266, 610} + }; + Mat contour(N, 1, CV_32FC2, (void*)pts_); + + RotatedRect rr = cv::minAreaRect(contour); + + EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; +} + +TEST(Imgproc_minAreaRect, reproducer_19769) +{ + const int N = 169; + float pts_[N][2] = { + {1854, 227}, {1850, 228}, {1847, 229}, {1835, 235}, + {1832, 237}, {1829, 239}, {1825, 242}, {1818, 248}, + {1807, 258}, {1759, 306}, {1712, 351}, {1708, 356}, + {1658, 404}, {1655, 408}, {1602, 459}, {1599, 463}, + {1542, 518}, {1477, 582}, {1402, 656}, {1325, 732}, + {1248, 808}, {1161, 894}, {1157, 898}, {1155, 900}, + {1068, 986}, {1060, 995}, {1058, 997}, {957, 1097}, + {956, 1097}, {814, 1238}, {810, 1242}, {805, 1248}, + {610, 1442}, {603, 1450}, {599, 1455}, {596, 1459}, + {594, 1462}, {592, 1465}, {590, 1470}, {588, 1472}, + {586, 1476}, {586, 1478}, {584, 1481}, {583, 1485}, + {582, 1490}, {582, 1510}, {583, 1515}, {584, 1518}, + {585, 1521}, {586, 1524}, {593, 1538}, {595, 1541}, + {597, 1544}, {599, 1547}, {603, 1552}, {609, 1559}, + {623, 1574}, {645, 1597}, {677, 1630}, {713, 1667}, + {753, 1707}, {789, 1744}, {789, 1745}, {829, 1786}, + {871, 1828}, {909, 1867}, {909, 1868}, {950, 1910}, + {953, 1912}, {997, 1958}, {1047, 2009}, {1094, 2056}, + {1105, 2066}, {1110, 2070}, {1113, 2072}, {1116, 2074}, + {1119, 2076}, {1122, 2077}, {1124, 2079}, {1130, 2082}, + {1133, 2083}, {1136, 2084}, {1139, 2085}, {1142, 2086}, + {1148, 2087}, {1166, 2087}, {1170, 2086}, {1174, 2085}, + {1177, 2084}, {1180, 2083}, {1188, 2079}, {1190, 2077}, + {1193, 2076}, {1196, 2074}, {1199, 2072}, {1202, 2070}, + {1207, 2066}, {1216, 2058}, {1231, 2044}, {1265, 2011}, + {1314, 1962}, {1360, 1917}, {1361, 1917}, {1408, 1871}, + {1457, 1822}, {1508, 1773}, {1512, 1768}, {1560, 1722}, + {1617, 1665}, {1671, 1613}, {1730, 1554}, {1784, 1502}, + {1786, 1500}, {1787, 1498}, {1846, 1440}, {1850, 1437}, + {1908, 1380}, {1974, 1314}, {2034, 1256}, {2036, 1254}, + {2100, 1191}, {2169, 1123}, {2242, 1051}, {2315, 979}, + {2395, 900}, {2426, 869}, {2435, 859}, {2438, 855}, + {2440, 852}, {2442, 849}, {2443, 846}, {2445, 844}, + {2446, 842}, {2446, 840}, {2448, 837}, {2449, 834}, + {2450, 829}, {2450, 814}, {2449, 809}, {2448, 806}, + {2447, 803}, {2442, 793}, {2440, 790}, {2438, 787}, + {2434, 782}, {2428, 775}, {2416, 762}, {2411, 758}, + {2342, 688}, {2340, 686}, {2338, 684}, {2266, 610}, + {2260, 605}, {2170, 513}, {2075, 417}, {2073, 415}, + {2069, 412}, {1955, 297}, {1955, 296}, {1913, 254}, + {1904, 246}, {1897, 240}, {1894, 238}, {1891, 236}, + {1888, 234}, {1880, 230}, {1877, 229}, {1874, 228}, + {1870, 227} + }; + Mat contour(N, 1, CV_32FC2, (void*)pts_); + + RotatedRect rr = cv::minAreaRect(contour); + + EXPECT_TRUE(checkMinAreaRect(rr, contour)) << rr.center << " " << rr.size << " " << rr.angle; +} + }} // namespace /* End of file. */ diff --git a/modules/imgproc/test/test_intersection.cpp b/modules/imgproc/test/test_intersection.cpp index 7527dd9a22..c455c439fc 100644 --- a/modules/imgproc/test/test_intersection.cpp +++ b/modules/imgproc/test/test_intersection.cpp @@ -391,4 +391,21 @@ TEST(Imgproc_RotatedRectangleIntersection, regression_18520) } } +TEST(Imgproc_RotatedRectangleIntersection, regression_19824) +{ + RotatedRect r1( + Point2f(246805.033f, 4002326.94f), + Size2f(26.40587f, 6.20026f), + -62.10156f); + RotatedRect r2( + Point2f(246805.122f, 4002326.59f), + Size2f(27.4821f, 8.5361f), + -56.33761f); + + std::vector intersections; + int interType = cv::rotatedRectangleIntersection(r1, r2, intersections); + EXPECT_EQ(INTERSECT_PARTIAL, interType); + EXPECT_LE(intersections.size(), (size_t)7); +} + }} // namespace diff --git a/modules/imgproc/test/test_lsd.cpp b/modules/imgproc/test/test_lsd.cpp new file mode 100644 index 0000000000..43d00b4928 --- /dev/null +++ b/modules/imgproc/test/test_lsd.cpp @@ -0,0 +1,405 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +const Size img_size(640, 480); +const int LSD_TEST_SEED = 0x134679; +const int EPOCHS = 20; + +class LSDBase : public testing::Test +{ +public: + LSDBase() { } + +protected: + Mat test_image; + vector lines; + RNG rng; + int passedtests; + + void GenerateWhiteNoise(Mat& image); + void GenerateConstColor(Mat& image); + void GenerateLines(Mat& image, const unsigned int numLines); + void GenerateRotatedRect(Mat& image); + virtual void SetUp(); +}; + +class Imgproc_LSD_ADV: public LSDBase +{ +public: + Imgproc_LSD_ADV() { } +protected: + +}; + +class Imgproc_LSD_STD: public LSDBase +{ +public: + Imgproc_LSD_STD() { } +protected: + +}; + +class Imgproc_LSD_NONE: public LSDBase +{ +public: + Imgproc_LSD_NONE() { } +protected: + +}; + +class Imgproc_LSD_Common : public LSDBase +{ +public: + Imgproc_LSD_Common() { } +protected: + +}; + +void LSDBase::GenerateWhiteNoise(Mat& image) +{ + image = Mat(img_size, CV_8UC1); + rng.fill(image, RNG::UNIFORM, 0, 256); +} + +void LSDBase::GenerateConstColor(Mat& image) +{ + image = Mat(img_size, CV_8UC1, Scalar::all(rng.uniform(0, 256))); +} + +void LSDBase::GenerateLines(Mat& image, const unsigned int numLines) +{ + image = Mat(img_size, CV_8UC1, Scalar::all(rng.uniform(0, 128))); + + for(unsigned int i = 0; i < numLines; ++i) + { + int y = rng.uniform(10, img_size.width - 10); + Point p1(y, 10); + Point p2(y, img_size.height - 10); + line(image, p1, p2, Scalar(255), 3); + } +} + +void LSDBase::GenerateRotatedRect(Mat& image) +{ + image = Mat::zeros(img_size, CV_8UC1); + + Point center(rng.uniform(img_size.width/4, img_size.width*3/4), + rng.uniform(img_size.height/4, img_size.height*3/4)); + Size rect_size(rng.uniform(img_size.width/8, img_size.width/6), + rng.uniform(img_size.height/8, img_size.height/6)); + float angle = rng.uniform(0.f, 360.f); + + Point2f vertices[4]; + + RotatedRect rRect = RotatedRect(center, rect_size, angle); + + rRect.points(vertices); + for (int i = 0; i < 4; i++) + { + line(image, vertices[i], vertices[(i + 1) % 4], Scalar(255), 3); + } +} + +void LSDBase::SetUp() +{ + lines.clear(); + test_image = Mat(); + rng = RNG(LSD_TEST_SEED); + passedtests = 0; +} + + +TEST_F(Imgproc_LSD_ADV, whiteNoise) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateWhiteNoise(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_ADV); + detector->detect(test_image, lines); + + if(40u >= lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_ADV, constColor) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateConstColor(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_ADV); + detector->detect(test_image, lines); + + if(0u == lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_ADV, lines) +{ + for (int i = 0; i < EPOCHS; ++i) + { + const unsigned int numOfLines = 1; + GenerateLines(test_image, numOfLines); + Ptr detector = createLineSegmentDetector(LSD_REFINE_ADV); + detector->detect(test_image, lines); + + if(numOfLines * 2 == lines.size()) ++passedtests; // * 2 because of Gibbs effect + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_ADV, rotatedRect) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateRotatedRect(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_ADV); + detector->detect(test_image, lines); + + if(2u <= lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_STD, whiteNoise) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateWhiteNoise(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + detector->detect(test_image, lines); + + if(50u >= lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_STD, constColor) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateConstColor(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + detector->detect(test_image, lines); + + if(0u == lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_STD, lines) +{ + for (int i = 0; i < EPOCHS; ++i) + { + const unsigned int numOfLines = 1; + GenerateLines(test_image, numOfLines); + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + detector->detect(test_image, lines); + + if(numOfLines * 2 == lines.size()) ++passedtests; // * 2 because of Gibbs effect + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_STD, rotatedRect) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateRotatedRect(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + detector->detect(test_image, lines); + + if(4u <= lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_NONE, whiteNoise) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateWhiteNoise(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_NONE); + detector->detect(test_image, lines); + + if(50u >= lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_NONE, constColor) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateConstColor(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_NONE); + detector->detect(test_image, lines); + + if(0u == lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_NONE, lines) +{ + for (int i = 0; i < EPOCHS; ++i) + { + const unsigned int numOfLines = 1; + GenerateLines(test_image, numOfLines); + Ptr detector = createLineSegmentDetector(LSD_REFINE_NONE); + detector->detect(test_image, lines); + + if(numOfLines * 2 == lines.size()) ++passedtests; // * 2 because of Gibbs effect + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_NONE, rotatedRect) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateRotatedRect(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_NONE); + detector->detect(test_image, lines); + + if(8u <= lines.size()) ++passedtests; + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_Common, supportsVec4iResult) +{ + for (int i = 0; i < EPOCHS; ++i) + { + GenerateWhiteNoise(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + detector->detect(test_image, lines); + + std::vector linesVec4i; + detector->detect(test_image, linesVec4i); + + if (lines.size() == linesVec4i.size()) + { + bool pass = true; + for (size_t lineIndex = 0; pass && lineIndex < lines.size(); lineIndex++) + { + for (int ch = 0; ch < 4; ch++) + { + if (cv::saturate_cast(lines[lineIndex][ch]) != linesVec4i[lineIndex][ch]) + { + pass = false; + break; + } + } + } + if (pass) + ++passedtests; + } + } + ASSERT_EQ(EPOCHS, passedtests); +} + +TEST_F(Imgproc_LSD_Common, drawSegmentsVec4f) +{ + GenerateConstColor(test_image); + + std::vector linesVec4f; + RNG cr(0); // constant seed for deterministic test + for (int j = 0; j < 10; j++) { + linesVec4f.push_back( + Vec4f(float(cr) * test_image.cols, float(cr) * test_image.rows, float(cr) * test_image.cols, float(cr) * test_image.rows)); + } + + Mat actual = Mat::zeros(test_image.size(), CV_8UC3); + Mat expected = Mat::zeros(test_image.size(), CV_8UC3); + + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + detector->drawSegments(actual, linesVec4f); + + // something should be drawn + ASSERT_EQ(sum(actual == expected) != Scalar::all(0), true); + + for (size_t lineIndex = 0; lineIndex < linesVec4f.size(); lineIndex++) + { + const Vec4f &v = linesVec4f[lineIndex]; + const Point2f b(v[0], v[1]); + const Point2f e(v[2], v[3]); + line(expected, b, e, Scalar(0, 0, 255), 1); + } + + ASSERT_EQ(sum(actual != expected) == Scalar::all(0), true); +} + +TEST_F(Imgproc_LSD_Common, drawSegmentsVec4i) +{ + GenerateConstColor(test_image); + + std::vector linesVec4i; + RNG cr(0); // constant seed for deterministic test + for (int j = 0; j < 10; j++) { + linesVec4i.push_back( + Vec4i(cr(test_image.cols), cr(test_image.rows), cr(test_image.cols), cr(test_image.rows))); + } + + Mat actual = Mat::zeros(test_image.size(), CV_8UC3); + Mat expected = Mat::zeros(test_image.size(), CV_8UC3); + + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + detector->drawSegments(actual, linesVec4i); + + // something should be drawn + ASSERT_EQ(sum(actual == expected) != Scalar::all(0), true); + + for (size_t lineIndex = 0; lineIndex < linesVec4i.size(); lineIndex++) + { + const Vec4f &v = linesVec4i[lineIndex]; + const Point2f b(v[0], v[1]); + const Point2f e(v[2], v[3]); + line(expected, b, e, Scalar(0, 0, 255), 1); + } + + ASSERT_EQ(sum(actual != expected) == Scalar::all(0), true); +} + +TEST_F(Imgproc_LSD_Common, compareSegmentsVec4f) +{ + GenerateConstColor(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + + std::vector lines1, lines2; + lines1.push_back(Vec4f(0, 0, 100, 200)); + lines2.push_back(Vec4f(0, 0, 100, 200)); + int result1 = detector->compareSegments(test_image.size(), lines1, lines2); + + ASSERT_EQ(result1, 0); + + lines2.push_back(Vec4f(100, 100, 110, 100)); + int result2 = detector->compareSegments(test_image.size(), lines1, lines2); + + ASSERT_EQ(result2, 11); +} + +TEST_F(Imgproc_LSD_Common, compareSegmentsVec4i) +{ + GenerateConstColor(test_image); + Ptr detector = createLineSegmentDetector(LSD_REFINE_STD); + + std::vector lines1, lines2; + lines1.push_back(Vec4i(0, 0, 100, 200)); + lines2.push_back(Vec4i(0, 0, 100, 200)); + int result1 = detector->compareSegments(test_image.size(), lines1, lines2); + + ASSERT_EQ(result1, 0); + + lines2.push_back(Vec4i(100, 100, 110, 100)); + int result2 = detector->compareSegments(test_image.size(), lines1, lines2); + + ASSERT_EQ(result2, 11); +} + +}} // namespace diff --git a/modules/imgproc/test/test_smooth_bitexact.cpp b/modules/imgproc/test/test_smooth_bitexact.cpp index 246f1df798..d4ae2af833 100644 --- a/modules/imgproc/test/test_smooth_bitexact.cpp +++ b/modules/imgproc/test/test_smooth_bitexact.cpp @@ -249,4 +249,15 @@ TEST(GaussianBlur_Bitexact, regression_9863) checkGaussianBlur_8Uvs32F(src8u, src32f, 151, 30); } +TEST(GaussianBlur_Bitexact, overflow_20792) +{ + Mat src(128, 128, CV_16UC1, Scalar(255)); + Mat dst; + double sigma = theRNG().uniform(0.0, 0.2); // a peaky kernel + GaussianBlur(src, dst, Size(7, 7), sigma, 0.9); + int count = (int)countNonZero(dst); + int nintyPercent = (int)(src.rows*src.cols * 0.9); + EXPECT_GT(count, nintyPercent); +} + }} // namespace diff --git a/modules/java/android_sdk/android_gradle_lib/build.gradle b/modules/java/android_sdk/android_gradle_lib/build.gradle index 3966c9def6..41cdb9e5e1 100644 --- a/modules/java/android_sdk/android_gradle_lib/build.gradle +++ b/modules/java/android_sdk/android_gradle_lib/build.gradle @@ -36,10 +36,6 @@ android { proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' } } - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_6 - targetCompatibility JavaVersion.VERSION_1_6 - } sourceSets { main { diff --git a/modules/java/android_sdk/build.gradle.in b/modules/java/android_sdk/build.gradle.in index 3ac66b3c79..90f0e88fd7 100644 --- a/modules/java/android_sdk/build.gradle.in +++ b/modules/java/android_sdk/build.gradle.in @@ -58,7 +58,7 @@ // // - Use find_package() in app/CMakeLists.txt: // -// find_package(OpenCV 3.4 REQUIRED java) +// find_package(OpenCV @OPENCV_VERSION_MAJOR@.@OPENCV_VERSION_MINOR@ REQUIRED java) // ... // target_link_libraries(native-lib ${OpenCV_LIBRARIES}) // @@ -128,10 +128,6 @@ android { proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.txt' } } - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_6 - targetCompatibility JavaVersion.VERSION_1_6 - } sourceSets { main { diff --git a/modules/java/generator/gen_java.py b/modules/java/generator/gen_java.py index a5f403b246..3c26a8f5c5 100755 --- a/modules/java/generator/gen_java.py +++ b/modules/java/generator/gen_java.py @@ -267,6 +267,8 @@ class ClassInfo(GeneralInfo): for m in decl[2]: if m.startswith("="): self.jname = m[1:] + if m == '/Simple': + self.smart = False if self.classpath: prefix = self.classpath.replace('.', '_') @@ -457,7 +459,7 @@ class JavaWrapperGenerator(object): def clear(self): self.namespaces = ["cv"] - classinfo_Mat = ClassInfo([ 'class cv.Mat', '', [], [] ], self.namespaces) + classinfo_Mat = ClassInfo([ 'class cv.Mat', '', ['/Simple'], [] ], self.namespaces) self.classes = { "Mat" : classinfo_Mat } self.module = "" self.Module = "" @@ -478,10 +480,15 @@ class JavaWrapperGenerator(object): if name in type_dict and not classinfo.base: logging.warning('duplicated: %s', classinfo) return + if self.isSmartClass(classinfo): + jni_name = "*((*(Ptr<"+classinfo.fullNameCPP()+">*)%(n)s_nativeObj).get())" + else: + jni_name = "(*("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj)" type_dict.setdefault(name, {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "(*("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj)", "jni_type" : "jlong", + "jni_name" : jni_name, + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -489,7 +496,8 @@ class JavaWrapperGenerator(object): type_dict.setdefault(name+'*', {}).update( { "j_type" : classinfo.jname, "jn_type" : "long", "jn_args" : (("__int64", ".nativeObj"),), - "jni_name" : "("+classinfo.fullNameCPP()+"*)%(n)s_nativeObj", "jni_type" : "jlong", + "jni_name" : "&("+jni_name+")", + "jni_type" : "jlong", "suffix" : "J", "j_import" : "org.opencv.%s.%s" % (self.module, classinfo.jname) } @@ -979,7 +987,13 @@ class JavaWrapperGenerator(object): ret = "return env->NewStringUTF(_retval_.c_str());" default = 'return env->NewStringUTF("");' elif self.isWrapped(fi.ctype): # wrapped class: - ret = "return (jlong) new %s(_retval_);" % self.fullTypeNameCPP(fi.ctype) + ret = None + if fi.ctype in self.classes: + ret_ci = self.classes[fi.ctype] + if self.isSmartClass(ret_ci): + ret = "return (jlong)(new Ptr<%(ctype)s>(new %(ctype)s(_retval_)));" % { 'ctype': ret_ci.fullNameCPP() } + if ret is None: + ret = "return (jlong) new %s(_retval_);" % self.fullTypeNameCPP(fi.ctype) elif fi.ctype.startswith('Ptr_'): c_prologue.append("typedef Ptr<%s> %s;" % (self.fullTypeNameCPP(fi.ctype[4:]), fi.ctype)) ret = "return (jlong)(new %(ctype)s(_retval_));" % { 'ctype':fi.ctype } @@ -1221,17 +1235,7 @@ JNIEXPORT void JNICALL Java_org_opencv_%(module)s_%(j_cls)s_delete if ci.smart != None: return ci.smart - # if parents are smart (we hope) then children are! - # if not we believe the class is smart if it has "create" method - ci.smart = False - if ci.base or ci.name == 'Algorithm': - ci.smart = True - else: - for fi in ci.methods: - if fi.name == "create": - ci.smart = True - break - + ci.smart = True # smart class is not properly handled in case of base/derived classes return ci.smart def smartWrap(self, ci, fullname): diff --git a/modules/java/generator/src/cpp/Mat.cpp b/modules/java/generator/src/cpp/Mat.cpp index d59fe4a506..f43a4c6ed5 100644 --- a/modules/java/generator/src/cpp/Mat.cpp +++ b/modules/java/generator/src/cpp/Mat.cpp @@ -1372,6 +1372,32 @@ JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1mul__JJ } +// +// Mat Mat Mat::matMul(Mat m) +// + +JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1matMul__JJ + (JNIEnv* env, jclass, jlong self, jlong m_nativeObj); + +JNIEXPORT jlong JNICALL Java_org_opencv_core_Mat_n_1matMul__JJ + (JNIEnv* env, jclass, jlong self, jlong m_nativeObj) +{ + static const char method_name[] = "Mat::n_1matMul__JJ()"; + try { + LOGD("%s", method_name); + Mat* me = (Mat*) self; //TODO: check for NULL + Mat& m = *((Mat*)m_nativeObj); + Mat _retval_ = (*me) * m; + return (jlong) new Mat(_retval_); + } catch(const std::exception &e) { + throwJavaException(env, &e, method_name); + } catch (...) { + throwJavaException(env, 0, method_name); + } + + return 0; +} + // // static Mat Mat::ones(int rows, int cols, int type) diff --git a/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java b/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java index 54d2736c46..88d75dcdfb 100644 --- a/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java +++ b/modules/java/test/common_test/src/org/opencv/test/utils/ConvertersTest.java @@ -28,9 +28,9 @@ public class ConvertersTest extends OpenCVTestCase { byte value1 = 2; byte value2 = 4; byte value3 = 3; - truth.add(new Byte(value1)); - truth.add(new Byte(value2)); - truth.add(new Byte(value3)); + truth.add(Byte.valueOf(value1)); + truth.add(Byte.valueOf(value2)); + truth.add(Byte.valueOf(value3)); assertEquals(truth, bs); } @@ -248,9 +248,9 @@ public class ConvertersTest extends OpenCVTestCase { byte value1 = 2; byte value2 = 4; byte value3 = 3; - truth.add(new Byte(value1)); - truth.add(new Byte(value2)); - truth.add(new Byte(value3)); + truth.add(Byte.valueOf(value1)); + truth.add(Byte.valueOf(value2)); + truth.add(Byte.valueOf(value3)); assertEquals(truth, bs); } @@ -276,10 +276,10 @@ public class ConvertersTest extends OpenCVTestCase { byte value2 = 2; byte value3 = 3; byte value4 = 4; - bytes.add(new Byte(value1)); - bytes.add(new Byte(value2)); - bytes.add(new Byte(value3)); - bytes.add(new Byte(value4)); + bytes.add(Byte.valueOf(value1)); + bytes.add(Byte.valueOf(value2)); + bytes.add(Byte.valueOf(value3)); + bytes.add(Byte.valueOf(value4)); dst = Converters.vector_char_to_Mat(bytes); truth = new Mat(4, 1, CvType.CV_8SC1); @@ -499,10 +499,10 @@ public class ConvertersTest extends OpenCVTestCase { byte value2 = 2; byte value3 = 3; byte value4 = 4; - bytes.add(new Byte(value1)); - bytes.add(new Byte(value2)); - bytes.add(new Byte(value3)); - bytes.add(new Byte(value4)); + bytes.add(Byte.valueOf(value1)); + bytes.add(Byte.valueOf(value2)); + bytes.add(Byte.valueOf(value3)); + bytes.add(Byte.valueOf(value4)); dst = Converters.vector_uchar_to_Mat(bytes); truth = new Mat(4, 1, CvType.CV_8UC1); diff --git a/modules/js/generator/CMakeLists.txt b/modules/js/generator/CMakeLists.txt index 7a53429651..3d66df154f 100644 --- a/modules/js/generator/CMakeLists.txt +++ b/modules/js/generator/CMakeLists.txt @@ -60,6 +60,7 @@ add_custom_command( ${JS_SOURCE_DIR}/src/core_bindings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/embindgen.py ${CMAKE_CURRENT_SOURCE_DIR}/templates.py + "${OPENCV_JS_WHITELIST_FILE}" ${scripts_hdr_parser} #(not needed - generated by CMake) ${CMAKE_CURRENT_BINARY_DIR}/headers.txt ${opencv_hdrs} diff --git a/modules/js/generator/embindgen.py b/modules/js/generator/embindgen.py index dc7a001df1..2ad6479ba5 100644 --- a/modules/js/generator/embindgen.py +++ b/modules/js/generator/embindgen.py @@ -104,6 +104,10 @@ def makeWhiteList(module_list): return wl white_list = None +namespace_prefix_override = { + 'dnn' : '', + 'aruco' : '', +} # Features to be exported export_enums = False @@ -271,6 +275,8 @@ class FuncVariant(object): class FuncInfo(object): def __init__(self, class_name, name, cname, namespace, isconstructor): + self.name_id = '_'.join([namespace] + ([class_name] if class_name else []) + [name]) # unique id for dict key + self.class_name = class_name self.name = name self.cname = cname @@ -295,9 +301,9 @@ class JSWrapperGenerator(object): self.bindings = [] self.wrapper_funcs = [] - self.classes = {} + self.classes = {} # FIXIT 'classes' should belong to 'namespaces' self.namespaces = {} - self.enums = {} + self.enums = {} # FIXIT 'enums' should belong to 'namespaces' self.parser = hdr_parser.CppHeaderParser() self.class_idx = 0 @@ -419,7 +425,8 @@ class JSWrapperGenerator(object): else: func_map = self.namespaces.setdefault(namespace, Namespace()).funcs - func = func_map.setdefault(name, FuncInfo(class_name, name, cpp_name, namespace, is_constructor)) + fi = FuncInfo(class_name, name, cpp_name, namespace, is_constructor) + func = func_map.setdefault(fi.name_id, fi) variant = FuncVariant(class_name, name, decl, is_constructor, is_class_method, is_const_method, is_virtual_method, is_pure_virtual_method, ref_return, const_return) @@ -430,7 +437,7 @@ class JSWrapperGenerator(object): f.write(buf.getvalue()) f.close() - def gen_function_binding_with_wrapper(self, func, class_info): + def gen_function_binding_with_wrapper(self, func, ns_name, class_info): binding_text = None wrapper_func_text = None @@ -488,8 +495,23 @@ class JSWrapperGenerator(object): # Wrapper function - wrap_func_name = (func.class_name+"_" if class_info != None else "") + func.name.split("::")[-1] + "_wrapper" - js_func_name = func.name + if ns_name != None and ns_name != "cv": + ns_parts = ns_name.split(".") + if ns_parts[0] == "cv": + ns_parts = ns_parts[1:] + ns_part = "_".join(ns_parts) + "_" + ns_id = '_'.join(ns_parts) + ns_prefix = namespace_prefix_override.get(ns_id, ns_id) + if ns_prefix: + ns_prefix = ns_prefix + '_' + else: + ns_prefix = '' + if class_info == None: + js_func_name = ns_prefix + func.name + wrap_func_name = js_func_name + "_wrapper" + else: + wrap_func_name = ns_prefix + func.class_name + "_" + func.name + "_wrapper" + js_func_name = func.name # TODO: Name functions based wrap directives or based on arguments list if index > 0: @@ -740,12 +762,22 @@ class JSWrapperGenerator(object): # step 2: generate bindings # Global functions for ns_name, ns in sorted(self.namespaces.items()): - if ns_name.split('.')[0] != 'cv': + ns_parts = ns_name.split('.') + if ns_parts[0] != 'cv': + print('Ignore namespace: {}'.format(ns_name)) continue - for name, func in sorted(ns.funcs.items()): + else: + ns_parts = ns_parts[1:] + ns_id = '_'.join(ns_parts) + ns_prefix = namespace_prefix_override.get(ns_id, ns_id) + for name_id, func in sorted(ns.funcs.items()): + name = func.name + if ns_prefix: + name = ns_prefix + '_' + name if name in ignore_list: continue if not name in white_list['']: + #print('Not in whitelist: "{}" from ns={}'.format(name, ns_name)) continue ext_cnst = False @@ -769,7 +801,7 @@ class JSWrapperGenerator(object): continue if with_wrapped_functions: - binding, wrapper = self.gen_function_binding_with_wrapper(func, class_info=None) + binding, wrapper = self.gen_function_binding_with_wrapper(func, ns_name, class_info=None) self.bindings += binding self.wrapper_funcs += wrapper else: @@ -802,7 +834,7 @@ class JSWrapperGenerator(object): class_bindings.append(constructor_template.substitute(signature=', '.join(args))) else: if with_wrapped_functions and (len(method.variants) > 1 or len(method.variants[0].args)>0 or "String" in method.variants[0].rettype): - binding, wrapper = self.gen_function_binding_with_wrapper(method, class_info=class_info) + binding, wrapper = self.gen_function_binding_with_wrapper(method, None, class_info=class_info) self.wrapper_funcs = self.wrapper_funcs + wrapper class_bindings = class_bindings + binding else: diff --git a/modules/js/src/make_umd.py b/modules/js/src/make_umd.py index bed6ee9bcc..1096a8eb31 100644 --- a/modules/js/src/make_umd.py +++ b/modules/js/src/make_umd.py @@ -95,7 +95,7 @@ def make_umd(opencvjs, cvjs): root.cv = factory(); } else if (typeof importScripts === 'function') { // Web worker - root.cv = factory; + root.cv = factory(); } else { // Other shells, e.g. d8 root.cv = factory(); diff --git a/modules/ml/misc/java/test/MLTest.java b/modules/ml/misc/java/test/MLTest.java index 2b08543a84..504805dffa 100644 --- a/modules/ml/misc/java/test/MLTest.java +++ b/modules/ml/misc/java/test/MLTest.java @@ -36,7 +36,7 @@ public class MLTest extends OpenCVTestCase { String filename = OpenCVTestRunner.getTempFileName("yml"); saved.save(filename); SVM loaded = SVM.load(filename); - assertTrue(saved.isTrained()); + assertTrue(loaded.isTrained()); } } diff --git a/modules/objdetect/CMakeLists.txt b/modules/objdetect/CMakeLists.txt index 8f31d097eb..411386fd7d 100644 --- a/modules/objdetect/CMakeLists.txt +++ b/modules/objdetect/CMakeLists.txt @@ -1,5 +1,5 @@ set(the_description "Object Detection") -ocv_define_module(objdetect opencv_core opencv_imgproc opencv_3d WRAP java objc python js) +ocv_define_module(objdetect opencv_core opencv_imgproc opencv_3d opencv_dnn WRAP java objc python js) if(HAVE_QUIRC) get_property(QUIRC_INCLUDE GLOBAL PROPERTY QUIRC_INCLUDE_DIR) diff --git a/modules/objdetect/include/opencv2/objdetect.hpp b/modules/objdetect/include/opencv2/objdetect.hpp index eaee1290ce..59dca7399b 100644 --- a/modules/objdetect/include/opencv2/objdetect.hpp +++ b/modules/objdetect/include/opencv2/objdetect.hpp @@ -768,5 +768,6 @@ protected: } #include "opencv2/objdetect/detection_based_tracker.hpp" +#include "opencv2/objdetect/face.hpp" #endif diff --git a/modules/objdetect/include/opencv2/objdetect/face.hpp b/modules/objdetect/include/opencv2/objdetect/face.hpp new file mode 100644 index 0000000000..f2429c5f31 --- /dev/null +++ b/modules/objdetect/include/opencv2/objdetect/face.hpp @@ -0,0 +1,125 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_OBJDETECT_FACE_HPP +#define OPENCV_OBJDETECT_FACE_HPP + +#include + +/** @defgroup dnn_face DNN-based face detection and recognition + */ + +namespace cv +{ + +/** @brief DNN-based face detector, model download link: https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx. + */ +class CV_EXPORTS_W FaceDetectorYN +{ +public: + virtual ~FaceDetectorYN() {}; + + /** @brief Set the size for the network input, which overwrites the input size of creating model. Call this method when the size of input image does not match the input size when creating model + * + * @param input_size the size of the input image + */ + CV_WRAP virtual void setInputSize(const Size& input_size) = 0; + + CV_WRAP virtual Size getInputSize() = 0; + + /** @brief Set the score threshold to filter out bounding boxes of score less than the given value + * + * @param score_threshold threshold for filtering out bounding boxes + */ + CV_WRAP virtual void setScoreThreshold(float score_threshold) = 0; + + CV_WRAP virtual float getScoreThreshold() = 0; + + /** @brief Set the Non-maximum-suppression threshold to suppress bounding boxes that have IoU greater than the given value + * + * @param nms_threshold threshold for NMS operation + */ + CV_WRAP virtual void setNMSThreshold(float nms_threshold) = 0; + + CV_WRAP virtual float getNMSThreshold() = 0; + + /** @brief Set the number of bounding boxes preserved before NMS + * + * @param top_k the number of bounding boxes to preserve from top rank based on score + */ + CV_WRAP virtual void setTopK(int top_k) = 0; + + CV_WRAP virtual int getTopK() = 0; + + /** @brief A simple interface to detect face from given image + * + * @param image an image to detect + * @param faces detection results stored in a cv::Mat + */ + CV_WRAP virtual int detect(InputArray image, OutputArray faces) = 0; + + /** @brief Creates an instance of this class with given parameters + * + * @param model the path to the requested model + * @param config the path to the config file for compability, which is not requested for ONNX models + * @param input_size the size of the input image + * @param score_threshold the threshold to filter out bounding boxes of score smaller than the given value + * @param nms_threshold the threshold to suppress bounding boxes of IoU bigger than the given value + * @param top_k keep top K bboxes before NMS + * @param backend_id the id of backend + * @param target_id the id of target device + */ + CV_WRAP static Ptr create(const String& model, + const String& config, + const Size& input_size, + float score_threshold = 0.9f, + float nms_threshold = 0.3f, + int top_k = 5000, + int backend_id = 0, + int target_id = 0); +}; + +/** @brief DNN-based face recognizer, model download link: https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view. + */ +class CV_EXPORTS_W FaceRecognizerSF +{ +public: + virtual ~FaceRecognizerSF() {}; + + /** @brief Definition of distance used for calculating the distance between two face features + */ + enum DisType { FR_COSINE=0, FR_NORM_L2=1 }; + + /** @brief Aligning image to put face on the standard position + * @param src_img input image + * @param face_box the detection result used for indicate face in input image + * @param aligned_img output aligned image + */ + CV_WRAP virtual void alignCrop(InputArray src_img, InputArray face_box, OutputArray aligned_img) const = 0; + + /** @brief Extracting face feature from aligned image + * @param aligned_img input aligned image + * @param face_feature output face feature + */ + CV_WRAP virtual void feature(InputArray aligned_img, OutputArray face_feature) = 0; + + /** @brief Calculating the distance between two face features + * @param _face_feature1 the first input feature + * @param _face_feature2 the second input feature of the same size and the same type as _face_feature1 + * @param dis_type defining the similarity with optional values "FR_OSINE" or "FR_NORM_L2" + */ + CV_WRAP virtual double match(InputArray _face_feature1, InputArray _face_feature2, int dis_type = FaceRecognizerSF::FR_COSINE) const = 0; + + /** @brief Creates an instance of this class with given parameters + * @param model the path of the onnx model used for face recognition + * @param config the path to the config file for compability, which is not requested for ONNX models + * @param backend_id the id of backend + * @param target_id the id of target device + */ + CV_WRAP static Ptr create(const String& model, const String& config, int backend_id = 0, int target_id = 0); +}; + +} // namespace cv + +#endif diff --git a/modules/objdetect/src/face_detect.cpp b/modules/objdetect/src/face_detect.cpp new file mode 100644 index 0000000000..4095745b7e --- /dev/null +++ b/modules/objdetect/src/face_detect.cpp @@ -0,0 +1,288 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "opencv2/imgproc.hpp" +#include "opencv2/core.hpp" +#include "opencv2/dnn.hpp" + +#include + +namespace cv +{ + +class FaceDetectorYNImpl : public FaceDetectorYN +{ +public: + FaceDetectorYNImpl(const String& model, + const String& config, + const Size& input_size, + float score_threshold, + float nms_threshold, + int top_k, + int backend_id, + int target_id) + { + net = dnn::readNet(model, config); + CV_Assert(!net.empty()); + + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + + inputW = input_size.width; + inputH = input_size.height; + + scoreThreshold = score_threshold; + nmsThreshold = nms_threshold; + topK = top_k; + + generatePriors(); + } + + void setInputSize(const Size& input_size) override + { + inputW = input_size.width; + inputH = input_size.height; + generatePriors(); + } + + Size getInputSize() override + { + Size input_size; + input_size.width = inputW; + input_size.height = inputH; + return input_size; + } + + void setScoreThreshold(float score_threshold) override + { + scoreThreshold = score_threshold; + } + + float getScoreThreshold() override + { + return scoreThreshold; + } + + void setNMSThreshold(float nms_threshold) override + { + nmsThreshold = nms_threshold; + } + + float getNMSThreshold() override + { + return nmsThreshold; + } + + void setTopK(int top_k) override + { + topK = top_k; + } + + int getTopK() override + { + return topK; + } + + int detect(InputArray input_image, OutputArray faces) override + { + // TODO: more checkings should be done? + if (input_image.empty()) + { + return 0; + } + CV_CheckEQ(input_image.size(), Size(inputW, inputH), "Size does not match. Call setInputSize(size) if input size does not match the preset size"); + + // Build blob from input image + Mat input_blob = dnn::blobFromImage(input_image); + + // Forward + std::vector output_names = { "loc", "conf", "iou" }; + std::vector output_blobs; + net.setInput(input_blob); + net.forward(output_blobs, output_names); + + // Post process + Mat results = postProcess(output_blobs); + results.convertTo(faces, CV_32FC1); + return 1; + } +private: + void generatePriors() + { + // Calculate shapes of different scales according to the shape of input image + Size feature_map_2nd = { + int(int((inputW+1)/2)/2), int(int((inputH+1)/2)/2) + }; + Size feature_map_3rd = { + int(feature_map_2nd.width/2), int(feature_map_2nd.height/2) + }; + Size feature_map_4th = { + int(feature_map_3rd.width/2), int(feature_map_3rd.height/2) + }; + Size feature_map_5th = { + int(feature_map_4th.width/2), int(feature_map_4th.height/2) + }; + Size feature_map_6th = { + int(feature_map_5th.width/2), int(feature_map_5th.height/2) + }; + + std::vector feature_map_sizes; + feature_map_sizes.push_back(feature_map_3rd); + feature_map_sizes.push_back(feature_map_4th); + feature_map_sizes.push_back(feature_map_5th); + feature_map_sizes.push_back(feature_map_6th); + + // Fixed params for generating priors + const std::vector> min_sizes = { + {10.0f, 16.0f, 24.0f}, + {32.0f, 48.0f}, + {64.0f, 96.0f}, + {128.0f, 192.0f, 256.0f} + }; + const std::vector steps = { 8, 16, 32, 64 }; + + // Generate priors + priors.clear(); + for (size_t i = 0; i < feature_map_sizes.size(); ++i) + { + Size feature_map_size = feature_map_sizes[i]; + std::vector min_size = min_sizes[i]; + + for (int _h = 0; _h < feature_map_size.height; ++_h) + { + for (int _w = 0; _w < feature_map_size.width; ++_w) + { + for (size_t j = 0; j < min_size.size(); ++j) + { + float s_kx = min_size[j] / inputW; + float s_ky = min_size[j] / inputH; + + float cx = (_w + 0.5f) * steps[i] / inputW; + float cy = (_h + 0.5f) * steps[i] / inputH; + + Rect2f prior = { cx, cy, s_kx, s_ky }; + priors.push_back(prior); + } + } + } + } + } + + Mat postProcess(const std::vector& output_blobs) + { + // Extract from output_blobs + Mat loc = output_blobs[0]; + Mat conf = output_blobs[1]; + Mat iou = output_blobs[2]; + + // Decode from deltas and priors + const std::vector variance = {0.1f, 0.2f}; + float* loc_v = (float*)(loc.data); + float* conf_v = (float*)(conf.data); + float* iou_v = (float*)(iou.data); + Mat faces; + // (tl_x, tl_y, w, h, re_x, re_y, le_x, le_y, nt_x, nt_y, rcm_x, rcm_y, lcm_x, lcm_y, score) + // 'tl': top left point of the bounding box + // 're': right eye, 'le': left eye + // 'nt': nose tip + // 'rcm': right corner of mouth, 'lcm': left corner of mouth + Mat face(1, 15, CV_32FC1); + for (size_t i = 0; i < priors.size(); ++i) { + // Get score + float clsScore = conf_v[i*2+1]; + float iouScore = iou_v[i]; + // Clamp + if (iouScore < 0.f) { + iouScore = 0.f; + } + else if (iouScore > 1.f) { + iouScore = 1.f; + } + float score = std::sqrt(clsScore * iouScore); + face.at(0, 14) = score; + + // Get bounding box + float cx = (priors[i].x + loc_v[i*14+0] * variance[0] * priors[i].width) * inputW; + float cy = (priors[i].y + loc_v[i*14+1] * variance[0] * priors[i].height) * inputH; + float w = priors[i].width * exp(loc_v[i*14+2] * variance[0]) * inputW; + float h = priors[i].height * exp(loc_v[i*14+3] * variance[1]) * inputH; + float x1 = cx - w / 2; + float y1 = cy - h / 2; + face.at(0, 0) = x1; + face.at(0, 1) = y1; + face.at(0, 2) = w; + face.at(0, 3) = h; + + // Get landmarks + face.at(0, 4) = (priors[i].x + loc_v[i*14+ 4] * variance[0] * priors[i].width) * inputW; // right eye, x + face.at(0, 5) = (priors[i].y + loc_v[i*14+ 5] * variance[0] * priors[i].height) * inputH; // right eye, y + face.at(0, 6) = (priors[i].x + loc_v[i*14+ 6] * variance[0] * priors[i].width) * inputW; // left eye, x + face.at(0, 7) = (priors[i].y + loc_v[i*14+ 7] * variance[0] * priors[i].height) * inputH; // left eye, y + face.at(0, 8) = (priors[i].x + loc_v[i*14+ 8] * variance[0] * priors[i].width) * inputW; // nose tip, x + face.at(0, 9) = (priors[i].y + loc_v[i*14+ 9] * variance[0] * priors[i].height) * inputH; // nose tip, y + face.at(0, 10) = (priors[i].x + loc_v[i*14+10] * variance[0] * priors[i].width) * inputW; // right corner of mouth, x + face.at(0, 11) = (priors[i].y + loc_v[i*14+11] * variance[0] * priors[i].height) * inputH; // right corner of mouth, y + face.at(0, 12) = (priors[i].x + loc_v[i*14+12] * variance[0] * priors[i].width) * inputW; // left corner of mouth, x + face.at(0, 13) = (priors[i].y + loc_v[i*14+13] * variance[0] * priors[i].height) * inputH; // left corner of mouth, y + + faces.push_back(face); + } + + if (faces.rows > 1) + { + // Retrieve boxes and scores + std::vector faceBoxes; + std::vector faceScores; + for (int rIdx = 0; rIdx < faces.rows; rIdx++) + { + faceBoxes.push_back(Rect2i(int(faces.at(rIdx, 0)), + int(faces.at(rIdx, 1)), + int(faces.at(rIdx, 2)), + int(faces.at(rIdx, 3)))); + faceScores.push_back(faces.at(rIdx, 14)); + } + + std::vector keepIdx; + dnn::NMSBoxes(faceBoxes, faceScores, scoreThreshold, nmsThreshold, keepIdx, 1.f, topK); + + // Get NMS results + Mat nms_faces; + for (int idx: keepIdx) + { + nms_faces.push_back(faces.row(idx)); + } + return nms_faces; + } + else + { + return faces; + } + } +private: + dnn::Net net; + + int inputW; + int inputH; + float scoreThreshold; + float nmsThreshold; + int topK; + + std::vector priors; +}; + +Ptr FaceDetectorYN::create(const String& model, + const String& config, + const Size& input_size, + const float score_threshold, + const float nms_threshold, + const int top_k, + const int backend_id, + const int target_id) +{ + return makePtr(model, config, input_size, score_threshold, nms_threshold, top_k, backend_id, target_id); +} + +} // namespace cv diff --git a/modules/objdetect/src/face_recognize.cpp b/modules/objdetect/src/face_recognize.cpp new file mode 100644 index 0000000000..6550a13b4b --- /dev/null +++ b/modules/objdetect/src/face_recognize.cpp @@ -0,0 +1,182 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "precomp.hpp" + +#include "opencv2/dnn.hpp" + +#include + +namespace cv +{ + +class FaceRecognizerSFImpl : public FaceRecognizerSF +{ +public: + FaceRecognizerSFImpl(const String& model, const String& config, int backend_id, int target_id) + { + net = dnn::readNet(model, config); + CV_Assert(!net.empty()); + + net.setPreferableBackend(backend_id); + net.setPreferableTarget(target_id); + }; + void alignCrop(InputArray _src_img, InputArray _face_mat, OutputArray _aligned_img) const override + { + Mat face_mat = _face_mat.getMat(); + float src_point[5][2]; + for (int row = 0; row < 5; ++row) + { + for(int col = 0; col < 2; ++col) + { + src_point[row][col] = face_mat.at(0, row*2+col+4); + } + } + Mat warp_mat = getSimilarityTransformMatrix(src_point); + warpAffine(_src_img, _aligned_img, warp_mat, Size(112, 112), INTER_LINEAR); + }; + void feature(InputArray _aligned_img, OutputArray _face_feature) override + { + Mat inputBolb = dnn::blobFromImage(_aligned_img, 1, Size(112, 112), Scalar(0, 0, 0), true, false); + net.setInput(inputBolb); + net.forward(_face_feature); + }; + double match(InputArray _face_feature1, InputArray _face_feature2, int dis_type) const override + { + Mat face_feature1 = _face_feature1.getMat(), face_feature2 = _face_feature2.getMat(); + face_feature1 /= norm(face_feature1); + face_feature2 /= norm(face_feature2); + + if(dis_type == DisType::FR_COSINE){ + return sum(face_feature1.mul(face_feature2))[0]; + }else if(dis_type == DisType::FR_NORM_L2){ + return norm(face_feature1, face_feature2); + }else{ + throw std::invalid_argument("invalid parameter " + std::to_string(dis_type)); + } + + }; + +private: + Mat getSimilarityTransformMatrix(float src[5][2]) const { + float dst[5][2] = { {38.2946f, 51.6963f}, {73.5318f, 51.5014f}, {56.0252f, 71.7366f}, {41.5493f, 92.3655f}, {70.7299f, 92.2041f} }; + float avg0 = (src[0][0] + src[1][0] + src[2][0] + src[3][0] + src[4][0]) / 5; + float avg1 = (src[0][1] + src[1][1] + src[2][1] + src[3][1] + src[4][1]) / 5; + //Compute mean of src and dst. + float src_mean[2] = { avg0, avg1 }; + float dst_mean[2] = { 56.0262f, 71.9008f }; + //Subtract mean from src and dst. + float src_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + src_demean[j][i] = src[j][i] - src_mean[i]; + } + } + float dst_demean[5][2]; + for (int i = 0; i < 2; i++) + { + for (int j = 0; j < 5; j++) + { + dst_demean[j][i] = dst[j][i] - dst_mean[i]; + } + } + double A00 = 0.0, A01 = 0.0, A10 = 0.0, A11 = 0.0; + for (int i = 0; i < 5; i++) + A00 += dst_demean[i][0] * src_demean[i][0]; + A00 = A00 / 5; + for (int i = 0; i < 5; i++) + A01 += dst_demean[i][0] * src_demean[i][1]; + A01 = A01 / 5; + for (int i = 0; i < 5; i++) + A10 += dst_demean[i][1] * src_demean[i][0]; + A10 = A10 / 5; + for (int i = 0; i < 5; i++) + A11 += dst_demean[i][1] * src_demean[i][1]; + A11 = A11 / 5; + Mat A = (Mat_(2, 2) << A00, A01, A10, A11); + double d[2] = { 1.0, 1.0 }; + double detA = A00 * A11 - A01 * A10; + if (detA < 0) + d[1] = -1; + double T[3][3] = { {1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0} }; + Mat s, u, vt, v; + SVD::compute(A, s, u, vt); + double smax = s.ptr(0)[0]>s.ptr(1)[0] ? s.ptr(0)[0] : s.ptr(1)[0]; + double tol = smax * 2 * FLT_MIN; + int rank = 0; + if (s.ptr(0)[0]>tol) + rank += 1; + if (s.ptr(1)[0]>tol) + rank += 1; + double arr_u[2][2] = { {u.ptr(0)[0], u.ptr(0)[1]}, {u.ptr(1)[0], u.ptr(1)[1]} }; + double arr_vt[2][2] = { {vt.ptr(0)[0], vt.ptr(0)[1]}, {vt.ptr(1)[0], vt.ptr(1)[1]} }; + double det_u = arr_u[0][0] * arr_u[1][1] - arr_u[0][1] * arr_u[1][0]; + double det_vt = arr_vt[0][0] * arr_vt[1][1] - arr_vt[0][1] * arr_vt[1][0]; + if (rank == 1) + { + if ((det_u*det_vt) > 0) + { + Mat uvt = u*vt; + T[0][0] = uvt.ptr(0)[0]; + T[0][1] = uvt.ptr(0)[1]; + T[1][0] = uvt.ptr(1)[0]; + T[1][1] = uvt.ptr(1)[1]; + } + else + { + double temp = d[1]; + d[1] = -1; + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + d[1] = temp; + } + } + else + { + Mat D = (Mat_(2, 2) << d[0], 0.0, 0.0, d[1]); + Mat Dvt = D*vt; + Mat uDvt = u*Dvt; + T[0][0] = uDvt.ptr(0)[0]; + T[0][1] = uDvt.ptr(0)[1]; + T[1][0] = uDvt.ptr(1)[0]; + T[1][1] = uDvt.ptr(1)[1]; + } + double var1 = 0.0; + for (int i = 0; i < 5; i++) + var1 += src_demean[i][0] * src_demean[i][0]; + var1 = var1 / 5; + double var2 = 0.0; + for (int i = 0; i < 5; i++) + var2 += src_demean[i][1] * src_demean[i][1]; + var2 = var2 / 5; + double scale = 1.0 / (var1 + var2)* (s.ptr(0)[0] * d[0] + s.ptr(1)[0] * d[1]); + double TS[2]; + TS[0] = T[0][0] * src_mean[0] + T[0][1] * src_mean[1]; + TS[1] = T[1][0] * src_mean[0] + T[1][1] * src_mean[1]; + T[0][2] = dst_mean[0] - scale*TS[0]; + T[1][2] = dst_mean[1] - scale*TS[1]; + T[0][0] *= scale; + T[0][1] *= scale; + T[1][0] *= scale; + T[1][1] *= scale; + Mat transform_mat = (Mat_(2, 3) << T[0][0], T[0][1], T[0][2], T[1][0], T[1][1], T[1][2]); + return transform_mat; + } +private: + dnn::Net net; +}; + +Ptr FaceRecognizerSF::create(const String& model, const String& config, int backend_id, int target_id) +{ + return makePtr(model, config, backend_id, target_id); +} + +} // namespace cv diff --git a/modules/objdetect/test/test_face.cpp b/modules/objdetect/test/test_face.cpp new file mode 100644 index 0000000000..2e944c50df --- /dev/null +++ b/modules/objdetect/test/test_face.cpp @@ -0,0 +1,219 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +namespace opencv_test { namespace { + +// label format: +// image_name +// num_face +// face_1 +// face_.. +// face_num +std::map blobFromTXT(const std::string& path, int numCoords) +{ + std::ifstream ifs(path.c_str()); + CV_Assert(ifs.is_open()); + + std::map gt; + + Mat faces; + int faceNum = -1; + int faceCount = 0; + for (std::string line, key; getline(ifs, line); ) + { + std::istringstream iss(line); + if (line.find(".png") != std::string::npos) + { + // Get filename + iss >> key; + } + else if (line.find(" ") == std::string::npos) + { + // Get the number of faces + iss >> faceNum; + } + else + { + // Get faces + Mat face(1, numCoords, CV_32FC1); + for (int j = 0; j < numCoords; j++) + { + iss >> face.at(0, j); + } + faces.push_back(face); + faceCount++; + } + + if (faceCount == faceNum) + { + // Store faces + gt[key] = faces; + + faces.release(); + faceNum = -1; + faceCount = 0; + } + } + + return gt; +} + +TEST(Objdetect_face_detection, regression) +{ + // Pre-set params + float scoreThreshold = 0.7f; + float matchThreshold = 0.9f; + float l2disThreshold = 5.0f; + int numLM = 5; + int numCoords = 4 + 2 * numLM; + + // Load ground truth labels + std::map gt = blobFromTXT(findDataFile("dnn_face/detection/cascades_labels.txt"), numCoords); + // for (auto item: gt) + // { + // std::cout << item.first << " " << item.second.size() << std::endl; + // } + + // Initialize detector + std::string model = findDataFile("dnn/onnx/models/yunet-202109.onnx", false); + Ptr faceDetector = FaceDetectorYN::create(model, "", Size(300, 300)); + faceDetector->setScoreThreshold(0.7f); + + // Detect and match + for (auto item: gt) + { + std::string imagePath = findDataFile("cascadeandhog/images/" + item.first); + Mat image = imread(imagePath); + + // Set input size + faceDetector->setInputSize(image.size()); + + // Run detection + Mat faces; + faceDetector->detect(image, faces); + // std::cout << item.first << " " << item.second.rows << " " << faces.rows << std::endl; + + // Match bboxes and landmarks + std::vector matchedItem(item.second.rows, false); + for (int i = 0; i < faces.rows; i++) + { + if (faces.at(i, numCoords) < scoreThreshold) + continue; + + bool boxMatched = false; + std::vector lmMatched(numLM, false); + cv::Rect2f resBox(faces.at(i, 0), faces.at(i, 1), faces.at(i, 2), faces.at(i, 3)); + for (int j = 0; j < item.second.rows && !boxMatched; j++) + { + if (matchedItem[j]) + continue; + + // Retrieve bbox and compare IoU + cv::Rect2f gtBox(item.second.at(j, 0), item.second.at(j, 1), item.second.at(j, 2), item.second.at(j, 3)); + double interArea = (resBox & gtBox).area(); + double iou = interArea / (resBox.area() + gtBox.area() - interArea); + if (iou >= matchThreshold) + { + boxMatched = true; + matchedItem[j] = true; + } + + // Match landmarks if bbox is matched + if (!boxMatched) + continue; + for (int lmIdx = 0; lmIdx < numLM; lmIdx++) + { + float gtX = item.second.at(j, 4 + 2 * lmIdx); + float gtY = item.second.at(j, 4 + 2 * lmIdx + 1); + float resX = faces.at(i, 4 + 2 * lmIdx); + float resY = faces.at(i, 4 + 2 * lmIdx + 1); + float l2dis = cv::sqrt((gtX - resX) * (gtX - resX) + (gtY - resY) * (gtY - resY)); + + if (l2dis <= l2disThreshold) + { + lmMatched[lmIdx] = true; + } + } + } + EXPECT_TRUE(boxMatched) << "In image " << item.first << ", cannot match resBox " << resBox << " with any ground truth."; + if (boxMatched) + { + EXPECT_TRUE(std::all_of(lmMatched.begin(), lmMatched.end(), [](bool v) { return v; })) << "In image " << item.first << ", resBox " << resBox << " matched but its landmarks failed to match."; + } + } + } +} + +TEST(Objdetect_face_recognition, regression) +{ + // Pre-set params + float score_thresh = 0.9f; + float nms_thresh = 0.3f; + double cosine_similar_thresh = 0.363; + double l2norm_similar_thresh = 1.128; + + // Load ground truth labels + std::ifstream ifs(findDataFile("dnn_face/recognition/cascades_label.txt").c_str()); + CV_Assert(ifs.is_open()); + + std::set fSet; + std::map featureMap; + std::map, int> gtMap; + + + for (std::string line, key; getline(ifs, line);) + { + std::string fname1, fname2; + int label; + std::istringstream iss(line); + iss>>fname1>>fname2>>label; + // std::cout< faceDetector = FaceDetectorYN::create(detect_model, "", Size(150, 150), score_thresh, nms_thresh); + + std::string recog_model = findDataFile("dnn/onnx/models/face_recognizer_fast.onnx", false); + Ptr faceRecognizer = FaceRecognizerSF::create(recog_model, ""); + + // Detect and match + for (auto fname: fSet) + { + std::string imagePath = findDataFile("dnn_face/recognition/" + fname); + Mat image = imread(imagePath); + + Mat faces; + faceDetector->detect(image, faces); + + Mat aligned_face; + faceRecognizer->alignCrop(image, faces.row(0), aligned_face); + + Mat feature; + faceRecognizer->feature(aligned_face, feature); + + featureMap[fname] = feature.clone(); + } + + for (auto item: gtMap) + { + Mat feature1 = featureMap[item.first.first]; + Mat feature2 = featureMap[item.first.second]; + int label = item.second; + + double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE); + double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2); + + EXPECT_TRUE(label == 0 ? cos_score <= cosine_similar_thresh : cos_score > cosine_similar_thresh) << "Cosine match result of images " << item.first.first << " and " << item.first.second << " is different from ground truth (score: "<< cos_score <<";Thresh: "<< cosine_similar_thresh <<")."; + EXPECT_TRUE(label == 0 ? L2_score > l2norm_similar_thresh : L2_score <= l2norm_similar_thresh) << "L2norm match result of images " << item.first.first << " and " << item.first.second << " is different from ground truth (score: "<< L2_score <<";Thresh: "<< l2norm_similar_thresh <<")."; + } +} + +}} // namespace diff --git a/modules/objdetect/test/test_main.cpp b/modules/objdetect/test/test_main.cpp index 93e4d2860e..4031f0522b 100644 --- a/modules/objdetect/test/test_main.cpp +++ b/modules/objdetect/test/test_main.cpp @@ -7,4 +7,19 @@ #include #endif -CV_TEST_MAIN("cv") +static +void initTests() +{ +#ifdef HAVE_OPENCV_DNN + const char* extraTestDataPath = +#ifdef WINRT + NULL; +#else + getenv("OPENCV_DNN_TEST_DATA_PATH"); +#endif + if (extraTestDataPath) + cvtest::addDataSearchPath(extraTestDataPath); +#endif // HAVE_OPENCV_DNN +} + +CV_TEST_MAIN("cv", initTests()) diff --git a/modules/photo/src/contrast_preserve.hpp b/modules/photo/src/contrast_preserve.hpp index 5681779fc9..4079272e99 100644 --- a/modules/photo/src/contrast_preserve.hpp +++ b/modules/photo/src/contrast_preserve.hpp @@ -358,9 +358,9 @@ void Decolor::grayImContruct(vector &wei, const Mat &img, Mat &Gray) co { for(int i = 0;i(i,j)=Gray.at(i,j) + + Gray.at(i,j)=static_cast(Gray.at(i,j) + static_cast(wei[kk])*pow(rgb_channel[2].at(i,j),r)*pow(rgb_channel[1].at(i,j),g)* - pow(rgb_channel[0].at(i,j),b); + pow(rgb_channel[0].at(i,j),b)); kk=kk+1; } diff --git a/modules/photo/src/inpaint.cpp b/modules/photo/src/inpaint.cpp index 36ab7bec8c..f3fcb47094 100644 --- a/modules/photo/src/inpaint.cpp +++ b/modules/photo/src/inpaint.cpp @@ -308,50 +308,58 @@ icvTeleaInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQu FastMarching_solve(i+1,j,i,j+1,f,t)); CV_MAT_ELEM(*t,float,i,j) = dist; + cv::Point2f gradT[3]; for (color=0; color<=2; color++) { - cv::Point2f gradI,gradT,r; - float Ia=0,Jx=0,Jy=0,s=1.0e-20f,w,dst,lev,dir,sat; - if (CV_MAT_ELEM(*f,uchar,i,j+1)!=INSIDE) { if (CV_MAT_ELEM(*f,uchar,i,j-1)!=INSIDE) { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j-1)))*0.5f; + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j-1)))*0.5f; } else { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j))); + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j+1)-CV_MAT_ELEM(*t,float,i,j))); } } else { if (CV_MAT_ELEM(*f,uchar,i,j-1)!=INSIDE) { - gradT.x=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i,j-1))); + gradT[color].x=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i,j-1))); } else { - gradT.x=0; + gradT[color].x=0; } } if (CV_MAT_ELEM(*f,uchar,i+1,j)!=INSIDE) { if (CV_MAT_ELEM(*f,uchar,i-1,j)!=INSIDE) { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i-1,j)))*0.5f; + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i-1,j)))*0.5f; } else { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i,j))); + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i+1,j)-CV_MAT_ELEM(*t,float,i,j))); } } else { if (CV_MAT_ELEM(*f,uchar,i-1,j)!=INSIDE) { - gradT.y=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i-1,j))); + gradT[color].y=(float)((CV_MAT_ELEM(*t,float,i,j)-CV_MAT_ELEM(*t,float,i-1,j))); } else { - gradT.y=0; + gradT[color].y=0; } } - for (k=i-range; k<=i+range; k++) { - int km=k-1+(k==1),kp=k-1-(k==t->rows-2); - for (l=j-range; l<=j+range; l++) { - int lm=l-1+(l==1),lp=l-1-(l==t->cols-2); - if (k>0&&l>0&&krows-1&&lcols-1) { - if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& - ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + } + + cv::Point2f gradI,r; + float Jx[3] = {0,0,0}; + float Jy[3] = {0,0,0}; + float Ia[3] = {0,0,0}; + float s[3] = {1.0e-20f,1.0e-20f,1.0e-20f}; + float w,dst,lev,dir,sat; + + for (k=i-range; k<=i+range; k++) { + int km=k-1+(k==1),kp=k-1-(k==t->rows-2); + for (l=j-range; l<=j+range; l++) { + int lm=l-1+(l==1),lp=l-1-(l==t->cols-2); + if (k>0&&l>0&&krows-1&&lcols-1) { + if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& + ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + for (color=0; color<=2; color++) { r.y = (float)(i-k); r.x = (float)(j-l); dst = (float)(1./(VectorLength(r)*sqrt((double)VectorLength(r)))); lev = (float)(1./(1+fabs(CV_MAT_ELEM(*t,float,k,l)-CV_MAT_ELEM(*t,float,i,j)))); - dir=VectorScalMult(r,gradT); + dir=VectorScalMult(r,gradT[color]); if (fabs(dir)<=0.01) dir=0.000001f; w = (float)fabs(dst*lev*dir); @@ -381,18 +389,18 @@ icvTeleaInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQu gradI.y=0; } } - Ia += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); - Jx -= (float)w * (float)(gradI.x*r.x); - Jy -= (float)w * (float)(gradI.y*r.y); - s += w; + Ia[color] += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); + Jx[color] -= (float)w * (float)(gradI.x*r.x); + Jy[color] -= (float)w * (float)(gradI.y*r.y); + s[color] += w; } } } } - sat = (float)((Ia/s+(Jx+Jy)/(sqrt(Jx*Jx+Jy*Jy)+1.0e-20f)+0.5f)); - { + } + for (color=0; color<=2; color++) { + sat = (float)((Ia[color]/s[color]+(Jx[color]+Jy[color])/(sqrt(Jx[color]*Jx[color]+Jy[color]*Jy[color])+1.0e-20f)+0.5f)); CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast(sat); - } } CV_MAT_ELEM(*f,uchar,i,j) = BAND; @@ -540,17 +548,19 @@ icvNSInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQueue FastMarching_solve(i+1,j,i,j+1,f,t)); CV_MAT_ELEM(*t,float,i,j) = dist; - for (color=0; color<=2; color++) { - cv::Point2f gradI,r; - float Ia=0,s=1.0e-20f,w,dst,dir; + cv::Point2f gradI,r; + float Ia[3]={0,0,0}; + float s[3]={1.0e-20f,1.0e-20f,1.0e-20f}; + float w,dst,dir; - for (k=i-range; k<=i+range; k++) { - int km=k-1+(k==1),kp=k-1-(k==f->rows-2); - for (l=j-range; l<=j+range; l++) { - int lm=l-1+(l==1),lp=l-1-(l==f->cols-2); - if (k>0&&l>0&&krows-1&&lcols-1) { - if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& - ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + for (k=i-range; k<=i+range; k++) { + int km=k-1+(k==1),kp=k-1-(k==f->rows-2); + for (l=j-range; l<=j+range; l++) { + int lm=l-1+(l==1),lp=l-1-(l==f->cols-2); + if (k>0&&l>0&&krows-1&&lcols-1) { + if ((CV_MAT_ELEM(*f,uchar,k,l)!=INSIDE)&& + ((l-j)*(l-j)+(k-i)*(k-i)<=range*range)) { + for (color=0; color<=2; color++) { r.y=(float)(k-i); r.x=(float)(l-j); @@ -594,13 +604,15 @@ icvNSInpaintFMM(const CvMat *f, CvMat *t, CvMat *out, int range, CvPriorityQueue dir = (float)fabs(VectorScalMult(r,gradI)/sqrt(VectorLength(r)*VectorLength(gradI))); } w = dst*dir; - Ia += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); - s += w; + Ia[color] += (float)w * (float)(CV_MAT_3COLOR_ELEM(*out,uchar,km,lm,color)); + s[color] += w; } } } } - CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast((double)Ia/s); + } + for (color=0; color<=2; color++) { + CV_MAT_3COLOR_ELEM(*out,uchar,i-1,j-1,color) = cv::saturate_cast((double)Ia[color]/s[color]); } CV_MAT_ELEM(*f,uchar,i,j) = BAND; diff --git a/modules/python/CMakeLists.txt b/modules/python/CMakeLists.txt index a51acf386e..c6a9075224 100644 --- a/modules/python/CMakeLists.txt +++ b/modules/python/CMakeLists.txt @@ -20,11 +20,6 @@ add_subdirectory(bindings) add_subdirectory(test) -if(NOT OPENCV_SKIP_PYTHON_LOADER) - include("./python_loader.cmake") - message(STATUS "OpenCV Python: during development append to PYTHONPATH: ${CMAKE_BINARY_DIR}/python_loader") -endif() - if(__disable_python2) ocv_module_disable_(python2) endif() diff --git a/modules/python/bindings/CMakeLists.txt b/modules/python/bindings/CMakeLists.txt index 442107b135..b39c679615 100644 --- a/modules/python/bindings/CMakeLists.txt +++ b/modules/python/bindings/CMakeLists.txt @@ -8,6 +8,10 @@ set(OPENCV_PYTHON_BINDINGS_DIR "${CMAKE_CURRENT_BINARY_DIR}" CACHE INTERNAL "") # This file is included from a subdirectory set(PYTHON_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../") +if(NOT OPENCV_SKIP_PYTHON_LOADER) + include("${PYTHON_SOURCE_DIR}/python_loader.cmake") +endif() + # get list of modules to wrap set(OPENCV_PYTHON_MODULES) foreach(m ${OPENCV_MODULES_BUILD}) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index 0ea4ee7b27..ebbb2e2f65 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -86,7 +86,7 @@ set_target_properties(${the_module} PROPERTIES ARCHIVE_OUTPUT_NAME ${the_module} # prevent name conflict for python2/3 outputs PREFIX "" OUTPUT_NAME cv2 - SUFFIX ${CVPY_SUFFIX}) + SUFFIX "${CVPY_SUFFIX}") if(ENABLE_SOLUTION_FOLDERS) set_target_properties(${the_module} PROPERTIES FOLDER "bindings") @@ -218,29 +218,6 @@ if(NOT OPENCV_SKIP_PYTHON_LOADER) endif() configure_file("${PYTHON_SOURCE_DIR}/package/template/config-x.y.py.in" "${__python_loader_install_tmp_path}/cv2/${__target_config}" @ONLY) install(FILES "${__python_loader_install_tmp_path}/cv2/${__target_config}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) - - # handle Python extra code - foreach(m ${OPENCV_MODULES_BUILD}) - if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} - AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" - ) - set(__base "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") - file(GLOB_RECURSE extra_py_files - RELATIVE "${__base}" - "${__base}/**/*.py" - ) - if(extra_py_files) - list(SORT extra_py_files) - foreach(f ${extra_py_files}) - get_filename_component(__dir "${f}" DIRECTORY) - configure_file("${__base}/${f}" "${__loader_path}/cv2/_extra_py_code/${f}" COPYONLY) - install(FILES "${__base}/${f}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/_extra_py_code/${__dir}/" COMPONENT python) - endforeach() - else() - message(WARNING "Module ${m} has no .py files in misc/python/package") - endif() - endif() - endforeach(m) endif() # NOT OPENCV_SKIP_PYTHON_LOADER unset(PYTHON_SRC_DIR) diff --git a/modules/python/package/cv2/__init__.py b/modules/python/package/cv2/__init__.py index 27e7edd083..07d1e0d21e 100644 --- a/modules/python/package/cv2/__init__.py +++ b/modules/python/package/cv2/__init__.py @@ -2,6 +2,7 @@ OpenCV Python binary extension loader ''' import os +import importlib import sys __all__ = [] @@ -15,17 +16,55 @@ except ImportError: print(' pip install numpy') raise - -py_code_loader = None -if sys.version_info[:2] >= (3, 0): - try: - from . import _extra_py_code as py_code_loader - except: - pass - # TODO # is_x64 = sys.maxsize > 2**32 + +def __load_extra_py_code_for_module(base, name, enable_debug_print=False): + module_name = "{}.{}".format(__name__, name) + export_module_name = "{}.{}".format(base, name) + native_module = sys.modules.pop(module_name, None) + try: + py_module = importlib.import_module(module_name) + except ImportError as err: + if enable_debug_print: + print("Can't load Python code for module:", module_name, + ". Reason:", err) + # Extension doesn't contain extra py code + return False + + if not hasattr(base, name): + setattr(sys.modules[base], name, py_module) + sys.modules[export_module_name] = py_module + # If it is C extension module it is already loaded by cv2 package + if native_module: + setattr(py_module, "_native", native_module) + for k, v in filter(lambda kv: not hasattr(py_module, kv[0]), + native_module.__dict__.items()): + if enable_debug_print: print(' symbol({}): {} = {}'.format(name, k, v)) + setattr(py_module, k, v) + return True + + +def __collect_extra_submodules(enable_debug_print=False): + def modules_filter(module): + return all(( + # module is not internal + not module.startswith("_"), + not module.startswith("python-"), + # it is not a file + os.path.isdir(os.path.join(_extra_submodules_init_path, module)) + )) + if sys.version_info[0] < 3: + if enable_debug_print: + print("Extra submodules is loaded only for Python 3") + return [] + + __INIT_FILE_PATH = os.path.abspath(__file__) + _extra_submodules_init_path = os.path.dirname(__INIT_FILE_PATH) + return filter(modules_filter, os.listdir(_extra_submodules_init_path)) + + def bootstrap(): import sys @@ -107,23 +146,36 @@ def bootstrap(): # amending of LD_LIBRARY_PATH works for sub-processes only os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '') - if DEBUG: print('OpenCV loader: replacing cv2 module') - del sys.modules['cv2'] - import cv2 + if DEBUG: print("Relink everything from native cv2 module to cv2 package") + + py_module = sys.modules.pop("cv2") + + native_module = importlib.import_module("cv2") + + sys.modules["cv2"] = py_module + setattr(py_module, "_native", native_module) + + for item_name, item in filter(lambda kv: kv[0] not in ("__file__", "__loader__", "__spec__", + "__name__", "__package__"), + native_module.__dict__.items()): + if item_name not in g_vars: + g_vars[item_name] = item sys.path = save_sys_path # multiprocessing should start from bootstrap code (https://github.com/opencv/opencv/issues/18502) try: - import sys del sys.OpenCV_LOADER - except: - pass + except Exception as e: + if DEBUG: + print("Exception during delete OpenCV_LOADER:", e) if DEBUG: print('OpenCV loader: binary extension... OK') - if py_code_loader: - py_code_loader.init('cv2') + for submodule in __collect_extra_submodules(DEBUG): + if __load_extra_py_code_for_module("cv2", submodule, DEBUG): + if DEBUG: print("Extra Python code for", submodule, "is loaded") if DEBUG: print('OpenCV loader: DONE') + bootstrap() diff --git a/modules/python/package/cv2/_extra_py_code/__init__.py b/modules/python/package/cv2/_extra_py_code/__init__.py deleted file mode 100644 index be84566825..0000000000 --- a/modules/python/package/cv2/_extra_py_code/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -import sys -import importlib - -__all__ = ['init'] - - -DEBUG = False -if hasattr(sys, 'OpenCV_LOADER_DEBUG'): - DEBUG = True - - -def _load_py_code(base, name): - try: - m = importlib.import_module(__name__ + name) - except ImportError: - return # extension doesn't exist? - - if DEBUG: print('OpenCV loader: added python code extension for: ' + name) - - if hasattr(m, '__all__'): - export_members = { k : getattr(m, k) for k in m.__all__ } - else: - export_members = m.__dict__ - - for k, v in export_members.items(): - if k.startswith('_'): # skip internals - continue - if isinstance(v, type(sys)): # don't bring modules - continue - if DEBUG: print(' symbol: {} = {}'.format(k, v)) - setattr(sys.modules[base + name ], k, v) - - del sys.modules[__name__ + name] - - -# TODO: listdir -def init(base): - _load_py_code(base, '.cv2') # special case - prefix = base - prefix_len = len(prefix) - - modules = [ m for m in sys.modules.keys() if m.startswith(prefix) ] - for m in modules: - m2 = m[prefix_len:] # strip prefix - if len(m2) == 0: - continue - if m2.startswith('._'): # skip internals - continue - if m2.startswith('.load_config_'): # skip helper files - continue - _load_py_code(base, m2) - - del sys.modules[__name__] diff --git a/modules/python/package/extra_modules/misc/__init__.py b/modules/python/package/extra_modules/misc/__init__.py new file mode 100644 index 0000000000..3e0559d3d4 --- /dev/null +++ b/modules/python/package/extra_modules/misc/__init__.py @@ -0,0 +1 @@ +from .version import get_ocv_version diff --git a/modules/python/package/extra_modules/misc/version.py b/modules/python/package/extra_modules/misc/version.py new file mode 100644 index 0000000000..d34705872e --- /dev/null +++ b/modules/python/package/extra_modules/misc/version.py @@ -0,0 +1,5 @@ +import cv2 + + +def get_ocv_version(): + return getattr(cv2, "__version__", "unavailable") diff --git a/modules/python/python_loader.cmake b/modules/python/python_loader.cmake index 677111b061..670d85154c 100644 --- a/modules/python/python_loader.cmake +++ b/modules/python/python_loader.cmake @@ -22,10 +22,15 @@ else() set(CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE "os.path.join(LOADER_DIR, 'not_installed')") endif() +if(OpenCV_FOUND) + return() # Ignore "standalone" builds of Python bindings +endif() + + + set(PYTHON_LOADER_FILES "setup.py" "cv2/__init__.py" "cv2/load_config_py2.py" "cv2/load_config_py3.py" - "cv2/_extra_py_code/__init__.py" ) foreach(fname ${PYTHON_LOADER_FILES}) get_filename_component(__dir "${fname}" DIRECTORY) @@ -40,34 +45,84 @@ foreach(fname ${PYTHON_LOADER_FILES}) endif() endforeach() -if(NOT OpenCV_FOUND) # Ignore "standalone" builds of Python bindings - if(WIN32) - if(CMAKE_GENERATOR MATCHES "Visual Studio") - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'") # TODO: CMAKE_BUILD_TYPE is not defined - else() - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}'") - endif() + + +if(WIN32) + if(CMAKE_GENERATOR MATCHES "Visual Studio") + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'") # TODO: CMAKE_BUILD_TYPE is not defined else() - list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${LIBRARY_OUTPUT_PATH}'") + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}'") + endif() +else() + list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${LIBRARY_OUTPUT_PATH}'") +endif() +string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") +configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__loader_path}/cv2/config.py" @ONLY) + + + +# install +if(DEFINED OPENCV_PYTHON_INSTALL_PATH) + if(WIN32) + list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_BIN_INSTALL_PATH}')") + else() + list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_LIB_INSTALL_PATH}')") + endif() + set(CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_INSTALL_PATH}") + if (WIN32 AND HAVE_CUDA) + if (DEFINED CUDA_TOOLKIT_ROOT_DIR) + list(APPEND CMAKE_PYTHON_BINARIES_PATH "os.path.join(os.getenv('CUDA_PATH', '${CUDA_TOOLKIT_ROOT_DIR}'), 'bin')") + endif() endif() string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") - configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__loader_path}/cv2/config.py" @ONLY) - - # install - if(DEFINED OPENCV_PYTHON_INSTALL_PATH) - if(WIN32) - list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_BIN_INSTALL_PATH}')") - else() - list(APPEND CMAKE_PYTHON_BINARIES_INSTALL_PATH "os.path.join(${CMAKE_PYTHON_EXTENSION_INSTALL_PATH_BASE}, '${OPENCV_LIB_INSTALL_PATH}')") - endif() - set(CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_INSTALL_PATH}") - if (WIN32 AND HAVE_CUDA) - if (DEFINED CUDA_TOOLKIT_ROOT_DIR) - list(APPEND CMAKE_PYTHON_BINARIES_PATH "os.path.join(os.getenv('CUDA_PATH', '${CUDA_TOOLKIT_ROOT_DIR}'), 'bin')") - endif() - endif() - string(REPLACE ";" ",\n " CMAKE_PYTHON_BINARIES_PATH "${CMAKE_PYTHON_BINARIES_PATH}") - configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__python_loader_install_tmp_path}/cv2/config.py" @ONLY) - install(FILES "${__python_loader_install_tmp_path}/cv2/config.py" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) - endif() + configure_file("${PYTHON_SOURCE_DIR}/package/template/config.py.in" "${__python_loader_install_tmp_path}/cv2/config.py" @ONLY) + install(FILES "${__python_loader_install_tmp_path}/cv2/config.py" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/" COMPONENT python) +endif() + + + +# +# Handle Python extra code (submodules) +# +function(ocv_add_python_files_from_path search_path) + file(GLOB_RECURSE extra_py_files + RELATIVE "${search_path}" + # Plain Python code + "${search_path}/*.py" + # Type annotations + "${search_path}/*.pyi" + ) + ocv_debug_message("Extra Py files for ${search_path}: ${extra_py_files}") + if(extra_py_files) + list(SORT extra_py_files) + foreach(filename ${extra_py_files}) + get_filename_component(module "${filename}" DIRECTORY) + if(NOT ${module} IN_LIST extra_modules) + list(APPEND extra_modules ${module}) + endif() + configure_file("${search_path}/${filename}" "${__loader_path}/cv2/${filename}" COPYONLY) + if(DEFINED OPENCV_PYTHON_INSTALL_PATH) + install(FILES "${search_path}/${filename}" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}/cv2/${module}/" COMPONENT python) + endif() + endforeach() + message(STATUS "Found '${extra_modules}' Python modules from ${search_path}") + else() + message(WARNING "Can't add Python files and modules from '${module_path}'. There is no .py or .pyi files") + endif() +endfunction() + +ocv_add_python_files_from_path("${PYTHON_SOURCE_DIR}/package/extra_modules") + +foreach(m ${OPENCV_MODULES_BUILD}) + if (";${OPENCV_MODULE_${m}_WRAPPERS};" MATCHES ";python;" AND HAVE_${m} + AND EXISTS "${OPENCV_MODULE_${m}_LOCATION}/misc/python/package" + ) + ocv_add_python_files_from_path("${OPENCV_MODULE_${m}_LOCATION}/misc/python/package") + endif() +endforeach(m) + +if(NOT "${OPENCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") + foreach(extra_ocv_py_modules_path ${OPENCV_PYTHON_EXTRA_MODULES_PATH}) + ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) + endforeach() endif() diff --git a/modules/python/src2/cv2.cpp b/modules/python/src2/cv2.cpp index 9e8a6ee13b..6231fde67f 100644 --- a/modules/python/src2/cv2.cpp +++ b/modules/python/src2/cv2.cpp @@ -49,6 +49,8 @@ static PyObject* opencv_error = NULL; +static PyTypeObject* pyopencv_Mat_TypePtr = nullptr; + class ArgInfo { public: @@ -496,6 +498,33 @@ bool parseSequence(PyObject* obj, RefWrapper (&value)[N], const ArgInfo& info } } // namespace +namespace traits { +template +struct BooleanConstant +{ + static const bool value = Value; + typedef BooleanConstant type; +}; + +typedef BooleanConstant TrueType; +typedef BooleanConstant FalseType; + +template +struct VoidType { + typedef void type; +}; + +template +struct IsRepresentableAsMatDataType : FalseType +{ +}; + +template +struct IsRepresentableAsMatDataType::channel_type>::type> : TrueType +{ +}; +} // namespace traits + typedef std::vector vector_uchar; typedef std::vector vector_char; typedef std::vector vector_int; @@ -611,10 +640,20 @@ static bool isBool(PyObject* obj) CV_NOEXCEPT return PyArray_IsScalar(obj, Bool) || PyBool_Check(obj); } +template +static std::string pycv_dumpArray(const T* arr, int n) +{ + std::ostringstream out; + out << "["; + for (int i = 0; i < n; ++i) + out << " " << arr[i]; + out << " ]"; + return out.str(); +} + // special case, when the converter needs full ArgInfo structure static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) { - bool allowND = true; if(!o || o == Py_None) { if( !m.data ) @@ -700,12 +739,29 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) return false; } - int size[CV_MAX_DIM+1]; - size_t step[CV_MAX_DIM+1]; size_t elemsize = CV_ELEM_SIZE1(type); const npy_intp* _sizes = PyArray_DIMS(oarr); const npy_intp* _strides = PyArray_STRIDES(oarr); + + CV_LOG_DEBUG(NULL, "Incoming ndarray '" << info.name << "': ndims=" << ndims << " _sizes=" << pycv_dumpArray(_sizes, ndims) << " _strides=" << pycv_dumpArray(_strides, ndims)); + bool ismultichannel = ndims == 3 && _sizes[2] <= CV_CN_MAX; + if (pyopencv_Mat_TypePtr && PyObject_TypeCheck(o, pyopencv_Mat_TypePtr)) + { + bool wrapChannels = false; + PyObject* pyobj_wrap_channels = PyObject_GetAttrString(o, "wrap_channels"); + if (pyobj_wrap_channels) + { + if (!pyopencv_to_safe(pyobj_wrap_channels, wrapChannels, ArgInfo("cv.Mat.wrap_channels", 0))) + { + // TODO extra message + Py_DECREF(pyobj_wrap_channels); + return false; + } + Py_DECREF(pyobj_wrap_channels); + } + ismultichannel = wrapChannels && ndims >= 1; + } for( int i = ndims-1; i >= 0 && !needcopy; i-- ) { @@ -719,14 +775,26 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) needcopy = true; } - if( ismultichannel && _strides[1] != (npy_intp)elemsize*_sizes[2] ) - needcopy = true; + if (ismultichannel) + { + int channels = ndims >= 1 ? (int)_sizes[ndims - 1] : 1; + if (channels > CV_CN_MAX) + { + failmsg("%s unable to wrap channels, too high (%d > CV_CN_MAX=%d)", info.name, (int)channels, (int)CV_CN_MAX); + return false; + } + ndims--; + type |= CV_MAKETYPE(0, channels); + + if (ndims >= 1 && _strides[ndims - 1] != (npy_intp)elemsize*_sizes[ndims]) + needcopy = true; + } if (needcopy) { if (info.outputarg) { - failmsg("Layout of the output array %s is incompatible with cv::Mat (step[ndims-1] != elemsize or step[1] != elemsize*nchannels)", info.name); + failmsg("Layout of the output array %s is incompatible with cv::Mat", info.name); return false; } @@ -742,6 +810,9 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) _strides = PyArray_STRIDES(oarr); } + int size[CV_MAX_DIM+1] = {}; + size_t step[CV_MAX_DIM+1] = {}; + // Normalize strides in case NPY_RELAXED_STRIDES is set size_t default_step = elemsize; for ( int i = ndims - 1; i >= 0; --i ) @@ -760,23 +831,16 @@ static bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) } // handle degenerate case + // FIXIT: Don't force 1D for Scalars if( ndims == 0) { size[ndims] = 1; step[ndims] = elemsize; ndims++; } - if( ismultichannel ) - { - ndims--; - type |= CV_MAKETYPE(0, size[2]); - } - - if( ndims > 2 && !allowND ) - { - failmsg("%s has more than 2 dimensions", info.name); - return false; - } +#if 1 + CV_LOG_DEBUG(NULL, "Construct Mat: ndims=" << ndims << " size=" << pycv_dumpArray(size, ndims) << " step=" << pycv_dumpArray(step, ndims) << " type=" << cv::typeToString(type)); +#endif m = Mat(ndims, size, type, PyArray_DATA(oarr), step); m.u = g_numpyAllocator.allocate(o, ndims, size, type, step); @@ -1072,6 +1136,30 @@ bool pyopencv_to(PyObject* obj, uchar& value, const ArgInfo& info) return ivalue != -1 || !PyErr_Occurred(); } +template<> +bool pyopencv_to(PyObject* obj, char& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (isBool(obj)) + { + failmsg("Argument '%s' must be an integer, not bool", info.name); + return false; + } + if (PyArray_IsIntegerScalar(obj)) + { + value = saturate_cast(PyArray_PyIntAsInt(obj)); + } + else + { + failmsg("Argument '%s' is required to be an integer", info.name); + return false; + } + return !CV_HAS_CONVERSION_ERROR(value); +} + template<> PyObject* pyopencv_from(const double& value) { @@ -1484,357 +1572,12 @@ PyObject* pyopencv_from(const Point3d& p) return Py_BuildValue("(ddd)", p.x, p.y, p.z); } -template struct pyopencvVecConverter -{ - typedef typename DataType<_Tp>::channel_type _Cp; - static inline bool copyOneItem(PyObject *obj, size_t start, int channels, _Cp * data) - { - for(size_t j = 0; (int)j < channels; j++ ) - { - SafeSeqItem sub_item_wrap(obj, start + j); - PyObject* item_ij = sub_item_wrap.item; - if( PyInt_Check(item_ij)) - { - int v = (int)PyInt_AsLong(item_ij); - if( v == -1 && PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else if( PyLong_Check(item_ij)) - { - int v = (int)PyLong_AsLong(item_ij); - if( v == -1 && PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else if( PyFloat_Check(item_ij)) - { - double v = PyFloat_AsDouble(item_ij); - if( PyErr_Occurred() ) - return false; - data[j] = saturate_cast<_Cp>(v); - } - else - return false; - } - return true; - } - static bool to(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) - { - if(!obj || obj == Py_None) - return true; - if (PyArray_Check(obj)) - { - Mat m; - pyopencv_to(obj, m, info); - m.copyTo(value); - return true; - } - else if (PySequence_Check(obj)) - { - const int type = traits::Type<_Tp>::value; - const int depth = CV_MAT_DEPTH(type), channels = CV_MAT_CN(type); - size_t i, n = PySequence_Size(obj); - value.resize(n); - for (i = 0; i < n; i++ ) - { - SafeSeqItem item_wrap(obj, i); - PyObject* item = item_wrap.item; - _Cp* data = (_Cp*)&value[i]; - - if( channels == 2 && PyComplex_Check(item) ) - { - data[0] = saturate_cast<_Cp>(PyComplex_RealAsDouble(item)); - data[1] = saturate_cast<_Cp>(PyComplex_ImagAsDouble(item)); - } - else if( channels > 1 ) - { - if( PyArray_Check(item)) - { - Mat src; - pyopencv_to(item, src, info); - if( src.dims != 2 || src.channels() != 1 || - ((src.cols != 1 || src.rows != channels) && - (src.cols != channels || src.rows != 1))) - break; - Mat dst(src.rows, src.cols, depth, data); - src.convertTo(dst, type); - if( dst.data != (uchar*)data ) - break; - } - else if (PySequence_Check(item)) - { - if (!copyOneItem(item, 0, channels, data)) - break; - } - else - { - break; - } - } - else if (channels == 1) - { - if (!copyOneItem(obj, i, channels, data)) - break; - } - else - { - break; - } - } - if (i != n) - { - failmsg("Can't convert vector element for '%s', index=%d", info.name, i); - } - return i == n; - } - failmsg("Can't convert object to vector for '%s', unsupported type", info.name); - return false; - } - - static PyObject* from(const std::vector<_Tp>& value) - { - if(value.empty()) - return PyTuple_New(0); - int type = traits::Type<_Tp>::value; - int depth = CV_MAT_DEPTH(type), channels = CV_MAT_CN(type); - Mat src((int)value.size(), channels, depth, (uchar*)&value[0]); - return pyopencv_from(src); - } -}; - -template -bool pyopencv_to(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) -{ - return pyopencvVecConverter<_Tp>::to(obj, value, info); -} - -template -PyObject* pyopencv_from(const std::vector<_Tp>& value) -{ - return pyopencvVecConverter<_Tp>::from(value); -} - -template static inline bool pyopencv_to_generic_vec(PyObject* obj, std::vector<_Tp>& value, const ArgInfo& info) -{ - if(!obj || obj == Py_None) - return true; - if (!PySequence_Check(obj)) - return false; - size_t n = PySequence_Size(obj); - value.resize(n); - for(size_t i = 0; i < n; i++ ) - { - SafeSeqItem item_wrap(obj, i); - if(!pyopencv_to(item_wrap.item, value[i], info)) - return false; - } - return true; -} - -template<> inline bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) -{ - if(!obj || obj == Py_None) - return true; - if (!PySequence_Check(obj)) - return false; - size_t n = PySequence_Size(obj); - value.resize(n); - for(size_t i = 0; i < n; i++ ) - { - SafeSeqItem item_wrap(obj, i); - bool elem{}; - if(!pyopencv_to(item_wrap.item, elem, info)) - return false; - value[i] = elem; - } - return true; -} - -template static inline PyObject* pyopencv_from_generic_vec(const std::vector<_Tp>& value) -{ - int i, n = (int)value.size(); - PyObject* seq = PyList_New(n); - for( i = 0; i < n; i++ ) - { - _Tp elem = value[i]; - PyObject* item = pyopencv_from(elem); - if(!item) - break; - PyList_SetItem(seq, i, item); - } - if( i < n ) - { - Py_DECREF(seq); - return 0; - } - return seq; -} - -template<> inline PyObject* pyopencv_from_generic_vec(const std::vector& value) -{ - int i, n = (int)value.size(); - PyObject* seq = PyList_New(n); - for( i = 0; i < n; i++ ) - { - bool elem = value[i]; - PyObject* item = pyopencv_from(elem); - if(!item) - break; - PyList_SetItem(seq, i, item); - } - if( i < n ) - { - Py_DECREF(seq); - return 0; - } - return seq; -} - -template -inline typename std::enable_if::type -convert_to_python_tuple(const std::tuple&, PyObject*) { } - -template -inline typename std::enable_if::type -convert_to_python_tuple(const std::tuple& cpp_tuple, PyObject* py_tuple) -{ - PyObject* item = pyopencv_from(std::get(cpp_tuple)); - - if (!item) - return; - - PyTuple_SetItem(py_tuple, I, item); - convert_to_python_tuple(cpp_tuple, py_tuple); -} - - -template -PyObject* pyopencv_from(const std::tuple& cpp_tuple) -{ - size_t size = sizeof...(Ts); - PyObject* py_tuple = PyTuple_New(size); - convert_to_python_tuple(cpp_tuple, py_tuple); - size_t actual_size = PyTuple_Size(py_tuple); - - if (actual_size < size) - { - Py_DECREF(py_tuple); - return NULL; - } - - return py_tuple; -} - template<> PyObject* pyopencv_from(const std::pair& src) { return Py_BuildValue("(id)", src.first, src.second); } -template struct pyopencvVecConverter > -{ - static bool to(PyObject* obj, std::vector >& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector >& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template struct pyopencvVecConverter > -{ - static bool to(PyObject* obj, std::vector >& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector >& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; - template<> bool pyopencv_to(PyObject* obj, TermCriteria& dst, const ArgInfo& info) { @@ -1962,6 +1705,266 @@ PyObject* pyopencv_from(const Moments& m) "nu30", m.nu30, "nu21", m.nu21, "nu12", m.nu12, "nu03", m.nu03); } +template +struct pyopencvVecConverter; + +template +bool pyopencv_to(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + return pyopencvVecConverter::to(obj, value, info); +} + +template +PyObject* pyopencv_from(const std::vector& value) +{ + return pyopencvVecConverter::from(value); +} + +template +static bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s'. Input argument doesn't provide sequence protocol", info.name); + return false; + } + const size_t n = static_cast(PySequence_Size(obj)); + value.resize(n); + for (size_t i = 0; i < n; i++) + { + SafeSeqItem item_wrap(obj, i); + if (!pyopencv_to(item_wrap.item, value[i], info)) + { + failmsg("Can't parse '%s'. Sequence item with index %lu has a wrong type", info.name, i); + return false; + } + } + return true; +} + +template<> inline bool pyopencv_to_generic_vec(PyObject* obj, std::vector& value, const ArgInfo& info) +{ + if (!obj || obj == Py_None) + { + return true; + } + if (!PySequence_Check(obj)) + { + failmsg("Can't parse '%s'. Input argument doesn't provide sequence protocol", info.name); + return false; + } + const size_t n = static_cast(PySequence_Size(obj)); + value.resize(n); + for (size_t i = 0; i < n; i++) + { + SafeSeqItem item_wrap(obj, i); + bool elem{}; + if (!pyopencv_to(item_wrap.item, elem, info)) + { + failmsg("Can't parse '%s'. Sequence item with index %lu has a wrong type", info.name, i); + return false; + } + value[i] = elem; + } + return true; +} + + +template +static PyObject* pyopencv_from_generic_vec(const std::vector& value) +{ + Py_ssize_t n = static_cast(value.size()); + PySafeObject seq(PyTuple_New(n)); + for (Py_ssize_t i = 0; i < n; i++) + { + PyObject* item = pyopencv_from(value[i]); + // If item can't be assigned - PyTuple_SetItem raises exception and returns -1. + if (!item || PyTuple_SetItem(seq, i, item) == -1) + { + return NULL; + } + } + return seq.release(); +} + +template<> inline PyObject* pyopencv_from_generic_vec(const std::vector& value) +{ + Py_ssize_t n = static_cast(value.size()); + PySafeObject seq(PyTuple_New(n)); + for (Py_ssize_t i = 0; i < n; i++) + { + bool elem = value[i]; + PyObject* item = pyopencv_from(elem); + // If item can't be assigned - PyTuple_SetItem raises exception and returns -1. + if (!item || PyTuple_SetItem(seq, i, item) == -1) + { + return NULL; + } + } + return seq.release(); +} + + +template +inline typename std::enable_if::type +convert_to_python_tuple(const std::tuple&, PyObject*) { } + +template +inline typename std::enable_if::type +convert_to_python_tuple(const std::tuple& cpp_tuple, PyObject* py_tuple) +{ + PyObject* item = pyopencv_from(std::get(cpp_tuple)); + + if (!item) + return; + + PyTuple_SetItem(py_tuple, I, item); + convert_to_python_tuple(cpp_tuple, py_tuple); +} + + +template +PyObject* pyopencv_from(const std::tuple& cpp_tuple) +{ + size_t size = sizeof...(Ts); + PyObject* py_tuple = PyTuple_New(size); + convert_to_python_tuple(cpp_tuple, py_tuple); + size_t actual_size = PyTuple_Size(py_tuple); + + if (actual_size < size) + { + Py_DECREF(py_tuple); + return NULL; + } + + return py_tuple; +} + +template +struct pyopencvVecConverter +{ + typedef typename std::vector::iterator VecIt; + + static bool to(PyObject* obj, std::vector& value, const ArgInfo& info) + { + if (!PyArray_Check(obj)) + { + return pyopencv_to_generic_vec(obj, value, info); + } + // If user passed an array it is possible to make faster conversions in several cases + PyArrayObject* array_obj = reinterpret_cast(obj); + const NPY_TYPES target_type = asNumpyType(); + const NPY_TYPES source_type = static_cast(PyArray_TYPE(array_obj)); + if (target_type == NPY_OBJECT) + { + // Non-planar arrays representing objects (e.g. array of N Rect is an array of shape Nx4) have NPY_OBJECT + // as their target type. + return pyopencv_to_generic_vec(obj, value, info); + } + if (PyArray_NDIM(array_obj) > 1) + { + failmsg("Can't parse %dD array as '%s' vector argument", PyArray_NDIM(array_obj), info.name); + return false; + } + if (target_type != source_type) + { + // Source type requires conversion + // Allowed conversions for target type is handled in the corresponding pyopencv_to function + return pyopencv_to_generic_vec(obj, value, info); + } + // For all other cases, all array data can be directly copied to std::vector data + // Simple `memcpy` is not possible because NumPy array can reference a slice of the bigger array: + // ``` + // arr = np.ones((8, 4, 5), dtype=np.int32) + // convertible_to_vector_of_int = arr[:, 0, 1] + // ``` + value.resize(static_cast(PyArray_SIZE(array_obj))); + const npy_intp item_step = PyArray_STRIDE(array_obj, 0) / PyArray_ITEMSIZE(array_obj); + const Tp* data_ptr = static_cast(PyArray_DATA(array_obj)); + for (VecIt it = value.begin(); it != value.end(); ++it, data_ptr += item_step) { + *it = *data_ptr; + } + return true; + } + + static PyObject* from(const std::vector& value) + { + if (value.empty()) + { + return PyTuple_New(0); + } + return from(value, ::traits::IsRepresentableAsMatDataType()); + } + +private: + static PyObject* from(const std::vector& value, ::traits::FalseType) + { + // Underlying type is not representable as Mat Data Type + return pyopencv_from_generic_vec(value); + } + + static PyObject* from(const std::vector& value, ::traits::TrueType) + { + // Underlying type is representable as Mat Data Type, so faster return type is available + typedef DataType DType; + typedef typename DType::channel_type UnderlyingArrayType; + + // If Mat is always exposed as NumPy array this code path can be reduced to the following snipped: + // Mat src(value); + // PyObject* array = pyopencv_from(src); + // return PyArray_Squeeze(reinterpret_cast(array)); + // This puts unnecessary restrictions on Mat object those might be avoided without losing the performance. + // Moreover, this version is a bit faster, because it doesn't create temporary objects with reference counting. + + const NPY_TYPES target_type = asNumpyType(); + const int cols = DType::channels; + PyObject* array = NULL; + if (cols == 1) + { + npy_intp dims = static_cast(value.size()); + array = PyArray_SimpleNew(1, &dims, target_type); + } + else + { + npy_intp dims[2] = {static_cast(value.size()), cols}; + array = PyArray_SimpleNew(2, dims, target_type); + } + if(!array) + { + // NumPy arrays with shape (N, 1) and (N) are not equal, so correct error message should distinguish + // them too. + String shape; + if (cols > 1) + { + shape = format("(%d x %d)", static_cast(value.size()), cols); + } + else + { + shape = format("(%d)", static_cast(value.size())); + } + const String error_message = format("Can't allocate NumPy array for vector with dtype=%d and shape=%s", + static_cast(target_type), shape.c_str()); + emit_failmsg(PyExc_MemoryError, error_message.c_str()); + return array; + } + // Fill the array + PyArrayObject* array_obj = reinterpret_cast(array); + UnderlyingArrayType* array_data = static_cast(PyArray_DATA(array_obj)); + // if Tp is representable as Mat DataType, so the following cast is pretty safe... + const UnderlyingArrayType* value_data = reinterpret_cast(value.data()); + memcpy(array_data, value_data, sizeof(UnderlyingArrayType) * value.size() * static_cast(cols)); + return array; + } +}; + static int OnError(int status, const char *func_name, const char *err_msg, const char *file_name, int line, void *userdata) { PyGILState_STATE gstate; @@ -2081,15 +2084,23 @@ static void OnChange(int pos, void *param) } #ifdef HAVE_OPENCV_HIGHGUI +// workaround for #20408, use nullptr, set value later +static int _createTrackbar(const String &trackbar_name, const String &window_name, int value, int count, + TrackbarCallback onChange, PyObject* py_callback_info) +{ + int n = createTrackbar(trackbar_name, window_name, NULL, count, onChange, py_callback_info); + setTrackbarPos(trackbar_name, window_name, value); + return n; +} static PyObject *pycvCreateTrackbar(PyObject*, PyObject *args) { PyObject *on_change; char* trackbar_name; char* window_name; - int *value = new int; + int value; int count; - if (!PyArg_ParseTuple(args, "ssiiO", &trackbar_name, &window_name, value, &count, &on_change)) + if (!PyArg_ParseTuple(args, "ssiiO", &trackbar_name, &window_name, &value, &count, &on_change)) return NULL; if (!PyCallable_Check(on_change)) { PyErr_SetString(PyExc_TypeError, "on_change must be callable"); @@ -2108,7 +2119,7 @@ static PyObject *pycvCreateTrackbar(PyObject*, PyObject *args) { registered_callbacks.insert(std::pair(name, py_callback_info)); } - ERRWRAP2(createTrackbar(trackbar_name, window_name, value, count, OnChange, py_callback_info)); + ERRWRAP2(_createTrackbar(trackbar_name, window_name, value, count, OnChange, py_callback_info)); Py_RETURN_NONE; } @@ -2209,7 +2220,24 @@ static int convert_to_char(PyObject *o, char *dst, const ArgInfo& info) #include "pyopencv_generated_types_content.h" #include "pyopencv_generated_funcs.h" +static PyObject* pycvRegisterMatType(PyObject *self, PyObject *value) +{ + CV_LOG_DEBUG(NULL, cv::format("pycvRegisterMatType %p %p\n", self, value)); + + if (0 == PyType_Check(value)) + { + PyErr_SetString(PyExc_TypeError, "Type argument is expected"); + return NULL; + } + + Py_INCREF(value); + pyopencv_Mat_TypePtr = (PyTypeObject*)value; + + Py_RETURN_NONE; +} + static PyMethodDef special_methods[] = { + {"_registerMatType", (PyCFunction)(pycvRegisterMatType), METH_O, "_registerMatType(cv.Mat) -> None (Internal)"}, {"redirectError", CV_PY_FN_WITH_KW(pycvRedirectError), "redirectError(onError) -> None"}, #ifdef HAVE_OPENCV_HIGHGUI {"createTrackbar", (PyCFunction)pycvCreateTrackbar, METH_VARARGS, "createTrackbar(trackbarName, windowName, value, count, onChange) -> None"}, @@ -2219,12 +2247,6 @@ static PyMethodDef special_methods[] = { #ifdef HAVE_OPENCV_DNN {"dnn_registerLayer", CV_PY_FN_WITH_KW(pyopencv_cv_dnn_registerLayer), "registerLayer(type, class) -> None"}, {"dnn_unregisterLayer", CV_PY_FN_WITH_KW(pyopencv_cv_dnn_unregisterLayer), "unregisterLayer(type) -> None"}, -#endif -#ifdef HAVE_OPENCV_GAPI - {"GIn", CV_PY_FN_WITH_KW(pyopencv_cv_GIn), "GIn(...) -> GInputProtoArgs"}, - {"GOut", CV_PY_FN_WITH_KW(pyopencv_cv_GOut), "GOut(...) -> GOutputProtoArgs"}, - {"gin", CV_PY_FN_WITH_KW(pyopencv_cv_gin), "gin(...) -> ExtractArgsCallback"}, - {"descr_of", CV_PY_FN_WITH_KW(pyopencv_cv_descr_of), "descr_of(...) -> ExtractMetaCallback"}, #endif {NULL, NULL}, }; diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index c6ed806017..babded785d 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -214,6 +214,16 @@ simple_argtype_mapping = { "Stream": ArgTypeInfo("Stream", FormatStrings.object, 'Stream::Null()', True), } +# Set of reserved keywords for Python. Can be acquired via the following call +# $ python -c "help('keywords')" +# Keywords that are reserved in C/C++ are excluded because they can not be +# used as variables identifiers +python_reserved_keywords = { + "True", "None", "False", "as", "assert", "def", "del", "elif", "except", "exec", + "finally", "from", "global", "import", "in", "is", "lambda", "nonlocal", + "pass", "print", "raise", "with", "yield" +} + def normalize_class_name(name): return re.sub(r"^cv\.", "", name).replace(".", "_") @@ -387,6 +397,8 @@ class ArgInfo(object): def __init__(self, arg_tuple): self.tp = handle_ptr(arg_tuple[0]) self.name = arg_tuple[1] + if self.name in python_reserved_keywords: + self.name += "_" self.defval = arg_tuple[2] self.isarray = False self.arraylen = 0 diff --git a/modules/python/src2/hdr_parser.py b/modules/python/src2/hdr_parser.py index ace22ee0ee..d304e6cfcf 100755 --- a/modules/python/src2/hdr_parser.py +++ b/modules/python/src2/hdr_parser.py @@ -55,13 +55,13 @@ class CppHeaderParser(object): def get_macro_arg(self, arg_str, npos): npos2 = npos3 = arg_str.find("(", npos) if npos2 < 0: - print("Error: no arguments for the macro at %d" % (self.lineno,)) + print("Error: no arguments for the macro at %s:%d" % (self.hname, self.lineno)) sys.exit(-1) balance = 1 while 1: t, npos3 = self.find_next_token(arg_str, ['(', ')'], npos3+1) if npos3 < 0: - print("Error: no matching ')' in the macro call at %d" % (self.lineno,)) + print("Error: no matching ')' in the macro call at %s:%d" % (self.hname, self.lineno)) sys.exit(-1) if t == '(': balance += 1 @@ -168,7 +168,7 @@ class CppHeaderParser(object): angle_stack.append(0) elif w == "," or w == '>': if not angle_stack: - print("Error at %d: argument contains ',' or '>' not within template arguments" % (self.lineno,)) + print("Error at %s:%d: argument contains ',' or '>' not within template arguments" % (self.hname, self.lineno)) sys.exit(-1) if w == ",": arg_type += "_and_" @@ -198,7 +198,7 @@ class CppHeaderParser(object): p1 = arg_name.find("[") p2 = arg_name.find("]",p1+1) if p2 < 0: - print("Error at %d: no closing ]" % (self.lineno,)) + print("Error at %s:%d: no closing ]" % (self.hname, self.lineno)) sys.exit(-1) counter_str = arg_name[p1+1:p2].strip() if counter_str == "": @@ -443,11 +443,18 @@ class CppHeaderParser(object): # filter off some common prefixes, which are meaningless for Python wrappers. # note that we do not strip "static" prefix, which does matter; # it means class methods, not instance methods - decl_str = self.batch_replace(decl_str, [("static inline", ""), ("inline", ""), ("explicit ", ""), - ("CV_EXPORTS_W", ""), ("CV_EXPORTS", ""), ("CV_CDECL", ""), - ("CV_WRAP ", " "), ("CV_INLINE", ""), - ("CV_DEPRECATED", ""), ("CV_DEPRECATED_EXTERNAL", "")]).strip() - + decl_str = self.batch_replace(decl_str, [("static inline", ""), + ("inline", ""), + ("explicit ", ""), + ("CV_EXPORTS_W", ""), + ("CV_EXPORTS", ""), + ("CV_CDECL", ""), + ("CV_WRAP ", " "), + ("CV_INLINE", ""), + ("CV_DEPRECATED", ""), + ("CV_DEPRECATED_EXTERNAL", ""), + ("CV_NODISCARD_STD", ""), + ("CV_NODISCARD", "")]).strip() if decl_str.strip().startswith('virtual'): virtual_method = True @@ -843,6 +850,7 @@ class CppHeaderParser(object): ("GAPI_EXPORTS_W_SIMPLE","CV_EXPORTS_W_SIMPLE"), ("GAPI_WRAP", "CV_WRAP"), ("GAPI_PROP", "CV_PROP"), + ("GAPI_PROP_RW", "CV_PROP_RW"), ('defined(GAPI_STANDALONE)', '0'), ]) @@ -989,7 +997,8 @@ class CppHeaderParser(object): has_mat = len(list(filter(lambda x: x[0] in {"Mat", "vector_Mat"}, args))) > 0 if has_mat: _, _, _, gpumat_decl = self.parse_stmt(stmt, token, mat="cuda::GpuMat", docstring=docstring) - decls.append(gpumat_decl) + if gpumat_decl != decl: + decls.append(gpumat_decl) if self._generate_umat_decls: # If function takes as one of arguments Mat or vector - we want to create the @@ -998,7 +1007,8 @@ class CppHeaderParser(object): has_mat = len(list(filter(lambda x: x[0] in {"Mat", "vector_Mat"}, args))) > 0 if has_mat: _, _, _, umat_decl = self.parse_stmt(stmt, token, mat="UMat", docstring=docstring) - decls.append(umat_decl) + if umat_decl != decl: + decls.append(umat_decl) docstring = "" if stmt_type == "namespace": diff --git a/modules/python/test/test_filestorage_io.py b/modules/python/test/test_filestorage_io.py index 62b540d79c..01e0a72300 100644 --- a/modules/python/test/test_filestorage_io.py +++ b/modules/python/test/test_filestorage_io.py @@ -1,6 +1,8 @@ #!/usr/bin/env python """Algorithm serialization test.""" from __future__ import print_function +import base64 +import json import tempfile import os import cv2 as cv @@ -109,5 +111,96 @@ class filestorage_io_test(NewOpenCVTests): def test_json(self): self.run_fs_test(".json") + def test_base64(self): + fd, fname = tempfile.mkstemp(prefix="opencv_python_sample_filestorage_base64", suffix=".json") + os.close(fd) + np.random.seed(42) + self.write_base64_json(fname) + os.remove(fname) + + @staticmethod + def get_normal_2d_mat(): + rows = 10 + cols = 20 + cn = 3 + + image = np.zeros((rows, cols, cn), np.uint8) + image[:] = (1, 2, 127) + + for i in range(rows): + for j in range(cols): + image[i, j, 1] = (i + j) % 256 + + return image + + @staticmethod + def get_normal_nd_mat(): + shape = (2, 2, 1, 2) + cn = 4 + + image = np.zeros(shape + (cn,), np.float64) + image[:] = (0.888, 0.111, 0.666, 0.444) + + return image + + @staticmethod + def get_empty_2d_mat(): + shape = (0, 0) + cn = 1 + + image = np.zeros(shape + (cn,), np.uint8) + + return image + + @staticmethod + def get_random_mat(): + rows = 8 + cols = 16 + cn = 1 + + image = np.random.rand(rows, cols, cn) + + return image + + @staticmethod + def decode(data): + # strip $base64$ + encoded = data[8:] + + if len(encoded) == 0: + return b'' + + # strip info about datatype and padding + return base64.b64decode(encoded)[24:] + + def write_base64_json(self, fname): + fs = cv.FileStorage(fname, cv.FileStorage_WRITE_BASE64) + + mats = {'normal_2d_mat': self.get_normal_2d_mat(), + 'normal_nd_mat': self.get_normal_nd_mat(), + 'empty_2d_mat': self.get_empty_2d_mat(), + 'random_mat': self.get_random_mat()} + + for name, mat in mats.items(): + fs.write(name, mat) + + fs.release() + + data = {} + with open(fname) as file: + data = json.load(file) + + for name, mat in mats.items(): + buffer = b'' + + if mat.size != 0: + if hasattr(mat, 'tobytes'): + buffer = mat.tobytes() + else: + buffer = mat.tostring() + + self.assertEqual(buffer, self.decode(data[name]['data'])) + + if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/python/test/test_houghcircles.py b/modules/python/test/test_houghcircles.py index 90cd184139..2cfc6c9d68 100644 --- a/modules/python/test/test_houghcircles.py +++ b/modules/python/test/test_houghcircles.py @@ -80,5 +80,52 @@ class houghcircles_test(NewOpenCVTests): self.assertLess(float(len(circles) - matches_counter) / len(circles), .75) + def test_houghcircles_alt(self): + + fn = "samples/data/board.jpg" + + src = self.get_sample(fn, 1) + img = cv.cvtColor(src, cv.COLOR_BGR2GRAY) + img = cv.medianBlur(img, 5) + + circles = cv.HoughCircles(img, cv.HOUGH_GRADIENT_ALT, 1, 10, np.array([]), 300, 0.9, 1, 30) + + self.assertEqual(circles.shape, (1, 18, 3)) + + circles = circles[0] + + testCircles = [[38, 181, 17.6], + [99.7, 166, 13.12], + [142.7, 160, 13.52], + [223.6, 110, 8.62], + [79.1, 206.7, 8.62], + [47.5, 351.6, 11.64], + [189.5, 354.4, 11.64], + [189.8, 298.9, 10.64], + [189.5, 252.4, 14.62], + [252.5, 393.4, 15.62], + [602.9, 467.5, 11.42], + [222, 210.4, 9.12], + [263.1, 216.7, 9.12], + [359.8, 222.6, 9.12], + [518.9, 120.9, 9.12], + [413.8, 113.4, 9.12], + [489, 127.2, 9.12], + [448.4, 121.3, 9.12], + [384.6, 128.9, 8.62]] + + matches_counter = 0 + + for i in range(len(testCircles)): + for j in range(len(circles)): + + tstCircle = circleApproximation(testCircles[i]) + circle = circleApproximation(circles[j]) + if convContoursIntersectiponRate(tstCircle, circle) > 0.6: + matches_counter += 1 + + self.assertGreater(float(matches_counter) / len(testCircles), .5) + self.assertLess(float(len(circles) - matches_counter) / len(circles), .75) + if __name__ == '__main__': NewOpenCVTests.bootstrap() diff --git a/modules/python/test/test_legacy.py b/modules/python/test/test_legacy.py index ab0a8bdc35..e550ab73c8 100644 --- a/modules/python/test/test_legacy.py +++ b/modules/python/test/test_legacy.py @@ -20,8 +20,13 @@ class Hackathon244Tests(NewOpenCVTests): flag, ajpg = cv.imencode("img_q90.jpg", a, [cv.IMWRITE_JPEG_QUALITY, 90]) self.assertEqual(flag, True) self.assertEqual(ajpg.dtype, np.uint8) - self.assertGreater(ajpg.shape[0], 1) - self.assertEqual(ajpg.shape[1], 1) + self.assertTrue(isinstance(ajpg, np.ndarray), "imencode returned buffer of wrong type: {}".format(type(ajpg))) + self.assertEqual(len(ajpg.shape), 1, "imencode returned buffer with wrong shape: {}".format(ajpg.shape)) + self.assertGreaterEqual(len(ajpg), 1, "imencode length of the returned buffer should be at least 1") + self.assertLessEqual( + len(ajpg), a.size, + "imencode length of the returned buffer shouldn't exceed number of elements in original image" + ) def test_projectPoints(self): objpt = np.float64([[1,2,3]]) diff --git a/modules/python/test/test_mat.py b/modules/python/test/test_mat.py new file mode 100644 index 0000000000..72614fda36 --- /dev/null +++ b/modules/python/test/test_mat.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +from __future__ import print_function + +import numpy as np +import cv2 as cv + +import os +import sys +import unittest + +from tests_common import NewOpenCVTests + +try: + if sys.version_info[:2] < (3, 0): + raise unittest.SkipTest('Python 2.x is not supported') + + + class MatTest(NewOpenCVTests): + + def test_mat_construct(self): + data = np.random.random([10, 10, 3]) + + #print(np.ndarray.__dictoffset__) # 0 + #print(cv.Mat.__dictoffset__) # 88 (> 0) + #print(cv.Mat) # + #print(cv.Mat.__base__) # + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=300 dims(-1)=3 size(-1)=[10 10 3] type(-1)=CV_64FC1") + + mat_data1 = cv.Mat(data, wrap_channels=True) + assert isinstance(mat_data1, cv.Mat) + assert isinstance(mat_data1, np.ndarray) + self.assertEqual(mat_data1.wrap_channels, True) + res1 = cv.utils.dumpInputArray(mat_data1) + self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3") + + mat_data2 = cv.Mat(mat_data1) + assert isinstance(mat_data2, cv.Mat) + assert isinstance(mat_data2, np.ndarray) + self.assertEqual(mat_data2.wrap_channels, True) # fail if __array_finalize__ doesn't work + res2 = cv.utils.dumpInputArray(mat_data2) + self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=100 dims(-1)=2 size(-1)=10x10 type(-1)=CV_64FC3") + + + def test_mat_construct_4d(self): + data = np.random.random([5, 10, 10, 3]) + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=1500 dims(-1)=4 size(-1)=[5 10 10 3] type(-1)=CV_64FC1") + + mat_data1 = cv.Mat(data, wrap_channels=True) + assert isinstance(mat_data1, cv.Mat) + assert isinstance(mat_data1, np.ndarray) + self.assertEqual(mat_data1.wrap_channels, True) + res1 = cv.utils.dumpInputArray(mat_data1) + self.assertEqual(res1, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3") + + mat_data2 = cv.Mat(mat_data1) + assert isinstance(mat_data2, cv.Mat) + assert isinstance(mat_data2, np.ndarray) + self.assertEqual(mat_data2.wrap_channels, True) # __array_finalize__ doesn't work + res2 = cv.utils.dumpInputArray(mat_data2) + self.assertEqual(res2, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=500 dims(-1)=3 size(-1)=[5 10 10] type(-1)=CV_64FC3") + + + def test_mat_wrap_channels_fail(self): + data = np.random.random([2, 3, 4, 520]) + + mat_data0 = cv.Mat(data) + assert isinstance(mat_data0, cv.Mat) + assert isinstance(mat_data0, np.ndarray) + self.assertEqual(mat_data0.wrap_channels, False) + res0 = cv.utils.dumpInputArray(mat_data0) + self.assertEqual(res0, "InputArray: empty()=false kind=0x00010000 flags=0x01010000 total(-1)=12480 dims(-1)=4 size(-1)=[2 3 4 520] type(-1)=CV_64FC1") + + with self.assertRaises(cv.error): + mat_data1 = cv.Mat(data, wrap_channels=True) # argument unable to wrap channels, too high (520 > CV_CN_MAX=512) + res1 = cv.utils.dumpInputArray(mat_data1) + print(mat_data1.__dict__) + print(res1) + + + def test_ufuncs(self): + data = np.arange(10) + mat_data = cv.Mat(data) + mat_data2 = 2 * mat_data + self.assertEqual(type(mat_data2), cv.Mat) + np.testing.assert_equal(2 * data, 2 * mat_data) + + + def test_comparison(self): + # Undefined behavior, do NOT use that. + # Behavior may be changed in the future + + data = np.ones((10, 10, 3)) + mat_wrapped = cv.Mat(data, wrap_channels=True) + mat_simple = cv.Mat(data) + np.testing.assert_equal(mat_wrapped, mat_simple) # ???: wrap_channels is not checked for now + np.testing.assert_equal(data, mat_simple) + np.testing.assert_equal(data, mat_wrapped) + + #self.assertEqual(mat_wrapped, mat_simple) # ??? + #self.assertTrue(mat_wrapped == mat_simple) # ??? + #self.assertTrue((mat_wrapped == mat_simple).all()) + + +except unittest.SkipTest as e: + + message = str(e) + + class TestSkip(unittest.TestCase): + def setUp(self): + self.skipTest('Skip tests: ' + message) + + def test_skip(): + pass + + pass + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index 4c98b92833..76803992dc 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -4,6 +4,12 @@ from __future__ import print_function import ctypes from functools import partial from collections import namedtuple +import sys + +if sys.version_info[0] < 3: + from collections import Sequence +else: + from collections.abc import Sequence import numpy as np import cv2 as cv @@ -464,6 +470,151 @@ class Arguments(NewOpenCVTests): with self.assertRaises((TypeError), msg=get_no_exception_msg(not_convertible)): _ = cv.utils.dumpRange(not_convertible) + def test_reserved_keywords_are_transformed(self): + default_lambda_value = 2 + default_from_value = 3 + format_str = "arg={}, lambda={}, from={}" + self.assertEqual( + cv.utils.testReservedKeywordConversion(20), format_str.format(20, default_lambda_value, default_from_value) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(10, lambda_=10), format_str.format(10, 10, default_from_value) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(10, from_=10), format_str.format(10, default_lambda_value, 10) + ) + self.assertEqual( + cv.utils.testReservedKeywordConversion(20, lambda_=-4, from_=12), format_str.format(20, -4, 12) + ) + + def test_parse_vector_int_convertible(self): + np.random.seed(123098765) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfInt) + arr = np.random.randint(-20, 20, 40).astype(np.int32).reshape(10, 2, 2) + int_min, int_max = get_limits(ctypes.c_int) + for convertible in ((int_min, 1, 2, 3, int_max), [40, 50], tuple(), + np.array([int_min, -10, 24, int_max], dtype=np.int32), + np.array([10, 230, 12], dtype=np.uint8), arr[:, 0, 1],): + expected = "[" + ", ".join(map(str, convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_int_not_convertible(self): + np.random.seed(123098765) + arr = np.random.randint(-20, 20, 40).astype(np.float).reshape(10, 2, 2) + int_min, int_max = get_limits(ctypes.c_int) + test_dict = {1: 2, 3: 10, 10: 20} + for not_convertible in ((int_min, 1, 2.5, 3, int_max), [True, 50], 'test', test_dict, + reversed([1, 2, 3]), + np.array([int_min, -10, 24, [1, 2]], dtype=np.object), + np.array([[1, 2], [3, 4]]), arr[:, 0, 1],): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfInt(not_convertible) + + def test_parse_vector_double_convertible(self): + np.random.seed(1230965) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfDouble) + arr = np.random.randint(-20, 20, 40).astype(np.int32).reshape(10, 2, 2) + for convertible in ((1, 2.12, 3.5), [40, 50], tuple(), + np.array([-10, 24], dtype=np.int32), + np.array([-12.5, 1.4], dtype=np.double), + np.array([10, 230, 12], dtype=np.float), arr[:, 0, 1], ): + expected = "[" + ", ".join(map(lambda v: "{:.2f}".format(v), convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_double_not_convertible(self): + test_dict = {1: 2, 3: 10, 10: 20} + for not_convertible in (('t', 'e', 's', 't'), [True, 50.55], 'test', test_dict, + np.array([-10.1, 24.5, [1, 2]], dtype=np.object), + np.array([[1, 2], [3, 4]]),): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfDouble(not_convertible) + + def test_parse_vector_rect_convertible(self): + np.random.seed(1238765) + try_to_convert = partial(self._try_to_convert, cv.utils.dumpVectorOfRect) + arr_of_rect_int32 = np.random.randint(5, 20, 4 * 3).astype(np.int32).reshape(3, 4) + arr_of_rect_cast = np.random.randint(10, 40, 4 * 5).astype(np.uint8).reshape(5, 4) + for convertible in (((1, 2, 3, 4), (10, -20, 30, 10)), arr_of_rect_int32, arr_of_rect_cast, + arr_of_rect_int32.astype(np.int8), [[5, 3, 1, 4]], + ((np.int8(4), np.uint8(10), np.int(32), np.int16(55)),)): + expected = "[" + ", ".join(map(lambda v: "[x={}, y={}, w={}, h={}]".format(*v), convertible)) + "]" + actual = try_to_convert(convertible) + self.assertEqual(expected, actual, + msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_parse_vector_rect_not_convertible(self): + np.random.seed(1238765) + arr = np.random.randint(5, 20, 4 * 3).astype(np.float).reshape(3, 4) + for not_convertible in (((1, 2, 3, 4), (10.5, -20, 30.1, 10)), arr, + [[5, 3, 1, 4], []], + ((np.float(4), np.uint8(10), np.int(32), np.int16(55)),)): + with self.assertRaises(TypeError, msg=get_no_exception_msg(not_convertible)): + _ = cv.utils.dumpVectorOfRect(not_convertible) + + def test_vector_general_return(self): + expected_number_of_mats = 5 + expected_shape = (10, 10, 3) + expected_type = np.uint8 + mats = cv.utils.generateVectorOfMat(5, 10, 10, cv.CV_8UC3) + self.assertTrue(isinstance(mats, tuple), + "Vector of Mats objects should be returned as tuple. Got: {}".format(type(mats))) + self.assertEqual(len(mats), expected_number_of_mats, "Returned array has wrong length") + for mat in mats: + self.assertEqual(mat.shape, expected_shape, "Returned Mat has wrong shape") + self.assertEqual(mat.dtype, expected_type, "Returned Mat has wrong elements type") + empty_mats = cv.utils.generateVectorOfMat(0, 10, 10, cv.CV_32FC1) + self.assertTrue(isinstance(empty_mats, tuple), + "Empty vector should be returned as empty tuple. Got: {}".format(type(mats))) + self.assertEqual(len(empty_mats), 0, "Vector of size 0 should be returned as tuple of length 0") + + def test_vector_fast_return(self): + expected_shape = (5, 4) + rects = cv.utils.generateVectorOfRect(expected_shape[0]) + self.assertTrue(isinstance(rects, np.ndarray), + "Vector of rectangles should be returned as numpy array. Got: {}".format(type(rects))) + self.assertEqual(rects.dtype, np.int32, "Vector of rectangles has wrong elements type") + self.assertEqual(rects.shape, expected_shape, "Vector of rectangles has wrong shape") + empty_rects = cv.utils.generateVectorOfRect(0) + self.assertTrue(isinstance(empty_rects, tuple), + "Empty vector should be returned as empty tuple. Got: {}".format(type(empty_rects))) + self.assertEqual(len(empty_rects), 0, "Vector of size 0 should be returned as tuple of length 0") + + expected_shape = (10,) + ints = cv.utils.generateVectorOfInt(expected_shape[0]) + self.assertTrue(isinstance(ints, np.ndarray), + "Vector of integers should be returned as numpy array. Got: {}".format(type(ints))) + self.assertEqual(ints.dtype, np.int32, "Vector of integers has wrong elements type") + self.assertEqual(ints.shape, expected_shape, "Vector of integers has wrong shape.") + + +class CanUsePurePythonModuleFunction(NewOpenCVTests): + def test_can_get_ocv_version(self): + import sys + if sys.version_info[0] < 3: + raise unittest.SkipTest('Python 2.x is not supported') + + self.assertEqual(cv.misc.get_ocv_version(), cv.__version__, + "Can't get package version using Python misc module") + + def test_native_method_can_be_patched(self): + import sys + + if sys.version_info[0] < 3: + raise unittest.SkipTest('Python 2.x is not supported') + + res = cv.utils.testOverwriteNativeMethod(10) + self.assertTrue(isinstance(res, Sequence), + msg="Overwritten method should return sequence. " + "Got: {} of type {}".format(res, type(res))) + self.assertSequenceEqual(res, (11, 10), + msg="Failed to overwrite native method") + res = cv.utils._native.testOverwriteNativeMethod(123) + self.assertEqual(res, 123, msg="Failed to call native method implementation") + class SamplesFindFile(NewOpenCVTests): diff --git a/modules/python/test/tests_common.py b/modules/python/test/tests_common.py index a938a8e2cb..2245722939 100644 --- a/modules/python/test/tests_common.py +++ b/modules/python/test/tests_common.py @@ -10,6 +10,7 @@ import random import argparse import numpy as np +#sys.OpenCV_LOADER_DEBUG = True import cv2 as cv # Python 3 moved urlopen to urllib.requests diff --git a/modules/ts/CMakeLists.txt b/modules/ts/CMakeLists.txt index f95bed0793..c1d249ea14 100644 --- a/modules/ts/CMakeLists.txt +++ b/modules/ts/CMakeLists.txt @@ -41,3 +41,9 @@ endif() if(NOT OPENCV_TESTS_CONFIG_STR STREQUAL "${__content}") file(WRITE "${OPENCV_TESTS_CONFIG_FILE}" "${OPENCV_TESTS_CONFIG_STR}") endif() + +if(OPENCV_DISABLE_THREAD_SUPPORT) + # This is required to disable threads in the ts module, as + # described in `ts_gtest.h`. + ocv_target_compile_definitions(${the_module} PUBLIC GTEST_HAS_PTHREAD=0) +endif() diff --git a/modules/ts/include/opencv2/ts.hpp b/modules/ts/include/opencv2/ts.hpp index 394bc6e0fa..2e7a241d8e 100644 --- a/modules/ts/include/opencv2/ts.hpp +++ b/modules/ts/include/opencv2/ts.hpp @@ -37,6 +37,7 @@ #include #include #include +#include #ifndef OPENCV_32BIT_CONFIGURATION diff --git a/modules/ts/src/ocl_perf.cpp b/modules/ts/src/ocl_perf.cpp index 8dacf219f6..fe521f2c00 100644 --- a/modules/ts/src/ocl_perf.cpp +++ b/modules/ts/src/ocl_perf.cpp @@ -70,7 +70,7 @@ void randu(InputOutputArray dst) cv::randu(dst, -128, 128); else if (dst.depth() == CV_16U) cv::randu(dst, 0, 1024); - else if (dst.depth() == CV_32F || dst.depth() == CV_64F) + else if (dst.depth() == CV_32F || dst.depth() == CV_64F || dst.depth() == CV_16F) cv::randu(dst, -1.0, 1.0); else if (dst.depth() == CV_16S || dst.depth() == CV_32S) cv::randu(dst, -4096, 4096); diff --git a/modules/ts/src/precomp.hpp b/modules/ts/src/precomp.hpp index a9d4404e3b..2725f63429 100644 --- a/modules/ts/src/precomp.hpp +++ b/modules/ts/src/precomp.hpp @@ -1,7 +1,9 @@ #include "opencv2/ts.hpp" #include #include "opencv2/core/utility.hpp" +#if !defined(__EMSCRIPTEN__) #include "opencv2/core/private.hpp" +#endif #ifdef GTEST_LINKED_AS_SHARED_LIBRARY #error ts module should not have GTEST_LINKED_AS_SHARED_LIBRARY defined diff --git a/modules/ts/src/ts.cpp b/modules/ts/src/ts.cpp index 061816d810..611ef64fb7 100644 --- a/modules/ts/src/ts.cpp +++ b/modules/ts/src/ts.cpp @@ -72,7 +72,9 @@ #if defined _WIN32 || defined WINCE # include #else +#if OPENCV_HAVE_FILESYSTEM_SUPPORT # include +#endif # include #endif diff --git a/modules/ts/src/ts_gtest.cpp b/modules/ts/src/ts_gtest.cpp index a65ef721a2..b3debd54d2 100644 --- a/modules/ts/src/ts_gtest.cpp +++ b/modules/ts/src/ts_gtest.cpp @@ -1067,6 +1067,7 @@ class GTEST_API_ UnitTestImpl { void AddTestInfo(Test::SetUpTestCaseFunc set_up_tc, Test::TearDownTestCaseFunc tear_down_tc, TestInfo* test_info) { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT // In order to support thread-safe death tests, we need to // remember the original working directory when the test program // was first invoked. We cannot do this in RUN_ALL_TESTS(), as @@ -1079,6 +1080,7 @@ class GTEST_API_ UnitTestImpl { GTEST_CHECK_(!original_working_dir_.IsEmpty()) << "Failed to get the current working directory."; } +#endif GetTestCase(test_info->test_case_name(), test_info->type_param(), @@ -9165,6 +9167,7 @@ static bool IsPathSeparator(char c) { // Returns the current working directory, or "" if unsuccessful. FilePath FilePath::GetCurrentDir() { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT // Windows CE doesn't have a current directory, so we just return // something reasonable. @@ -9183,6 +9186,9 @@ FilePath FilePath::GetCurrentDir() { # endif // GTEST_OS_NACL return FilePath(result == NULL ? "" : cwd); #endif // GTEST_OS_WINDOWS_MOBILE +#else // OPENCV_HAVE_FILESYSTEM_SUPPORT + return FilePath(""); +#endif // OPENCV_HAVE_FILESYSTEM_SUPPORT } // Returns a copy of the FilePath with the case-insensitive extension removed. @@ -9391,6 +9397,7 @@ bool FilePath::CreateDirectoriesRecursively() const { // directory for any reason, including if the parent directory does not // exist. Not named "CreateDirectory" because that's a macro on Windows. bool FilePath::CreateFolder() const { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT #if GTEST_OS_WINDOWS_MOBILE FilePath removed_sep(this->RemoveTrailingPathSeparator()); LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); @@ -9406,6 +9413,9 @@ bool FilePath::CreateFolder() const { return this->DirectoryExists(); // An error is OK if the directory exists. } return true; // No error. +#else // OPENCV_HAVE_FILESYSTEM_SUPPORT + return false; +#endif // OPENCV_HAVE_FILESYSTEM_SUPPORT } // If input name has a trailing separator character, remove it and return the diff --git a/modules/ts/src/ts_perf.cpp b/modules/ts/src/ts_perf.cpp index 2a9169fd13..5a42ca01cd 100644 --- a/modules/ts/src/ts_perf.cpp +++ b/modules/ts/src/ts_perf.cpp @@ -1297,7 +1297,7 @@ void TestBase::warmup(cv::InputOutputArray a, WarmUpType wtype) cv::randu(a, -128, 128); else if (depth == CV_16U) cv::randu(a, 0, 1024); - else if (depth == CV_32F || depth == CV_64F) + else if (depth == CV_32F || depth == CV_64F || depth == CV_16F) cv::randu(a, -1.0, 1.0); else if (depth == CV_16S || depth == CV_32S) cv::randu(a, -4096, 4096); diff --git a/modules/video/misc/java/test/TrackerCreateTest.java b/modules/video/misc/java/test/TrackerCreateTest.java index dad696bebf..83bbd0b5d5 100644 --- a/modules/video/misc/java/test/TrackerCreateTest.java +++ b/modules/video/misc/java/test/TrackerCreateTest.java @@ -1,7 +1,10 @@ package org.opencv.test.video; import org.opencv.core.Core; +import org.opencv.core.CvType; import org.opencv.core.CvException; +import org.opencv.core.Mat; +import org.opencv.core.Rect; import org.opencv.test.OpenCVTestCase; import org.opencv.video.Tracker; @@ -27,6 +30,10 @@ public class TrackerCreateTest extends OpenCVTestCase { public void testCreateTrackerMIL() { Tracker tracker = TrackerMIL.create(); + assert(tracker != null); + Mat mat = new Mat(100, 100, CvType.CV_8UC1); + Rect rect = new Rect(10, 10, 30, 30); + tracker.init(mat, rect); // should not crash (https://github.com/opencv/opencv/issues/19915) } } diff --git a/modules/video/perf/opencl/perf_bgfg_knn.cpp b/modules/video/perf/opencl/perf_bgfg_knn.cpp index 833db66d86..6e95e52b2f 100644 --- a/modules/video/perf/opencl/perf_bgfg_knn.cpp +++ b/modules/video/perf/opencl/perf_bgfg_knn.cpp @@ -19,7 +19,7 @@ typedef TestBaseWithParam KNN_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/vid } OCL_PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/opencl/perf_bgfg_mog2.cpp b/modules/video/perf/opencl/perf_bgfg_mog2.cpp index 8e5f095257..3de03fa1ee 100644 --- a/modules/video/perf/opencl/perf_bgfg_mog2.cpp +++ b/modules/video/perf/opencl/perf_bgfg_mog2.cpp @@ -19,7 +19,7 @@ typedef TestBaseWithParam MOG2_GetBackgroundImage; using namespace opencv_test; -OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -51,8 +51,8 @@ OCL_PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/v } OCL_PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_knn.cpp b/modules/video/perf/perf_bgfg_knn.cpp index 23bb1fe0a5..42d6368fd3 100644 --- a/modules/video/perf/perf_bgfg_knn.cpp +++ b/modules/video/perf/perf_bgfg_knn.cpp @@ -14,7 +14,7 @@ typedef tuple VideoKNNParamType; typedef TestBaseWithParam KNN_Apply; typedef TestBaseWithParam KNN_GetBackgroundImage; -PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(KNN_Apply, KNN, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoKNNParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(KNN_Apply, KNN, Combine(Values("gpu/video/768x576.avi", "gpu/video/1 } PERF_TEST_P(KNN_GetBackgroundImage, KNN, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoKNNParamType params = GetParam(); diff --git a/modules/video/perf/perf_bgfg_mog2.cpp b/modules/video/perf/perf_bgfg_mog2.cpp index f911a9c74e..610138f85c 100644 --- a/modules/video/perf/perf_bgfg_mog2.cpp +++ b/modules/video/perf/perf_bgfg_mog2.cpp @@ -14,7 +14,7 @@ typedef tuple VideoMOG2ParamType; typedef TestBaseWithParam MOG2_Apply; typedef TestBaseWithParam MOG2_GetBackgroundImage; -PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video/1920x1080.avi"), Values(1,3))) +PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("cv/video/768x576.avi", "cv/video/1920x1080.avi"), Values(1,3))) { VideoMOG2ParamType params = GetParam(); @@ -46,8 +46,8 @@ PERF_TEST_P(MOG2_Apply, Mog2, Combine(Values("gpu/video/768x576.avi", "gpu/video } PERF_TEST_P(MOG2_GetBackgroundImage, Mog2, Values( - std::make_pair("gpu/video/768x576.avi", 5), - std::make_pair("gpu/video/1920x1080.avi", 5))) + std::make_pair("cv/video/768x576.avi", 5), + std::make_pair("cv/video/1920x1080.avi", 5))) { VideoMOG2ParamType params = GetParam(); diff --git a/modules/video/test/test_trackers.impl.hpp b/modules/video/test/test_trackers.impl.hpp index 7fce94e748..5ae98d1eb8 100644 --- a/modules/video/test/test_trackers.impl.hpp +++ b/modules/video/test/test_trackers.impl.hpp @@ -46,7 +46,8 @@ float calcDistance(const Rect& a, const Rect& b) { Point2f p_a((float)(a.x + a.width / 2), (float)(a.y + a.height / 2)); Point2f p_b((float)(b.x + b.width / 2), (float)(b.y + b.height / 2)); - return sqrt(pow(p_a.x - p_b.x, 2) + pow(p_a.y - p_b.y, 2)); + Point2f diff = p_a - p_b; + return sqrt(diff.dot(diff)); } float calcOverlap(const Rect& a, const Rect& b) diff --git a/modules/videoio/CMakeLists.txt b/modules/videoio/CMakeLists.txt index 534fcf0e37..98127d1c98 100644 --- a/modules/videoio/CMakeLists.txt +++ b/modules/videoio/CMakeLists.txt @@ -1,5 +1,10 @@ +set(VIDEOIO_ENABLE_PLUGINS_DEFAULT ON) +if(EMSCRIPTEN OR IOS OR WINRT) + set(VIDEOIO_ENABLE_PLUGINS_DEFAULT OFF) +endif() + set(VIDEOIO_PLUGIN_LIST "" CACHE STRING "List of videoio backends to be compiled as plugins (ffmpeg, gstreamer, mfx, msmf or special value 'all')") -set(VIDEOIO_ENABLE_PLUGINS "ON" CACHE BOOL "Allow building and using of videoio plugins") +set(VIDEOIO_ENABLE_PLUGINS "${VIDEOIO_ENABLE_PLUGINS_DEFAULT}" CACHE BOOL "Allow building and using of videoio plugins") mark_as_advanced(VIDEOIO_PLUGIN_LIST VIDEOIO_ENABLE_PLUGINS) string(REPLACE "," ";" VIDEOIO_PLUGIN_LIST "${VIDEOIO_PLUGIN_LIST}") # support comma-separated list (,) too diff --git a/modules/videoio/cmake/detect_android_camera.cmake b/modules/videoio/cmake/detect_android_camera.cmake index ded4c91ccf..a465751334 100644 --- a/modules/videoio/cmake/detect_android_camera.cmake +++ b/modules/videoio/cmake/detect_android_camera.cmake @@ -4,5 +4,3 @@ if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER 23) set(libs "-landroid -llog -lcamera2ndk") ocv_add_external_target(android_native_camera "" "${libs}" "HAVE_ANDROID_NATIVE_CAMERA") endif() - -set(HAVE_ANDROID_NATIVE_CAMERA ${HAVE_ANDROID_NATIVE_CAMERA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_android_mediandk.cmake b/modules/videoio/cmake/detect_android_mediandk.cmake index edfb4bbbc5..cee64ab549 100644 --- a/modules/videoio/cmake/detect_android_mediandk.cmake +++ b/modules/videoio/cmake/detect_android_mediandk.cmake @@ -4,5 +4,3 @@ if(ANDROID AND ANDROID_NATIVE_API_LEVEL GREATER 20) set(libs "-landroid -llog -lmediandk") ocv_add_external_target(android_mediandk "" "${libs}" "HAVE_ANDROID_MEDIANDK") endif() - -set(HAVE_ANDROID_MEDIANDK ${HAVE_ANDROID_MEDIANDK} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_aravis.cmake b/modules/videoio/cmake/detect_aravis.cmake index 79d6a217db..e7b3828993 100644 --- a/modules/videoio/cmake/detect_aravis.cmake +++ b/modules/videoio/cmake/detect_aravis.cmake @@ -21,7 +21,7 @@ if(NOT HAVE_ARAVIS_API) string(REGEX REPLACE ".*ARAVIS_MAJOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*ARAVIS_MINOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*ARAVIS_MICRO_VERSION[^0-9]+([0-9]+).*" "\\1" ver_micro "${ver_strings}") - set(ARAVIS_VERSION "${ver_major}.${ver_minor}.${ver_micro}" PARENT_SCOPE) # informational + set(ARAVIS_VERSION "${ver_major}.${ver_minor}.${ver_micro}") # informational set(ARAVIS_INCLUDE_DIRS "${ARAVIS_INCLUDE}") set(ARAVIS_LIBRARIES "${ARAVIS_LIBRARY}") endif() @@ -30,5 +30,3 @@ endif() if(HAVE_ARAVIS_API) ocv_add_external_target(aravis "${ARAVIS_INCLUDE_DIRS}" "${ARAVIS_LIBRARIES}" "HAVE_ARAVIS_API") endif() - -set(HAVE_ARAVIS_API ${HAVE_ARAVIS_API} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_avfoundation.cmake b/modules/videoio/cmake/detect_avfoundation.cmake index a341f587a1..2da4fabfab 100644 --- a/modules/videoio/cmake/detect_avfoundation.cmake +++ b/modules/videoio/cmake/detect_avfoundation.cmake @@ -14,5 +14,3 @@ if(APPLE) endif() ocv_add_external_target(avfoundation "" "${libs}" "HAVE_AVFOUNDATION") endif() - -set(HAVE_AVFOUNDATION ${HAVE_AVFOUNDATION} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_dc1394.cmake b/modules/videoio/cmake/detect_dc1394.cmake index 51ab2dd80e..8bcee4bf70 100644 --- a/modules/videoio/cmake/detect_dc1394.cmake +++ b/modules/videoio/cmake/detect_dc1394.cmake @@ -2,7 +2,6 @@ if(NOT HAVE_DC1394_2 AND PKG_CONFIG_FOUND) ocv_check_modules(DC1394_2 libdc1394-2) if(DC1394_2_FOUND) - set(DC1394_2_VERSION "${DC1394_2_VERSION}" PARENT_SCOPE) # informational set(HAVE_DC1394_2 TRUE) endif() endif() @@ -20,12 +19,10 @@ if(NOT HAVE_DC1394_2) set(HAVE_DC1394_2 TRUE) set(DC1394_2_INCLUDE_DIRS "${DC1394_INCLUDE}") set(DC1394_2_LIBRARIES "${DC1394_LIBRARY}") - set(DC1394_2_VERSION "unknown" PARENT_SCOPE) # informational + set(DC1394_2_VERSION "unknown") # informational endif() endif() if(HAVE_DC1394_2) ocv_add_external_target(dc1394_2 "${DC1394_2_INCLUDE_DIRS}" "${DC1394_2_LIBRARIES}" "HAVE_DC1394_2") endif() - -set(HAVE_DC1394_2 ${HAVE_DC1394_2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_dshow.cmake b/modules/videoio/cmake/detect_dshow.cmake index 3f41b3fd34..928134c08c 100644 --- a/modules/videoio/cmake/detect_dshow.cmake +++ b/modules/videoio/cmake/detect_dshow.cmake @@ -10,5 +10,3 @@ endif() if(HAVE_DSHOW) ocv_add_external_target(dshow "" "" "HAVE_DSHOW") endif() - -set(HAVE_DSHOW ${HAVE_DSHOW} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ffmpeg.cmake b/modules/videoio/cmake/detect_ffmpeg.cmake index 58de4b9515..c33eaf221b 100644 --- a/modules/videoio/cmake/detect_ffmpeg.cmake +++ b/modules/videoio/cmake/detect_ffmpeg.cmake @@ -14,11 +14,6 @@ if(NOT HAVE_FFMPEG AND WIN32 AND NOT ARM AND NOT OPENCV_FFMPEG_SKIP_DOWNLOAD) download_win_ffmpeg(FFMPEG_CMAKE_SCRIPT) if(FFMPEG_CMAKE_SCRIPT) include("${FFMPEG_CMAKE_SCRIPT}") - set(FFMPEG_libavcodec_VERSION ${FFMPEG_libavcodec_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavformat_VERSION ${FFMPEG_libavformat_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavutil_VERSION ${FFMPEG_libavutil_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libswscale_VERSION ${FFMPEG_libswscale_VERSION} PARENT_SCOPE) # info - set(FFMPEG_libavresample_VERSION ${FFMPEG_libavresample_VERSION} PARENT_SCOPE) # info set(HAVE_FFMPEG TRUE) set(HAVE_FFMPEG_WRAPPER TRUE) endif() @@ -132,5 +127,3 @@ elseif(HAVE_FFMPEG) ocv_add_external_target(ffmpeg.plugin_deps "${__plugin_include_dirs}" "${__plugin_include_libs}" "${__plugin_defines}") endif() endif() - -set(HAVE_FFMPEG ${HAVE_FFMPEG} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_gphoto.cmake b/modules/videoio/cmake/detect_gphoto.cmake index 0d6f1212eb..2cb23c0033 100644 --- a/modules/videoio/cmake/detect_gphoto.cmake +++ b/modules/videoio/cmake/detect_gphoto.cmake @@ -9,5 +9,3 @@ endif() if(HAVE_GPHOTO2) ocv_add_external_target(gphoto2 "${GPHOTO2_INCLUDE_DIRS}" "${GPHOTO2_LIBRARIES}" "HAVE_GPHOTO2") endif() - -set(HAVE_GPHOTO2 ${HAVE_GPHOTO2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_gstreamer.cmake b/modules/videoio/cmake/detect_gstreamer.cmake index 2198786161..47ea7a0b30 100644 --- a/modules/videoio/cmake/detect_gstreamer.cmake +++ b/modules/videoio/cmake/detect_gstreamer.cmake @@ -69,7 +69,7 @@ if(NOT HAVE_GSTREAMER AND WIN32) string(REGEX REPLACE ".*GST_VERSION_MAJOR[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*GST_VERSION_MINOR[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*GST_VERSION_MICRO[^0-9]+([0-9]+).*" "\\1" ver_micro "${ver_strings}") - set(GSTREAMER_VERSION "${ver_major}.${ver_minor}.${ver_micro}" PARENT_SCOPE) # informational + set(GSTREAMER_VERSION "${ver_major}.${ver_minor}.${ver_micro}") # informational set(HAVE_GSTREAMER TRUE) set(GSTREAMER_LIBRARIES ${GSTREAMER_gstreamer_LIBRARY} @@ -95,7 +95,7 @@ if(NOT HAVE_GSTREAMER AND PKG_CONFIG_FOUND) ocv_check_modules(GSTREAMER_video gstreamer-video-1.0) if(GSTREAMER_base_FOUND AND GSTREAMER_app_FOUND AND GSTREAMER_riff_FOUND AND GSTREAMER_pbutils_FOUND AND GSTREAMER_video_FOUND) set(HAVE_GSTREAMER TRUE) - set(GSTREAMER_VERSION ${GSTREAMER_base_VERSION} PARENT_SCOPE) # informational + set(GSTREAMER_VERSION ${GSTREAMER_base_VERSION}) # informational set(GSTREAMER_LIBRARIES ${GSTREAMER_base_LIBRARIES} ${GSTREAMER_app_LIBRARIES} ${GSTREAMER_riff_LIBRARIES} ${GSTREAMER_pbutils_LIBRARIES} ${GSTREAMER_video_LIBRARIES}) set(GSTREAMER_INCLUDE_DIRS ${GSTREAMER_base_INCLUDE_DIRS} ${GSTREAMER_app_INCLUDE_DIRS} ${GSTREAMER_riff_INCLUDE_DIRS} ${GSTREAMER_pbutils_INCLUDE_DIRS} ${GSTREAMER_video_INCLUDE_DIRS}) endif() @@ -104,5 +104,3 @@ endif() if(HAVE_GSTREAMER) ocv_add_external_target(gstreamer "${GSTREAMER_INCLUDE_DIRS}" "${GSTREAMER_LIBRARIES}" "HAVE_GSTREAMER") endif() - -set(HAVE_GSTREAMER ${HAVE_GSTREAMER} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ios.cmake b/modules/videoio/cmake/detect_ios.cmake index c75426060b..8d48dd6f3b 100644 --- a/modules/videoio/cmake/detect_ios.cmake +++ b/modules/videoio/cmake/detect_ios.cmake @@ -11,5 +11,3 @@ if(APPLE AND IOS) "-framework UIKit") ocv_add_external_target(cap_ios "" "${libs}" "HAVE_CAP_IOS") endif() - -set(HAVE_CAP_IOS ${HAVE_CAP_IOS} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_msdk.cmake b/modules/videoio/cmake/detect_msdk.cmake index d035c3f5cc..83701425e1 100644 --- a/modules/videoio/cmake/detect_msdk.cmake +++ b/modules/videoio/cmake/detect_msdk.cmake @@ -70,5 +70,3 @@ if(HAVE_MFX) list(APPEND MFX_DEFS "HAVE_MFX") ocv_add_external_target(mediasdk "${MFX_INCLUDE_DIRS}" "${MFX_LIBRARIES}" "${MFX_DEFS}") endif() - -set(HAVE_MFX ${HAVE_MFX} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_msmf.cmake b/modules/videoio/cmake/detect_msmf.cmake index a1c91dab67..aebc226bcf 100644 --- a/modules/videoio/cmake/detect_msmf.cmake +++ b/modules/videoio/cmake/detect_msmf.cmake @@ -20,6 +20,3 @@ if(HAVE_MSMF) endif() ocv_add_external_target(msmf "" "" "${defs}") endif() - -set(HAVE_MSMF ${HAVE_MSMF} PARENT_SCOPE) -set(HAVE_MSMF_DXVA ${HAVE_MSMF_DXVA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_openni2.cmake b/modules/videoio/cmake/detect_openni2.cmake index 76c31454da..54a5c62bed 100644 --- a/modules/videoio/cmake/detect_openni2.cmake +++ b/modules/videoio/cmake/detect_openni2.cmake @@ -42,8 +42,6 @@ if(HAVE_OPENNI2) string(REGEX REPLACE ".*ONI_VERSION_MAJOR[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*ONI_VERSION_MINOR[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*ONI_VERSION_MAINTENANCE[^0-9]+([0-9]+).*" "\\1" ver_maint "${ver_strings}") - set(OPENNI2_VERSION "${ver_major}.${ver_minor}.${ver_maint}" PARENT_SCOPE) # informational + set(OPENNI2_VERSION "${ver_major}.${ver_minor}.${ver_maint}") # informational ocv_add_external_target(openni2 "${OPENNI2_INCLUDE_DIRS}" "${OPENNI2_LIBRARIES}" "HAVE_OPENNI2") endif() - -set(HAVE_OPENNI2 ${HAVE_OPENNI2} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_pvapi.cmake b/modules/videoio/cmake/detect_pvapi.cmake index a0f4673fdc..f2c6d4bcea 100644 --- a/modules/videoio/cmake/detect_pvapi.cmake +++ b/modules/videoio/cmake/detect_pvapi.cmake @@ -19,5 +19,3 @@ endif() if(HAVE_PVAPI) ocv_add_external_target(pvapi "${PVAPI_INCLUDE}" "${PVAPI_LIBRARY}" "HAVE_PVAPI") endif() - -set(HAVE_PVAPI ${HAVE_PVAPI} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_realsense.cmake b/modules/videoio/cmake/detect_realsense.cmake index 32e5e02c9e..065f548830 100644 --- a/modules/videoio/cmake/detect_realsense.cmake +++ b/modules/videoio/cmake/detect_realsense.cmake @@ -4,7 +4,7 @@ if(NOT HAVE_LIBREALSENSE) find_package(realsense2 QUIET) if(realsense2_FOUND) set(HAVE_LIBREALSENSE TRUE) - set(LIBREALSENSE_VERSION "${realsense2_VERSION}" PARENT_SCOPE) # informational + set(LIBREALSENSE_VERSION "${realsense2_VERSION}") # informational ocv_add_external_target(librealsense "" "${realsense2_LIBRARY}" "HAVE_LIBREALSENSE") endif() endif() @@ -20,7 +20,7 @@ if(NOT HAVE_LIBREALSENSE) string(REGEX REPLACE ".*RS2_API_MAJOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_major "${ver_strings}") string(REGEX REPLACE ".*RS2_API_MINOR_VERSION[^0-9]+([0-9]+).*" "\\1" ver_minor "${ver_strings}") string(REGEX REPLACE ".*RS2_API_PATCH_VERSION[^0-9]+([0-9]+).*" "\\1" ver_patch "${ver_strings}") - set(LIBREALSENSE_VERSION "${ver_major}.${ver_minor}.${ver_patch}" PARENT_SCOPE) # informational + set(LIBREALSENSE_VERSION "${ver_major}.${ver_minor}.${ver_patch}") # informational ocv_add_external_target(librealsense "${LIBREALSENSE_INCLUDE_DIR}" "${LIBREALSENSE_LIBRARIES}" "HAVE_LIBREALSENSE") endif() endif() diff --git a/modules/videoio/cmake/detect_ueye.cmake b/modules/videoio/cmake/detect_ueye.cmake index 495e9c2450..9428f9e596 100644 --- a/modules/videoio/cmake/detect_ueye.cmake +++ b/modules/videoio/cmake/detect_ueye.cmake @@ -21,5 +21,3 @@ unset(_WIN_LIB_SUFFIX) if(HAVE_UEYE) ocv_add_external_target(ueye "${UEYE_INCLUDE}" "${UEYE_LIBRARY}" "HAVE_UEYE") endif() - -set(HAVE_UEYE ${HAVE_UEYE} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_v4l.cmake b/modules/videoio/cmake/detect_v4l.cmake index 05b73b003c..e413dae9ca 100644 --- a/modules/videoio/cmake/detect_v4l.cmake +++ b/modules/videoio/cmake/detect_v4l.cmake @@ -15,5 +15,3 @@ if(NOT HAVE_V4L) ocv_add_external_target(v4l "" "" "${defs}") endif() endif() - -set(HAVE_V4L ${HAVE_V4L} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_ximea.cmake b/modules/videoio/cmake/detect_ximea.cmake index 9cf295e352..7521e619b0 100644 --- a/modules/videoio/cmake/detect_ximea.cmake +++ b/modules/videoio/cmake/detect_ximea.cmake @@ -28,5 +28,3 @@ endif() if(HAVE_XIMEA) ocv_add_external_target(ximea "${XIMEA_INCLUDE}" "${XIMEA_LIBRARY}" "HAVE_XIMEA") endif() - -set(HAVE_XIMEA ${HAVE_XIMEA} PARENT_SCOPE) diff --git a/modules/videoio/cmake/detect_xine.cmake b/modules/videoio/cmake/detect_xine.cmake index 3e1f3010a4..0a6f642353 100644 --- a/modules/videoio/cmake/detect_xine.cmake +++ b/modules/videoio/cmake/detect_xine.cmake @@ -5,5 +5,3 @@ endif() if(HAVE_XINE) ocv_add_external_target(xine "${XINE_INCLUDE_DIRS}" "${XINE_LIBRARIES}" "HAVE_XINE") endif() - -set(HAVE_XINE ${HAVE_XINE} PARENT_SCOPE) diff --git a/modules/videoio/cmake/init.cmake b/modules/videoio/cmake/init.cmake index 310df2d249..af664f94df 100644 --- a/modules/videoio/cmake/init.cmake +++ b/modules/videoio/cmake/init.cmake @@ -1,20 +1,13 @@ -include(FindPkgConfig) - -# FIXIT: stop using PARENT_SCOPE in dependencies -if(PROJECT_NAME STREQUAL "OpenCV") - macro(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endmacro() -else() - function(add_backend backend_id cond_var) - if(${cond_var}) - include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") - endif() - endfunction() +if(NOT PROJECT_NAME STREQUAL "OpenCV") + include(FindPkgConfig) endif() +macro(add_backend backend_id cond_var) + if(${cond_var}) + include("${CMAKE_CURRENT_LIST_DIR}/detect_${backend_id}.cmake") + endif() +endmacro() + add_backend("ffmpeg" WITH_FFMPEG) add_backend("gstreamer" WITH_GSTREAMER) add_backend("v4l" WITH_V4L) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 348448bda7..3276d0d5e4 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -186,6 +186,9 @@ enum VideoCaptureProperties { CAP_PROP_HW_ACCELERATION=50, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in cv::VideoCapture constructor / .open() method. Default value is backend-specific. CAP_PROP_HW_DEVICE =51, //!< (**open-only**) Hardware device index (select GPU if multiple available). Device enumeration is acceleration type specific. CAP_PROP_HW_ACCELERATION_USE_OPENCL=52, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between HW accelerated decoder and cv::UMat. + CAP_PROP_OPEN_TIMEOUT_MSEC=53, //!< (**open-only**) timeout in milliseconds for opening a video capture (applicable for FFmpeg back-end only) + CAP_PROP_READ_TIMEOUT_MSEC=54, //!< (**open-only**) timeout in milliseconds for reading from a video capture (applicable for FFmpeg back-end only) + CAP_PROP_STREAM_OPEN_TIME_USEC =55, // getPluginCandidates(const std::string& baseName) continue; std::vector candidates; cv::glob(utils::fs::join(path, plugin_expr), candidates); + // Prefer candisates with higher versions + // TODO: implemented accurate versions-based comparator + std::sort(candidates.begin(), candidates.end(), std::greater()); CV_LOG_INFO(NULL, " - " << path << ": " << candidates.size()); copy(candidates.begin(), candidates.end(), back_inserter(results)); } diff --git a/modules/videoio/src/cap.cpp b/modules/videoio/src/cap.cpp index f8967514ed..fa958e2c8f 100644 --- a/modules/videoio/src/cap.cpp +++ b/modules/videoio/src/cap.cpp @@ -122,7 +122,11 @@ bool VideoCapture::open(const String& filename, int apiPreference, const std::ve const VideoBackendInfo& info = backends[i]; if (apiPreference == CAP_ANY || apiPreference == info.id) { - + if (!info.backendFactory) + { + CV_LOG_DEBUG(NULL, "VIDEOIO(" << info.name << "): factory is not available (plugins require filesystem support)"); + continue; + } CV_CAPTURE_LOG_DEBUG(NULL, cv::format("VIDEOIO(%s): trying capture filename='%s' ...", info.name, filename.c_str())); @@ -232,10 +236,14 @@ bool VideoCapture::open(int cameraNum, int apiPreference, const std::vector const VideoBackendInfo& info = backends[i]; if (apiPreference == CAP_ANY || apiPreference == info.id) { + if (!info.backendFactory) + { + CV_LOG_DEBUG(NULL, "VIDEOIO(" << info.name << "): factory is not available (plugins require filesystem support)"); + continue; + } CV_CAPTURE_LOG_DEBUG(NULL, cv::format("VIDEOIO(%s): trying capture cameraNum=%d ...", info.name, cameraNum)); - CV_Assert(!info.backendFactory.empty()); const Ptr backend = info.backendFactory->getBackend(); if (!backend.empty()) diff --git a/modules/videoio/src/cap_android_camera.cpp b/modules/videoio/src/cap_android_camera.cpp index b369a12a68..5952b6f08c 100644 --- a/modules/videoio/src/cap_android_camera.cpp +++ b/modules/videoio/src/cap_android_camera.cpp @@ -304,8 +304,8 @@ public: AImage_getPlaneRowStride(image.get(), 0, &yStride); AImage_getPlaneRowStride(image.get(), 1, &uvStride); AImage_getPlaneData(image.get(), 0, &yPixel, &yLen); - AImage_getPlaneData(image.get(), 1, &vPixel, &vLen); - AImage_getPlaneData(image.get(), 2, &uPixel, &uLen); + AImage_getPlaneData(image.get(), 1, &uPixel, &uLen); + AImage_getPlaneData(image.get(), 2, &vPixel, &vLen); AImage_getPlanePixelStride(image.get(), 1, &uvPixelStride); if ( (uvPixelStride == 2) && (vPixel == uPixel + 1) && (yLen == frameWidth * frameHeight) && (uLen == ((yLen / 2) - 1)) && (vLen == uLen) ) { @@ -313,7 +313,7 @@ public: if (fourCC == FOURCC_UNKNOWN) { fourCC = FOURCC_NV21; } - } else if ( (uvPixelStride == 1) && (vPixel = uPixel + uLen) && (yLen == frameWidth * frameHeight) && (uLen == yLen / 4) && (vLen == uLen) ) { + } else if ( (uvPixelStride == 1) && (vPixel == uPixel + uLen) && (yLen == frameWidth * frameHeight) && (uLen == yLen / 4) && (vLen == uLen) ) { colorFormat = COLOR_FormatYUV420Planar; if (fourCC == FOURCC_UNKNOWN) { fourCC = FOURCC_YV12; diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 1e73cb8fc8..de072ef290 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -183,8 +183,8 @@ extern "C" { #endif #if USE_AV_INTERRUPT_CALLBACK -#define LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS 30000 -#define LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS 30000 +#define LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS 30000 #ifdef _WIN32 // http://stackoverflow.com/questions/5404277/porting-clock-gettime-to-windows @@ -523,6 +523,8 @@ struct CvCapture_FFMPEG AVDictionary *dict; #if USE_AV_INTERRUPT_CALLBACK + int open_timeout; + int read_timeout; AVInterruptCallbackMetadata interrupt_metadata; #endif @@ -569,6 +571,11 @@ void CvCapture_FFMPEG::init() #endif dict = NULL; +#if USE_AV_INTERRUPT_CALLBACK + open_timeout = LIBAVFORMAT_INTERRUPT_OPEN_DEFAULT_TIMEOUT_MS; + read_timeout = LIBAVFORMAT_INTERRUPT_READ_DEFAULT_TIMEOUT_MS; +#endif + rawMode = false; rawModeInitialized = false; memset(&packet_filtered, 0, sizeof(packet_filtered)); @@ -928,6 +935,16 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& if (params.has(CAP_PROP_HW_ACCELERATION_USE_OPENCL)) { use_opencl = params.get(CAP_PROP_HW_ACCELERATION_USE_OPENCL); } +#if USE_AV_INTERRUPT_CALLBACK + if (params.has(CAP_PROP_OPEN_TIMEOUT_MSEC)) + { + open_timeout = params.get(CAP_PROP_OPEN_TIMEOUT_MSEC); + } + if (params.has(CAP_PROP_READ_TIMEOUT_MSEC)) + { + read_timeout = params.get(CAP_PROP_READ_TIMEOUT_MSEC); + } +#endif if (params.warnUnusedParameters()) { CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in .open(), see logger INFO channel for details. Bailout"); @@ -937,7 +954,7 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& #if USE_AV_INTERRUPT_CALLBACK /* interrupt callback */ - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_OPEN_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = open_timeout; get_monotonic_time(&interrupt_metadata.value); ic = avformat_alloc_context(); @@ -1282,7 +1299,7 @@ bool CvCapture_FFMPEG::grabFrame() #if USE_AV_INTERRUPT_CALLBACK // activate interrupt callback get_monotonic_time(&interrupt_metadata.value); - interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS; + interrupt_metadata.timeout_after_ms = read_timeout; #endif #if USE_AV_SEND_FRAME_API @@ -1574,6 +1591,9 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const case CAP_PROP_HW_ACCELERATION_USE_OPENCL: return static_cast(use_opencl); #endif // USE_AV_HW_CODECS + case CAP_PROP_STREAM_OPEN_TIME_USEC: + //ic->start_time_realtime is in microseconds + return ((double)ic->start_time_realtime); default: break; } diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 60ecf6611a..dd91bf1db4 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -221,10 +221,10 @@ private: start_loop = utils::getConfigurationParameterBool("OPENCV_VIDEOIO_GSTREAMER_START_MAINLOOP", false); GSafePtr err; - gst_init_check(NULL, NULL, err.getRef()); - if (err) + gboolean gst_init_res = gst_init_check(NULL, NULL, err.getRef()); + if (!gst_init_res) { - CV_WARN("Can't initialize GStreamer: " << err->message); + CV_WARN("Can't initialize GStreamer: " << (err ? err->message : "")); isFailed = true; return; } @@ -475,8 +475,9 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst) // video/x-raw, format=I420 -> 8bit, 1 channel (height is 1.5x larger than true height) // video/x-bayer -> 8bit, 1 channel // image/jpeg -> 8bit, mjpeg: buffer_size x 1 x 1 + // video/x-raw, format=GRAY16_LE (BE) -> 16 bit, 1 channel + // video/x-raw, format={BGRA, RGBA, BGRx, RGBx} -> 8bit, 4 channels // bayer data is never decoded, the user is responsible for that - // everything is 8 bit, so we just test the caps for bit depth Size sz = Size(frame_width, frame_height); guint n_planes = GST_VIDEO_INFO_N_PLANES(&info); if (name == "video/x-raw") @@ -507,6 +508,24 @@ bool GStreamerCapture::retrieveFrame(int, OutputArray dst) src.copyTo(dst); return true; } + else if (format == "GRAY16_LE" || format == "GRAY16_BE") + { + CV_CheckEQ((int)n_planes, 1, ""); + size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); + CV_CheckGE(step, (size_t)frame_width, ""); + Mat src(sz, CV_16UC1, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); + src.copyTo(dst); + return true; + } + else if (format == "BGRA" || format == "RGBA" || format == "BGRX" || format == "RGBX") + { + CV_CheckEQ((int)n_planes, 1, ""); + size_t step = GST_VIDEO_INFO_PLANE_STRIDE(&info, 0); + CV_CheckGE(step, (size_t)frame_width, ""); + Mat src(sz, CV_8UC4, map_info.data + GST_VIDEO_INFO_PLANE_OFFSET(&info, 0), step); + src.copyTo(dst); + return true; + } else if (format == "UYVY" || format == "YUY2" || format == "YVYU") { CV_CheckEQ((int)n_planes, 1, ""); @@ -841,7 +860,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam } else { - CV_WARN("Error opening file: " << filename << " (" << err->message << ")"); + CV_WARN("Error opening file: " << filename << " (" << (err ? err->message : "") << ")"); return false; } } @@ -849,9 +868,9 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam { GSafePtr err; uridecodebin.attach(gst_parse_launch(filename, err.getRef())); - if (err) + if (!uridecodebin) { - CV_WARN("Error opening bin: " << err->message); + CV_WARN("Error opening bin: " << (err ? err->message : "")); return false; } manualpipeline = true; @@ -1008,7 +1027,7 @@ bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParam sink_pad.attach(gst_element_get_static_pad(sink, "sink")); peer_caps.attach(gst_pad_peer_query_caps(sink_pad, NULL)); if (!gst_caps_can_intersect(caps, peer_caps)) { - caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420}")); + caps.attach(gst_caps_from_string("video/x-raw, format=(string){UYVY,YUY2,YVYU,NV12,NV21,YV12,I420,BGRA,RGBA,BGRx,RGBx,GRAY16_LE,GRAY16_BE}")); CV_Assert(caps); } } @@ -2054,7 +2073,7 @@ void handleMessage(GstElement * pipeline) gst_message_parse_error(msg, err.getRef(), debug.getRef()); GSafePtr name; name.attach(gst_element_get_name(GST_MESSAGE_SRC (msg))); CV_WARN("Embedded video playback halted; module " << name.get() << - " reported: " << err->message); + " reported: " << (err ? err->message : "")); CV_LOG_DEBUG(NULL, "GStreamer debug: " << debug.get()); gst_element_set_state(GST_ELEMENT(pipeline), GST_STATE_NULL); diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 73288c3d03..9e45fd1bac 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -708,9 +708,10 @@ bool CvCapture_MSMF::initStream(DWORD streamID, const MediaType& mt) _ComPtr CvCapture_MSMF::getDefaultSourceConfig(UINT32 num) { CV_Assert(num > 0); + const bool OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS = utils::getConfigurationParameterBool("OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS", true); _ComPtr res; if (FAILED(MFCreateAttributes(&res, num)) || - FAILED(res->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, true)) || + FAILED(res->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS)) || FAILED(res->SetUINT32(MF_SOURCE_READER_DISABLE_DXVA, false)) || FAILED(res->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, false)) || FAILED(res->SetUINT32(MF_SOURCE_READER_ENABLE_ADVANCED_VIDEO_PROCESSING, true)) diff --git a/modules/videoio/src/cap_ueye.cpp b/modules/videoio/src/cap_ueye.cpp index eadba15546..efc2cb990e 100644 --- a/modules/videoio/src/cap_ueye.cpp +++ b/modules/videoio/src/cap_ueye.cpp @@ -171,6 +171,13 @@ double VideoCapture_uEye::getProperty(int property_id) const case CAP_PROP_FPS: value = fps; break; + case CAP_PROP_EXPOSURE: + ASSERT_UEYE(is_Exposure(cam_id, IS_EXPOSURE_CMD_GET_EXPOSURE, (void*)&value, sizeof(value))); + break; + case CAP_PROP_GAIN: + auto gain = is_SetHWGainFactor(cam_id, IS_GET_MASTER_GAIN_FACTOR, 100); + value = static_cast(gain)/100.0; + break; } return value; } @@ -201,6 +208,12 @@ bool VideoCapture_uEye::setProperty(int property_id, double value) break; ASSERT_UEYE(is_SetFrameRate(cam_id, value, &fps)); break; + case CAP_PROP_EXPOSURE: + ASSERT_UEYE(is_Exposure(cam_id, IS_EXPOSURE_CMD_SET_EXPOSURE, (void*)&value, sizeof(value))); + break; + case CAP_PROP_GAIN: + is_SetHWGainFactor(cam_id, IS_SET_MASTER_GAIN_FACTOR, static_cast(value)); + break; } if(set_format) { diff --git a/modules/videoio/src/container_avi.cpp b/modules/videoio/src/container_avi.cpp index 2fb2ee14f8..3223e77090 100644 --- a/modules/videoio/src/container_avi.cpp +++ b/modules/videoio/src/container_avi.cpp @@ -124,6 +124,7 @@ struct RiffList uint32_t m_size; uint32_t m_list_type_cc; }; +#pragma pack(pop) class VideoInputStream { @@ -149,7 +150,6 @@ private: String m_fname; }; -#pragma pack(pop) inline VideoInputStream& operator >> (VideoInputStream& is, AviMainHeader& avih) { diff --git a/modules/videoio/src/videoio_registry.cpp b/modules/videoio/src/videoio_registry.cpp index 61e3ac0724..6cd32edc27 100644 --- a/modules/videoio/src/videoio_registry.cpp +++ b/modules/videoio/src/videoio_registry.cpp @@ -8,6 +8,8 @@ #include "opencv2/videoio/registry.hpp" +#include "opencv2/core/utils/filesystem.private.hpp" // OPENCV_HAVE_FILESYSTEM_SUPPORT + #include "cap_librealsense.hpp" #include "cap_dshow.hpp" @@ -34,15 +36,19 @@ namespace cv { namespace { +#if OPENCV_HAVE_FILESYSTEM_SUPPORT && defined(ENABLE_PLUGINS) #define DECLARE_DYNAMIC_BACKEND(cap, name, mode) \ { \ cap, (BackendMode)(mode), 1000, name, createPluginBackendFactory(cap, name) \ -} +}, +#else +#define DECLARE_DYNAMIC_BACKEND(cap, name, mode) /* nothing */ +#endif #define DECLARE_STATIC_BACKEND(cap, name, mode, createCaptureFile, createCaptureCamera, createWriter) \ { \ cap, (BackendMode)(mode), 1000, name, createBackendFactory(createCaptureFile, createCaptureCamera, createWriter) \ -} +}, /** Ordering guidelines: - modern optimized, multi-platform libraries: ffmpeg, gstreamer, Media SDK @@ -55,90 +61,90 @@ namespace { static const struct VideoBackendInfo builtin_backends[] = { #ifdef HAVE_FFMPEG - DECLARE_STATIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, cvCreateFileCapture_FFMPEG_proxy, 0, cvCreateVideoWriter_FFMPEG_proxy), + DECLARE_STATIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, cvCreateFileCapture_FFMPEG_proxy, 0, cvCreateVideoWriter_FFMPEG_proxy) #elif defined(ENABLE_PLUGINS) || defined(HAVE_FFMPEG_WRAPPER) - DECLARE_DYNAMIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_FFMPEG, "FFMPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER) #endif #ifdef HAVE_GSTREAMER - DECLARE_STATIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER, createGStreamerCapture_file, createGStreamerCapture_cam, create_GStreamer_writer), + DECLARE_STATIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER, createGStreamerCapture_file, createGStreamerCapture_cam, create_GStreamer_writer) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_GSTREAMER, "GSTREAMER", MODE_CAPTURE_ALL | MODE_WRITER) #endif #ifdef HAVE_MFX // Media SDK - DECLARE_STATIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_MFX_capture, 0, create_MFX_writer), + DECLARE_STATIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_MFX_capture, 0, create_MFX_writer) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_INTEL_MFX, "INTEL_MFX", MODE_CAPTURE_BY_FILENAME | MODE_WRITER) #endif // Apple platform #ifdef HAVE_AVFOUNDATION - DECLARE_STATIC_BACKEND(CAP_AVFOUNDATION, "AVFOUNDATION", MODE_CAPTURE_ALL | MODE_WRITER, create_AVFoundation_capture_file, create_AVFoundation_capture_cam, create_AVFoundation_writer), + DECLARE_STATIC_BACKEND(CAP_AVFOUNDATION, "AVFOUNDATION", MODE_CAPTURE_ALL | MODE_WRITER, create_AVFoundation_capture_file, create_AVFoundation_capture_cam, create_AVFoundation_writer) #endif // Windows #ifdef WINRT_VIDEO - DECLARE_STATIC_BACKEND(CAP_WINRT, "WINRT", MODE_CAPTURE_BY_INDEX, 0, create_WRT_capture, 0), + DECLARE_STATIC_BACKEND(CAP_WINRT, "WINRT", MODE_CAPTURE_BY_INDEX, 0, create_WRT_capture, 0) #endif #ifdef HAVE_MSMF - DECLARE_STATIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER, cvCreateCapture_MSMF, cvCreateCapture_MSMF, cvCreateVideoWriter_MSMF), + DECLARE_STATIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER, cvCreateCapture_MSMF, cvCreateCapture_MSMF, cvCreateVideoWriter_MSMF) #elif defined(ENABLE_PLUGINS) && defined(_WIN32) - DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER), + DECLARE_DYNAMIC_BACKEND(CAP_MSMF, "MSMF", MODE_CAPTURE_ALL | MODE_WRITER) #endif #ifdef HAVE_DSHOW - DECLARE_STATIC_BACKEND(CAP_DSHOW, "DSHOW", MODE_CAPTURE_BY_INDEX, 0, create_DShow_capture, 0), + DECLARE_STATIC_BACKEND(CAP_DSHOW, "DSHOW", MODE_CAPTURE_BY_INDEX, 0, create_DShow_capture, 0) #endif // Linux, some Unix #if defined HAVE_CAMV4L2 - DECLARE_STATIC_BACKEND(CAP_V4L2, "V4L2", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_V4L2, "V4L2", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0) #elif defined HAVE_VIDEOIO - DECLARE_STATIC_BACKEND(CAP_V4L, "V4L_BSD", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_V4L, "V4L_BSD", MODE_CAPTURE_ALL, create_V4L_capture_file, create_V4L_capture_cam, 0) #endif // RGB-D universal #ifdef HAVE_OPENNI2 - DECLARE_STATIC_BACKEND(CAP_OPENNI2, "OPENNI2", MODE_CAPTURE_ALL, create_OpenNI2_capture_file, create_OpenNI2_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_OPENNI2, "OPENNI2", MODE_CAPTURE_ALL, create_OpenNI2_capture_file, create_OpenNI2_capture_cam, 0) #endif #ifdef HAVE_LIBREALSENSE - DECLARE_STATIC_BACKEND(CAP_REALSENSE, "INTEL_REALSENSE", MODE_CAPTURE_BY_INDEX, 0, create_RealSense_capture, 0), + DECLARE_STATIC_BACKEND(CAP_REALSENSE, "INTEL_REALSENSE", MODE_CAPTURE_BY_INDEX, 0, create_RealSense_capture, 0) #endif // OpenCV file-based only - DECLARE_STATIC_BACKEND(CAP_IMAGES, "CV_IMAGES", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_Images_capture, 0, create_Images_writer), - DECLARE_STATIC_BACKEND(CAP_OPENCV_MJPEG, "CV_MJPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, createMotionJpegCapture, 0, createMotionJpegWriter), + DECLARE_STATIC_BACKEND(CAP_IMAGES, "CV_IMAGES", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, create_Images_capture, 0, create_Images_writer) + DECLARE_STATIC_BACKEND(CAP_OPENCV_MJPEG, "CV_MJPEG", MODE_CAPTURE_BY_FILENAME | MODE_WRITER, createMotionJpegCapture, 0, createMotionJpegWriter) // special interfaces / stereo cameras / other SDKs #if defined(HAVE_DC1394_2) - DECLARE_STATIC_BACKEND(CAP_FIREWIRE, "FIREWIRE", MODE_CAPTURE_BY_INDEX, 0, create_DC1394_capture, 0), + DECLARE_STATIC_BACKEND(CAP_FIREWIRE, "FIREWIRE", MODE_CAPTURE_BY_INDEX, 0, create_DC1394_capture, 0) #endif // GigE #ifdef HAVE_PVAPI - DECLARE_STATIC_BACKEND(CAP_PVAPI, "PVAPI", MODE_CAPTURE_BY_INDEX, 0, create_PvAPI_capture, 0), + DECLARE_STATIC_BACKEND(CAP_PVAPI, "PVAPI", MODE_CAPTURE_BY_INDEX, 0, create_PvAPI_capture, 0) #endif #ifdef HAVE_XIMEA - DECLARE_STATIC_BACKEND(CAP_XIAPI, "XIMEA", MODE_CAPTURE_ALL, create_XIMEA_capture_file, create_XIMEA_capture_cam, 0), + DECLARE_STATIC_BACKEND(CAP_XIAPI, "XIMEA", MODE_CAPTURE_ALL, create_XIMEA_capture_file, create_XIMEA_capture_cam, 0) #endif #ifdef HAVE_ARAVIS_API - DECLARE_STATIC_BACKEND(CAP_ARAVIS, "ARAVIS", MODE_CAPTURE_BY_INDEX, 0, create_Aravis_capture, 0), + DECLARE_STATIC_BACKEND(CAP_ARAVIS, "ARAVIS", MODE_CAPTURE_BY_INDEX, 0, create_Aravis_capture, 0) #endif #ifdef HAVE_UEYE // uEye - DECLARE_STATIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX, 0, create_ueye_camera, 0), + DECLARE_STATIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX, 0, create_ueye_camera, 0) #elif defined(ENABLE_PLUGINS) - DECLARE_DYNAMIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX), + DECLARE_DYNAMIC_BACKEND(CAP_UEYE, "UEYE", MODE_CAPTURE_BY_INDEX) #endif #ifdef HAVE_GPHOTO2 - DECLARE_STATIC_BACKEND(CAP_GPHOTO2, "GPHOTO2", MODE_CAPTURE_ALL, createGPhoto2Capture, createGPhoto2Capture, 0), + DECLARE_STATIC_BACKEND(CAP_GPHOTO2, "GPHOTO2", MODE_CAPTURE_ALL, createGPhoto2Capture, createGPhoto2Capture, 0) #endif #ifdef HAVE_XINE - DECLARE_STATIC_BACKEND(CAP_XINE, "XINE", MODE_CAPTURE_BY_FILENAME, createXINECapture, 0, 0), + DECLARE_STATIC_BACKEND(CAP_XINE, "XINE", MODE_CAPTURE_BY_FILENAME, createXINECapture, 0, 0) #endif #if defined(HAVE_ANDROID_MEDIANDK) || defined(HAVE_ANDROID_NATIVE_CAMERA) DECLARE_STATIC_BACKEND(CAP_ANDROID, "ANDROID_NATIVE", @@ -163,7 +169,7 @@ static const struct VideoBackendInfo builtin_backends[] = #else 0, #endif - 0), + 0) #endif // dropped backends: MIL, TYZX }; diff --git a/modules/videoio/test/test_gstreamer.cpp b/modules/videoio/test/test_gstreamer.cpp index ca100367b1..207f6de50b 100644 --- a/modules/videoio/test/test_gstreamer.cpp +++ b/modules/videoio/test/test_gstreamer.cpp @@ -35,6 +35,10 @@ TEST_P(videoio_gstreamer, read_check) cvtColor(decode_frame, rgb_frame, convertToRGB); cvtColor(rgb_frame, gray_frame, COLOR_RGB2GRAY); + if (gray_frame.depth() == CV_16U) + { + gray_frame.convertTo(gray_frame, CV_8U, 255.0/65535); + } vector circles; HoughCircles(gray_frame, circles, HOUGH_GRADIENT, 1, gray_frame.rows/16, 100, 30, 1, 30 ); @@ -58,6 +62,10 @@ TEST_P(videoio_gstreamer, read_check) static const Param test_data[] = { make_tuple("video/x-raw, format=BGR" , Size(640, 480), Size(640, 480), COLOR_BGR2RGB), + make_tuple("video/x-raw, format=BGRA" , Size(640, 480), Size(640, 480), COLOR_BGRA2RGB), + make_tuple("video/x-raw, format=RGBA" , Size(640, 480), Size(640, 480), COLOR_RGBA2RGB), + make_tuple("video/x-raw, format=BGRx" , Size(640, 480), Size(640, 480), COLOR_BGRA2RGB), + make_tuple("video/x-raw, format=RGBx" , Size(640, 480), Size(640, 480), COLOR_RGBA2RGB), make_tuple("video/x-raw, format=GRAY8", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), make_tuple("video/x-raw, format=UYVY" , Size(640, 480), Size(640, 480), COLOR_YUV2RGB_UYVY), make_tuple("video/x-raw, format=YUY2" , Size(640, 480), Size(640, 480), COLOR_YUV2RGB_YUY2), @@ -76,6 +84,10 @@ static const Param test_data[] = { make_tuple("video/x-raw, format=NV21" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_NV21), make_tuple("video/x-raw, format=YV12" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_YV12), make_tuple("video/x-raw, format=I420" , Size(322, 242), Size(322, 363), COLOR_YUV2RGB_I420), + + // 16 bit + make_tuple("video/x-raw, format=GRAY16_LE", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), + make_tuple("video/x-raw, format=GRAY16_BE", Size(640, 480), Size(640, 480), COLOR_GRAY2RGB), }; INSTANTIATE_TEST_CASE_P(videoio, videoio_gstreamer, testing::ValuesIn(test_data)); diff --git a/platforms/js/opencv_js.config.py b/platforms/js/opencv_js.config.py index 8549c2b6ab..7d2a52b31f 100644 --- a/platforms/js/opencv_js.config.py +++ b/platforms/js/opencv_js.config.py @@ -13,18 +13,86 @@ core = { 'Algorithm': [], } -imgproc = {'': ['Canny', 'GaussianBlur', 'Laplacian', 'HoughLines', 'HoughLinesP', 'HoughCircles', 'Scharr','Sobel', \ - 'adaptiveThreshold','approxPolyDP','arcLength','bilateralFilter','blur','boundingRect','boxFilter',\ - 'calcBackProject','calcHist','circle','compareHist','connectedComponents','connectedComponentsWithStats', \ - 'contourArea', 'convexHull', 'convexityDefects', 'cornerHarris','cornerMinEigenVal','createCLAHE', \ - 'createLineSegmentDetector','cvtColor','demosaicing','dilate', 'distanceTransform','distanceTransformWithLabels', \ - 'drawContours','ellipse','ellipse2Poly','equalizeHist','erode', 'filter2D', 'findContours','fitEllipse', \ - 'fitLine', 'floodFill','getAffineTransform', 'getPerspectiveTransform', 'getRotationMatrix2D', 'getStructuringElement', \ - 'goodFeaturesToTrack','grabCut','initUndistortRectifyMap', 'integral','integral2', 'isContourConvex', 'line', \ - 'matchShapes', 'matchTemplate','medianBlur', 'minAreaRect', 'minEnclosingCircle', 'moments', 'morphologyEx', \ - 'pointPolygonTest', 'putText','pyrDown','pyrUp','rectangle','remap', 'resize','sepFilter2D','threshold', \ - 'undistort','warpAffine','warpPerspective','warpPolar','watershed', \ - 'fillPoly', 'fillConvexPoly', 'polylines', +imgproc = { + '': [ + 'Canny', + 'GaussianBlur', + 'Laplacian', + 'HoughLines', + 'HoughLinesP', + 'HoughCircles', + 'Scharr', + 'Sobel', + 'adaptiveThreshold', + 'approxPolyDP', + 'arcLength', + 'bilateralFilter', + 'blur', + 'boundingRect', + 'boxFilter', + 'calcBackProject', + 'calcHist', + 'circle', + 'compareHist', + 'connectedComponents', + 'connectedComponentsWithStats', + 'contourArea', + 'convexHull', + 'convexityDefects', + 'cornerHarris', + 'cornerMinEigenVal', + 'createCLAHE', + 'createLineSegmentDetector', + 'cvtColor', + 'demosaicing', + 'dilate', + 'distanceTransform', + 'distanceTransformWithLabels', + 'drawContours', + 'ellipse', + 'ellipse2Poly', + 'equalizeHist', + 'erode', + 'filter2D', + 'findContours', + 'fitEllipse', + 'fitLine', + 'floodFill', + 'getAffineTransform', + 'getPerspectiveTransform', + 'getRotationMatrix2D', + 'getStructuringElement', + 'goodFeaturesToTrack', + 'grabCut', + #'initUndistortRectifyMap', # 4.x: moved to calib3d + 'integral', + 'integral2', + 'isContourConvex', + 'line', + 'matchShapes', + 'matchTemplate', + 'medianBlur', + 'minAreaRect', + 'minEnclosingCircle', + 'moments', + 'morphologyEx', + 'pointPolygonTest', + 'putText', + 'pyrDown', + 'pyrUp', + 'rectangle', + 'remap', + 'resize', + 'sepFilter2D', + 'threshold', + #'undistort', # 4.x: moved to calib3d + 'warpAffine', + 'warpPerspective', + 'warpPolar', + 'watershed', + 'fillPoly', + 'fillConvexPoly', + 'polylines', ], 'CLAHE': ['apply', 'collectGarbage', 'getClipLimit', 'getTilesGridSize', 'setClipLimit', 'setTilesGridSize'], 'segmentation_IntelligentScissorsMB': [ @@ -99,11 +167,32 @@ aruco = {'': ['detectMarkers', 'drawDetectedMarkers', 'drawAxis', 'estimatePoseS _3d = { '': [ - 'findHomography', 'calibrateCameraExtended', 'drawFrameAxes', 'estimateAffine2D', - 'getDefaultNewCameraMatrix', 'initUndistortRectifyMap', 'Rodrigues', - 'solvePnP', 'solvePnPRansac', 'solvePnPRefineLM' - ] + 'findHomography', + 'calibrateCameraExtended', + 'drawFrameAxes', + 'estimateAffine2D', + 'getDefaultNewCameraMatrix', + 'initUndistortRectifyMap', + 'Rodrigues', + 'solvePnP', + 'solvePnPRansac', + 'solvePnPRefineLM', + 'projectPoints', + 'undistort', + ], +} + +calib = { + '': [ + + # cv::fisheye namespace + 'fisheye_initUndistortRectifyMap', + 'fisheye_projectPoints', + ], } -white_list = makeWhiteList([core, imgproc, objdetect, video, dnn, features2d, photo, aruco, _3d]) +white_list = makeWhiteList([core, imgproc, objdetect, video, dnn, features2d, photo, aruco, _3d, calib]) + +# namespace_prefix_override['dnn'] = '' # compatibility stuff (enabled by default) +# namespace_prefix_override['aruco'] = '' # compatibility stuff (enabled by default) diff --git a/platforms/linux/gnu.toolchain.cmake b/platforms/linux/gnu.toolchain.cmake index cba08e7fbb..64258e65b3 100644 --- a/platforms/linux/gnu.toolchain.cmake +++ b/platforms/linux/gnu.toolchain.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) # load settings in case of "try compile" set(TOOLCHAIN_CONFIG_FILE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/toolchain.config.cmake") diff --git a/platforms/linux/riscv.toolchain.cmake b/platforms/linux/riscv.toolchain.cmake index 2a69d7e004..cea80bd9ba 100644 --- a/platforms/linux/riscv.toolchain.cmake +++ b/platforms/linux/riscv.toolchain.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) if(COMMAND toolchain_save_config) return() # prevent recursive call diff --git a/platforms/linux/riscv64-gcc.toolchain.cmake b/platforms/linux/riscv64-gcc.toolchain.cmake index c46d62a360..675879f86b 100644 --- a/platforms/linux/riscv64-gcc.toolchain.cmake +++ b/platforms/linux/riscv64-gcc.toolchain.cmake @@ -10,8 +10,8 @@ set(CMAKE_CXX_COMPILER ${RISCV_GCC_INSTALL_ROOT}/bin/riscv64-unknown-linux-gnu-g # Don't run the linker on compiler check set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) -set(CMAKE_C_FLAGS "-march=rv64gcv_zvqmac ${CMAKE_C_FLAGS}") -set(CMAKE_CXX_FLAGS "-march=rv64gcv_zvqmac ${CXX_FLAGS}") +set(CMAKE_C_FLAGS "-march=rv64gcv_zfh ${CMAKE_C_FLAGS}") +set(CMAKE_CXX_FLAGS "-march=rv64gcv_zfh ${CXX_FLAGS}") set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT}) set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) diff --git a/platforms/scripts/valgrind.supp b/platforms/scripts/valgrind.supp index 10ea856e9d..c71cdf965d 100644 --- a/platforms/scripts/valgrind.supp +++ b/platforms/scripts/valgrind.supp @@ -51,7 +51,7 @@ OpenCV-TLS-getTlsStorage Memcheck:Leak ... - fun:_ZN2cvL13getTlsStorageEv + fun:_ZN2cv*L13getTlsStorageEv } { @@ -159,7 +159,7 @@ OpenCV-DNN-getLayerFactoryMutex Memcheck:Leak ... - fun:_ZN2cv3dnn*L20getLayerFactoryMutexEv + fun:_ZN2cv3dnn*20getLayerFactoryMutexEv } { diff --git a/platforms/semihosting/aarch64-semihosting.toolchain.cmake b/platforms/semihosting/aarch64-semihosting.toolchain.cmake new file mode 100644 index 0000000000..95bbda3bed --- /dev/null +++ b/platforms/semihosting/aarch64-semihosting.toolchain.cmake @@ -0,0 +1,40 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(CMAKE_SYSTEM_NAME Generic) +set(CMAKE_SYSTEM_PROCESSOR AArch64) + +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + +set(PORT_FILE ${CMAKE_SOURCE_DIR}/platforms/semihosting/include/aarch64_semihosting_port.hpp) + +set(COMMON_FLAGS "--specs=rdimon.specs -DOPENCV_INCLUDE_PORT_FILE=\\\"${PORT_FILE}\\\"") + +set(CMAKE_AR ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ar${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_ASM_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-gcc${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_C_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-gcc${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_CXX_COMPILER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-g++${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_LINKER ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ld${CMAKE_EXECUTABLE_SUFFIX}) +set(CMAKE_OBJCOPY ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-objcopy${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_RANLIB ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-ranlib${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_SIZE ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-size${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_STRIP ${SEMIHOSTING_TOOLCHAIN_PATH}aarch64-none-elf-strip${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "") +set(CMAKE_C_FLAGS ${COMMON_FLAGS} CACHE INTERNAL "") +set(CMAKE_CXX_FLAGS ${COMMON_FLAGS} CACHE INTERNAL "") + +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +set(OPENCV_SEMIHOSTING ON) +set(OPENCV_DISABLE_THREAD_SUPPORT ON) +set(OPENCV_DISABLE_FILESYSTEM_SUPPORT ON) +set(BUILD_SHARED_LIBS OFF) +set(OPENCV_FORCE_3RDPARTY_BUILD OFF) + + +# Enable newlib. +add_definitions(-D_GNU_SOURCE) + +add_definitions(-D_POSIX_PATH_MAX=0) diff --git a/platforms/semihosting/include/aarch64_semihosting_port.hpp b/platforms/semihosting/include/aarch64_semihosting_port.hpp new file mode 100644 index 0000000000..d3151c240a --- /dev/null +++ b/platforms/semihosting/include/aarch64_semihosting_port.hpp @@ -0,0 +1,42 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef AARCH64_BAREMETAL_PORT_HPP +#define AARCH64_BAREMETAL_PORT_HPP + +#include // Needed for `memalign`. +#include // Needed for `ENOMEM`. + +// -std=c++11 is missing the following definitions when targeting +// semihosting on aarch64. +#if __cplusplus == 201103L +#include +#define M_PI 3.14159265358979323846 +#define M_SQRT2 1.41421356237309504880 + +namespace std { +inline double cbrt(double x) { + return ::cbrt(x); +} +inline double copysign(double mag, double sgn) { + return ::copysign(mag, sgn); +} +} //namespace std +#endif // __cplusplus == 201103L + +extern "C" { +// Redirect the implementation of `posix_memalign` to `memalign` +// as the former is +// missing at link time. https://pubs.opengroup.org/onlinepubs/9699919799/functions/posix_memalign.html +__attribute__((weak)) int posix_memalign(void **memptr, size_t alignment, size_t size) { + void * ptr = memalign(alignment, size); + if (ptr != NULL) { + *memptr = ptr; + return 0; + } + return ENOMEM; +} +} // extern "C" + +#endif diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch new file mode 100644 index 0000000000..f1e7487442 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-multidevice-autoplugin.patch @@ -0,0 +1,16 @@ +diff --git a/inference-engine/src/CMakeLists.txt b/inference-engine/src/CMakeLists.txt +index 0ba0dd78..7d34e7cb 100644 +--- a/inference-engine/src/CMakeLists.txt ++++ b/inference-engine/src/CMakeLists.txt +@@ -26,9 +26,9 @@ endif() + + add_subdirectory(hetero_plugin) + +-add_subdirectory(auto_plugin) ++#add_subdirectory(auto_plugin) + +-add_subdirectory(multi_device) ++#add_subdirectory(multi_device) + + add_subdirectory(transformations) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch new file mode 100644 index 0000000000..9d44cdadc6 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-disable-unused-targets.patch @@ -0,0 +1,219 @@ +diff --git a/cmake/developer_package/add_ie_target.cmake b/cmake/developer_package/add_ie_target.cmake +index d49f16a4d..2726ca787 100644 +--- a/cmake/developer_package/add_ie_target.cmake ++++ b/cmake/developer_package/add_ie_target.cmake +@@ -92,7 +92,7 @@ function(addIeTarget) + if (ARG_TYPE STREQUAL EXECUTABLE) + add_executable(${ARG_NAME} ${all_sources}) + elseif(ARG_TYPE STREQUAL STATIC OR ARG_TYPE STREQUAL SHARED) +- add_library(${ARG_NAME} ${ARG_TYPE} ${all_sources}) ++ add_library(${ARG_NAME} ${ARG_TYPE} EXCLUDE_FROM_ALL ${all_sources}) + else() + message(SEND_ERROR "Invalid target type ${ARG_TYPE} specified for target name ${ARG_NAME}") + endif() +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index 1ac7fd8bf..df7091e51 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -39,7 +39,7 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + function(ie_build_samples) + # samples should be build with the same flags as from OpenVINO package, +@@ -58,7 +58,7 @@ endfunction() + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-ie_build_samples() ++#ie_build_samples() + + if(ENABLE_PYTHON) + add_subdirectory(ie_bridges/python) +@@ -142,7 +142,7 @@ endif() + # Developer package + # + +-openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) ++#openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) + + # for Template plugin + if(NGRAPH_INTERPRETER_ENABLE) +@@ -166,7 +166,7 @@ function(ie_generate_dev_package_config) + @ONLY) + endfunction() + +-ie_generate_dev_package_config() ++#ie_generate_dev_package_config() + + # + # Coverage +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index e8ed1a5c4..1fc9fc3ff 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -110,7 +110,7 @@ add_cpplint_target(${TARGET_NAME}_plugin_api_cpplint FOR_SOURCES ${plugin_api_sr + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS} + ${PUBLIC_HEADERS}) +@@ -181,7 +181,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + $ + ${IE_STATIC_DEPENDENT_FILES}) +diff --git a/inference-engine/src/legacy_api/CMakeLists.txt b/inference-engine/src/legacy_api/CMakeLists.txt +index 8eae82bd2..e0e6745b1 100644 +--- a/inference-engine/src/legacy_api/CMakeLists.txt ++++ b/inference-engine/src/legacy_api/CMakeLists.txt +@@ -26,7 +26,7 @@ endif() + + file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${PUBLIC_HEADERS}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index fe57b29dd..07831e2fb 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -67,7 +67,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + target_link_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $ +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index f9548339d..ef962145a 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -101,7 +101,7 @@ endif() + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS}) + +@@ -153,7 +153,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 249e47c28..4ddf63049 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) + + function(add_common_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + ie_faster_build(${TARGET_NAME} + UNITY +@@ -60,7 +60,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index bc73ab5b1..b4c1547fc 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h *.inc) + + function(add_graph_transformer_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + set_ie_threading_interface_for(${TARGET_NAME}) + +@@ -70,7 +70,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb2801a..f7e031c01 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +diff --git a/ngraph/core/builder/CMakeLists.txt b/ngraph/core/builder/CMakeLists.txt +index ff5c381e7..2797ec9ab 100644 +--- a/ngraph/core/builder/CMakeLists.txt ++++ b/ngraph/core/builder/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt +index ef4a764ab..f6d3172e2 100644 +--- a/ngraph/core/reference/CMakeLists.txt ++++ b/ngraph/core/reference/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/openvino/itt/CMakeLists.txt b/openvino/itt/CMakeLists.txt +index e9f880b8c..c63f4df63 100644 +--- a/openvino/itt/CMakeLists.txt ++++ b/openvino/itt/CMakeLists.txt +@@ -6,7 +6,7 @@ set(TARGET_NAME itt) + + file(GLOB_RECURSE SOURCES "src/*.cpp" "src/*.hpp") + +-add_library(${TARGET_NAME} STATIC ${SOURCES}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + add_library(openvino::itt ALIAS ${TARGET_NAME}) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch new file mode 100644 index 0000000000..65e6f84dc8 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-pdb.patch @@ -0,0 +1,15 @@ +iff --git a/CMakeLists.txt b/CMakeLists.txt +index e0706a72e..9a053b1e4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -6,6 +6,10 @@ cmake_minimum_required(VERSION 3.13) + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /FS") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(IE_MAIN_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine) + diff --git a/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch b/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch new file mode 100644 index 0000000000..36b0068775 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/20210630-dldt-vs-version.patch @@ -0,0 +1,16 @@ +diff --git a/cmake/developer_package/vs_version/vs_version.cmake b/cmake/developer_package/vs_version/vs_version.cmake +index 14d4c0e1e..6a44f73b9 100644 +--- a/cmake/developer_package/vs_version/vs_version.cmake ++++ b/cmake/developer_package/vs_version/vs_version.cmake +@@ -8,9 +8,9 @@ set(IE_VS_VER_FILEVERSION_STR "${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERS + + set(IE_VS_VER_COMPANY_NAME_STR "Intel Corporation") + set(IE_VS_VER_PRODUCTVERSION_STR "${CI_BUILD_NUMBER}") +-set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit") ++set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit (for OpenCV Windows package)") + set(IE_VS_VER_COPYRIGHT_STR "Copyright (C) 2018-2021, Intel Corporation") +-set(IE_VS_VER_COMMENTS_STR "https://docs.openvinotoolkit.org/") ++set(IE_VS_VER_COMMENTS_STR "https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend") + + # + # ie_add_vs_version_file(NAME diff --git a/platforms/winpack_dldt/2021.4.1/build.config.py b/platforms/winpack_dldt/2021.4.1/build.config.py new file mode 100644 index 0000000000..a643c17928 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/build.config.py @@ -0,0 +1 @@ +os.environ['CI_BUILD_NUMBER'] = '2021.4.1-opencv_winpack_dldt' diff --git a/platforms/winpack_dldt/2021.4.1/patch.config.py b/platforms/winpack_dldt/2021.4.1/patch.config.py new file mode 100644 index 0000000000..7f8715aae2 --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/patch.config.py @@ -0,0 +1,4 @@ +applyPatch('20210630-dldt-disable-unused-targets.patch') +applyPatch('20210630-dldt-pdb.patch') +applyPatch('20210630-dldt-disable-multidevice-autoplugin.patch') +applyPatch('20210630-dldt-vs-version.patch') diff --git a/platforms/winpack_dldt/2021.4.1/sysroot.config.py b/platforms/winpack_dldt/2021.4.1/sysroot.config.py new file mode 100644 index 0000000000..fa4281107d --- /dev/null +++ b/platforms/winpack_dldt/2021.4.1/sysroot.config.py @@ -0,0 +1,56 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_ir_reader') +#copy_dll('inference_engine_ir_v7_reader') +copy_dll('inference_engine_legacy') +copy_dll('inference_engine_transformations') # runtime +copy_dll('inference_engine_lp_transformations') # runtime +#copy_dll('inference_engine_preproc') # runtime +copy_dll('MKLDNNPlugin') # runtime +copy_dll('myriadPlugin') # runtime +#copy_dll('MultiDevicePlugin') # runtime, not used +copy_dll('ngraph') +copy_bin('plugins.xml') +copy_bin('pcie-ma2x8x.elf') +copy_bin('usb-ma2x8x.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(build_bin_dir / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_ir_reader.lib', sysroot_ie_lib_dir / 'inference_engine_ir_reader.lib') + copytree(build_bin_dir / 'inference_engine_legacy.lib', sysroot_ie_lib_dir / 'inference_engine_legacy.lib') +else: + copytree(build_bin_dir / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_ir_readerd.lib', sysroot_ie_lib_dir / 'inference_engine_ir_readerd.lib') + copytree(build_bin_dir / 'inference_engine_legacyd.lib', sysroot_ie_lib_dir / 'inference_engine_legacyd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch new file mode 100644 index 0000000000..f1e7487442 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-multidevice-autoplugin.patch @@ -0,0 +1,16 @@ +diff --git a/inference-engine/src/CMakeLists.txt b/inference-engine/src/CMakeLists.txt +index 0ba0dd78..7d34e7cb 100644 +--- a/inference-engine/src/CMakeLists.txt ++++ b/inference-engine/src/CMakeLists.txt +@@ -26,9 +26,9 @@ endif() + + add_subdirectory(hetero_plugin) + +-add_subdirectory(auto_plugin) ++#add_subdirectory(auto_plugin) + +-add_subdirectory(multi_device) ++#add_subdirectory(multi_device) + + add_subdirectory(transformations) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch new file mode 100644 index 0000000000..9d44cdadc6 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-disable-unused-targets.patch @@ -0,0 +1,219 @@ +diff --git a/cmake/developer_package/add_ie_target.cmake b/cmake/developer_package/add_ie_target.cmake +index d49f16a4d..2726ca787 100644 +--- a/cmake/developer_package/add_ie_target.cmake ++++ b/cmake/developer_package/add_ie_target.cmake +@@ -92,7 +92,7 @@ function(addIeTarget) + if (ARG_TYPE STREQUAL EXECUTABLE) + add_executable(${ARG_NAME} ${all_sources}) + elseif(ARG_TYPE STREQUAL STATIC OR ARG_TYPE STREQUAL SHARED) +- add_library(${ARG_NAME} ${ARG_TYPE} ${all_sources}) ++ add_library(${ARG_NAME} ${ARG_TYPE} EXCLUDE_FROM_ALL ${all_sources}) + else() + message(SEND_ERROR "Invalid target type ${ARG_TYPE} specified for target name ${ARG_NAME}") + endif() +diff --git a/inference-engine/CMakeLists.txt b/inference-engine/CMakeLists.txt +index 1ac7fd8bf..df7091e51 100644 +--- a/inference-engine/CMakeLists.txt ++++ b/inference-engine/CMakeLists.txt +@@ -39,7 +39,7 @@ if(ENABLE_TESTS) + add_subdirectory(tests) + endif() + +-add_subdirectory(tools) ++#add_subdirectory(tools) + + function(ie_build_samples) + # samples should be build with the same flags as from OpenVINO package, +@@ -58,7 +58,7 @@ endfunction() + + # gflags and format_reader targets are kept inside of samples directory and + # they must be built even if samples build is disabled (required for tests and tools). +-ie_build_samples() ++#ie_build_samples() + + if(ENABLE_PYTHON) + add_subdirectory(ie_bridges/python) +@@ -142,7 +142,7 @@ endif() + # Developer package + # + +-openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) ++#openvino_developer_export_targets(COMPONENT openvino_common TARGETS format_reader gflags ie_samples_utils) + + # for Template plugin + if(NGRAPH_INTERPRETER_ENABLE) +@@ -166,7 +166,7 @@ function(ie_generate_dev_package_config) + @ONLY) + endfunction() + +-ie_generate_dev_package_config() ++#ie_generate_dev_package_config() + + # + # Coverage +diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt +index e8ed1a5c4..1fc9fc3ff 100644 +--- a/inference-engine/src/inference_engine/CMakeLists.txt ++++ b/inference-engine/src/inference_engine/CMakeLists.txt +@@ -110,7 +110,7 @@ add_cpplint_target(${TARGET_NAME}_plugin_api_cpplint FOR_SOURCES ${plugin_api_sr + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS} + ${PUBLIC_HEADERS}) +@@ -181,7 +181,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $ + $ + ${IE_STATIC_DEPENDENT_FILES}) +diff --git a/inference-engine/src/legacy_api/CMakeLists.txt b/inference-engine/src/legacy_api/CMakeLists.txt +index 8eae82bd2..e0e6745b1 100644 +--- a/inference-engine/src/legacy_api/CMakeLists.txt ++++ b/inference-engine/src/legacy_api/CMakeLists.txt +@@ -26,7 +26,7 @@ endif() + + file(TOUCH ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${PUBLIC_HEADERS}) + +diff --git a/inference-engine/src/mkldnn_plugin/CMakeLists.txt b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +index fe57b29dd..07831e2fb 100644 +--- a/inference-engine/src/mkldnn_plugin/CMakeLists.txt ++++ b/inference-engine/src/mkldnn_plugin/CMakeLists.txt +@@ -67,7 +67,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # add test object library + +-add_library(${TARGET_NAME}_obj OBJECT ${SOURCES} ${HEADERS}) ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL ${SOURCES} ${HEADERS}) + target_link_libraries(${TARGET_NAME}_obj PUBLIC mkldnn) + + target_include_directories(${TARGET_NAME}_obj PRIVATE $ +diff --git a/inference-engine/src/preprocessing/CMakeLists.txt b/inference-engine/src/preprocessing/CMakeLists.txt +index f9548339d..ef962145a 100644 +--- a/inference-engine/src/preprocessing/CMakeLists.txt ++++ b/inference-engine/src/preprocessing/CMakeLists.txt +@@ -101,7 +101,7 @@ endif() + + # Create object library + +-add_library(${TARGET_NAME}_obj OBJECT ++add_library(${TARGET_NAME}_obj OBJECT EXCLUDE_FROM_ALL + ${LIBRARY_SRC} + ${LIBRARY_HEADERS}) + +@@ -153,7 +153,7 @@ ie_add_api_validator_post_build_step(TARGET ${TARGET_NAME}) + + # Static library used for unit tests which are always built + +-add_library(${TARGET_NAME}_s STATIC ++add_library(${TARGET_NAME}_s STATIC EXCLUDE_FROM_ALL + $) + + set_ie_threading_interface_for(${TARGET_NAME}_s) +diff --git a/inference-engine/src/vpu/common/CMakeLists.txt b/inference-engine/src/vpu/common/CMakeLists.txt +index 249e47c28..4ddf63049 100644 +--- a/inference-engine/src/vpu/common/CMakeLists.txt ++++ b/inference-engine/src/vpu/common/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h) + + function(add_common_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + ie_faster_build(${TARGET_NAME} + UNITY +@@ -60,7 +60,7 @@ add_common_target("vpu_common_lib" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_common_target("vpu_common_lib_test_static" TRUE) ++ #add_common_target("vpu_common_lib_test_static" TRUE) + else() + add_library("vpu_common_lib_test_static" ALIAS "vpu_common_lib") + endif() +diff --git a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +index bc73ab5b1..b4c1547fc 100644 +--- a/inference-engine/src/vpu/graph_transformer/CMakeLists.txt ++++ b/inference-engine/src/vpu/graph_transformer/CMakeLists.txt +@@ -5,7 +5,7 @@ + file(GLOB_RECURSE SOURCES *.cpp *.hpp *.h *.inc) + + function(add_graph_transformer_target TARGET_NAME STATIC_IE) +- add_library(${TARGET_NAME} STATIC ${SOURCES}) ++ add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + set_ie_threading_interface_for(${TARGET_NAME}) + +@@ -70,7 +70,7 @@ add_graph_transformer_target("vpu_graph_transformer" FALSE) + + # Unit tests support for graph transformer + if(WIN32) +- add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) ++ #add_graph_transformer_target("vpu_graph_transformer_test_static" TRUE) + else() + add_library("vpu_graph_transformer_test_static" ALIAS "vpu_graph_transformer") + endif() +diff --git a/inference-engine/thirdparty/pugixml/CMakeLists.txt b/inference-engine/thirdparty/pugixml/CMakeLists.txt +index 8bcb2801a..f7e031c01 100644 +--- a/inference-engine/thirdparty/pugixml/CMakeLists.txt ++++ b/inference-engine/thirdparty/pugixml/CMakeLists.txt +@@ -41,7 +41,7 @@ if(BUILD_SHARED_LIBS) + else() + add_library(pugixml STATIC ${SOURCES}) + if (MSVC) +- add_library(pugixml_mt STATIC ${SOURCES}) ++ #add_library(pugixml_mt STATIC ${SOURCES}) + #if (WIN32) + # set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MT") + # set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MTd") +diff --git a/ngraph/core/builder/CMakeLists.txt b/ngraph/core/builder/CMakeLists.txt +index ff5c381e7..2797ec9ab 100644 +--- a/ngraph/core/builder/CMakeLists.txt ++++ b/ngraph/core/builder/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/ngraph/core/reference/CMakeLists.txt b/ngraph/core/reference/CMakeLists.txt +index ef4a764ab..f6d3172e2 100644 +--- a/ngraph/core/reference/CMakeLists.txt ++++ b/ngraph/core/reference/CMakeLists.txt +@@ -16,7 +16,7 @@ source_group("src" FILES ${LIBRARY_SRC}) + source_group("include" FILES ${PUBLIC_HEADERS}) + + # Create shared library +-add_library(${TARGET_NAME} STATIC ${LIBRARY_SRC} ${PUBLIC_HEADERS}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${LIBRARY_SRC} ${PUBLIC_HEADERS}) + + if(COMMAND ie_faster_build) + ie_faster_build(${TARGET_NAME} +diff --git a/openvino/itt/CMakeLists.txt b/openvino/itt/CMakeLists.txt +index e9f880b8c..c63f4df63 100644 +--- a/openvino/itt/CMakeLists.txt ++++ b/openvino/itt/CMakeLists.txt +@@ -6,7 +6,7 @@ set(TARGET_NAME itt) + + file(GLOB_RECURSE SOURCES "src/*.cpp" "src/*.hpp") + +-add_library(${TARGET_NAME} STATIC ${SOURCES}) ++add_library(${TARGET_NAME} STATIC EXCLUDE_FROM_ALL ${SOURCES}) + + add_library(openvino::itt ALIAS ${TARGET_NAME}) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch new file mode 100644 index 0000000000..65e6f84dc8 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-pdb.patch @@ -0,0 +1,15 @@ +iff --git a/CMakeLists.txt b/CMakeLists.txt +index e0706a72e..9a053b1e4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -6,6 +6,10 @@ cmake_minimum_required(VERSION 3.13) + + project(OpenVINO) + ++set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /FS") ++set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++set(CMAKE_MODULE_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF") ++ + set(OpenVINO_MAIN_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) + set(IE_MAIN_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine) + diff --git a/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch b/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch new file mode 100644 index 0000000000..36b0068775 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/20210630-dldt-vs-version.patch @@ -0,0 +1,16 @@ +diff --git a/cmake/developer_package/vs_version/vs_version.cmake b/cmake/developer_package/vs_version/vs_version.cmake +index 14d4c0e1e..6a44f73b9 100644 +--- a/cmake/developer_package/vs_version/vs_version.cmake ++++ b/cmake/developer_package/vs_version/vs_version.cmake +@@ -8,9 +8,9 @@ set(IE_VS_VER_FILEVERSION_STR "${IE_VERSION_MAJOR}.${IE_VERSION_MINOR}.${IE_VERS + + set(IE_VS_VER_COMPANY_NAME_STR "Intel Corporation") + set(IE_VS_VER_PRODUCTVERSION_STR "${CI_BUILD_NUMBER}") +-set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit") ++set(IE_VS_VER_PRODUCTNAME_STR "OpenVINO toolkit (for OpenCV Windows package)") + set(IE_VS_VER_COPYRIGHT_STR "Copyright (C) 2018-2021, Intel Corporation") +-set(IE_VS_VER_COMMENTS_STR "https://docs.openvinotoolkit.org/") ++set(IE_VS_VER_COMMENTS_STR "https://github.com/opencv/opencv/wiki/Intel%27s-Deep-Learning-Inference-Engine-backend") + + # + # ie_add_vs_version_file(NAME diff --git a/platforms/winpack_dldt/2021.4/build.config.py b/platforms/winpack_dldt/2021.4/build.config.py new file mode 100644 index 0000000000..33ef1050ca --- /dev/null +++ b/platforms/winpack_dldt/2021.4/build.config.py @@ -0,0 +1 @@ +os.environ['CI_BUILD_NUMBER'] = '2021.4.0-opencv_winpack_dldt' diff --git a/platforms/winpack_dldt/2021.4/patch.config.py b/platforms/winpack_dldt/2021.4/patch.config.py new file mode 100644 index 0000000000..7f8715aae2 --- /dev/null +++ b/platforms/winpack_dldt/2021.4/patch.config.py @@ -0,0 +1,4 @@ +applyPatch('20210630-dldt-disable-unused-targets.patch') +applyPatch('20210630-dldt-pdb.patch') +applyPatch('20210630-dldt-disable-multidevice-autoplugin.patch') +applyPatch('20210630-dldt-vs-version.patch') diff --git a/platforms/winpack_dldt/2021.4/sysroot.config.py b/platforms/winpack_dldt/2021.4/sysroot.config.py new file mode 100644 index 0000000000..fa4281107d --- /dev/null +++ b/platforms/winpack_dldt/2021.4/sysroot.config.py @@ -0,0 +1,56 @@ +sysroot_bin_dir = prepare_dir(self.sysrootdir / 'bin') +copytree(self.build_dir / 'install', self.sysrootdir / 'ngraph') +#rm_one(self.sysrootdir / 'ngraph' / 'lib' / 'ngraph.dll') + +build_config = 'Release' if not self.config.build_debug else 'Debug' +build_bin_dir = self.build_dir / 'bin' / 'intel64' / build_config + +def copy_bin(name): + global build_bin_dir, sysroot_bin_dir + copytree(build_bin_dir / name, sysroot_bin_dir / name) + +dll_suffix = 'd' if self.config.build_debug else '' +def copy_dll(name): + global copy_bin, dll_suffix + copy_bin(name + dll_suffix + '.dll') + copy_bin(name + dll_suffix + '.pdb') + +copy_bin('cache.json') +copy_dll('clDNNPlugin') +copy_dll('HeteroPlugin') +copy_dll('inference_engine') +copy_dll('inference_engine_ir_reader') +#copy_dll('inference_engine_ir_v7_reader') +copy_dll('inference_engine_legacy') +copy_dll('inference_engine_transformations') # runtime +copy_dll('inference_engine_lp_transformations') # runtime +#copy_dll('inference_engine_preproc') # runtime +copy_dll('MKLDNNPlugin') # runtime +copy_dll('myriadPlugin') # runtime +#copy_dll('MultiDevicePlugin') # runtime, not used +copy_dll('ngraph') +copy_bin('plugins.xml') +copy_bin('pcie-ma2x8x.elf') +copy_bin('usb-ma2x8x.mvcmd') + +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb' / 'bin', sysroot_bin_dir) +copytree(self.srcdir / 'inference-engine' / 'temp' / 'tbb', self.sysrootdir / 'tbb') + +sysroot_ie_dir = prepare_dir(self.sysrootdir / 'deployment_tools' / 'inference_engine') +sysroot_ie_lib_dir = prepare_dir(sysroot_ie_dir / 'lib' / 'intel64') + +copytree(self.srcdir / 'inference-engine' / 'include', sysroot_ie_dir / 'include') +if not self.config.build_debug: + copytree(build_bin_dir / 'ngraph.lib', sysroot_ie_lib_dir / 'ngraph.lib') + copytree(build_bin_dir / 'inference_engine.lib', sysroot_ie_lib_dir / 'inference_engine.lib') + copytree(build_bin_dir / 'inference_engine_ir_reader.lib', sysroot_ie_lib_dir / 'inference_engine_ir_reader.lib') + copytree(build_bin_dir / 'inference_engine_legacy.lib', sysroot_ie_lib_dir / 'inference_engine_legacy.lib') +else: + copytree(build_bin_dir / 'ngraphd.lib', sysroot_ie_lib_dir / 'ngraphd.lib') + copytree(build_bin_dir / 'inference_engined.lib', sysroot_ie_lib_dir / 'inference_engined.lib') + copytree(build_bin_dir / 'inference_engine_ir_readerd.lib', sysroot_ie_lib_dir / 'inference_engine_ir_readerd.lib') + copytree(build_bin_dir / 'inference_engine_legacyd.lib', sysroot_ie_lib_dir / 'inference_engine_legacyd.lib') + +sysroot_license_dir = prepare_dir(self.sysrootdir / 'etc' / 'licenses') +copytree(self.srcdir / 'LICENSE', sysroot_license_dir / 'dldt-LICENSE') +copytree(self.sysrootdir / 'tbb/LICENSE', sysroot_license_dir / 'tbb-LICENSE') diff --git a/platforms/winpack_dldt/build_package.py b/platforms/winpack_dldt/build_package.py index c3f835cac3..db5d75c17d 100644 --- a/platforms/winpack_dldt/build_package.py +++ b/platforms/winpack_dldt/build_package.py @@ -189,7 +189,10 @@ class BuilderDLDT: if self.srcdir is None: self.srcdir = prepare_dir(self.outdir / 'sources', clean=clean_src_dir) self.build_dir = prepare_dir(self.outdir / 'build', clean=self.config.clean_dldt) - self.sysrootdir = prepare_dir(self.outdir / 'sysroot', clean=self.config.clean_dldt) + self.sysrootdir = prepare_dir(self.outdir / 'sysroot', clean=self.config.clean_dldt or self.config.clean_dldt_sysroot) + if not (self.config.clean_dldt or self.config.clean_dldt_sysroot): + _ = prepare_dir(self.sysrootdir / 'bin', clean=True) # always clean sysroot/bin (package files) + _ = prepare_dir(self.sysrootdir / 'etc', clean=True) # always clean sysroot/etc (package files) if self.config.build_subst_drive: if os.path.exists(self.config.build_subst_drive + ':\\'): @@ -214,7 +217,7 @@ class BuilderDLDT: patch_hashsum = hashlib.md5(self.patch_file_contents.encode('utf-8')).hexdigest() except: log.warn("Can't compute hashsum of patches: %s", self.patch_file) - self.patch_hashsum = patch_hashsum + self.patch_hashsum = self.config.override_patch_hashsum if self.config.override_patch_hashsum else patch_hashsum def prepare_sources(self): @@ -355,7 +358,6 @@ class Builder: BUILD_PERF_TESTS='OFF', ENABLE_CXX11='ON', WITH_INF_ENGINE='ON', - INF_ENGINE_RELEASE=str(self.config.dldt_release), WITH_TBB='ON', CPU_BASELINE='AVX2', CMAKE_INSTALL_PREFIX=str(self.install_dir), @@ -383,6 +385,9 @@ class Builder: OPENCV_PYTHON_INSTALL_PATH='python', ) + if self.config.dldt_release: + cmake_vars['INF_ENGINE_RELEASE'] = str(self.config.dldt_release) + cmake_vars['INF_ENGINE_LIB_DIRS:PATH'] = str(builderDLDT.sysrootdir / 'deployment_tools/inference_engine/lib/intel64') assert os.path.exists(cmake_vars['INF_ENGINE_LIB_DIRS:PATH']), cmake_vars['INF_ENGINE_LIB_DIRS:PATH'] cmake_vars['INF_ENGINE_INCLUDE_DIRS:PATH'] = str(builderDLDT.sysrootdir / 'deployment_tools/inference_engine/include') @@ -466,8 +471,9 @@ class Builder: def main(): dldt_src_url = 'https://github.com/openvinotoolkit/openvino' - dldt_src_commit = '2021.3' - dldt_release = '2021030000' + dldt_src_commit = '2021.4.1' + dldt_config = None + dldt_release = None build_cache_dir_default = os.environ.get('BUILD_CACHE_DIR', '.build_cache') build_subst_drive = os.environ.get('BUILD_SUBST_DRIVE', None) @@ -483,8 +489,9 @@ def main(): parser.add_argument('--cmake_option', action='append', help='Append OpenCV CMake option') parser.add_argument('--cmake_option_dldt', action='append', help='Append CMake option for DLDT project') - parser.add_argument('--clean_dldt', action='store_true', help='Clear DLDT build and sysroot directories') - parser.add_argument('--clean_opencv', action='store_true', help='Clear OpenCV build directory') + parser.add_argument('--clean_dldt', action='store_true', help='Clean DLDT build and sysroot directories') + parser.add_argument('--clean_dldt_sysroot', action='store_true', help='Clean DLDT sysroot directories') + parser.add_argument('--clean_opencv', action='store_true', help='Clean OpenCV build directory') parser.add_argument('--build_debug', action='store_true', help='Build debug binaries') parser.add_argument('--build_tests', action='store_true', help='Build OpenCV tests') @@ -494,12 +501,14 @@ def main(): parser.add_argument('--dldt_src_branch', help='DLDT checkout branch') parser.add_argument('--dldt_src_commit', default=dldt_src_commit, help='DLDT source commit / tag (default: %s)' % dldt_src_commit) parser.add_argument('--dldt_src_git_clone_extra', action='append', help='DLDT git clone extra args') - parser.add_argument('--dldt_release', default=dldt_release, help='DLDT release code for INF_ENGINE_RELEASE (default: %s)' % dldt_release) + parser.add_argument('--dldt_release', default=dldt_release, help='DLDT release code for INF_ENGINE_RELEASE, e.g 2021030000 (default: %s)' % dldt_release) parser.add_argument('--dldt_reference_dir', help='DLDT reference git repository (optional)') parser.add_argument('--dldt_src_dir', help='DLDT custom source repository (skip git checkout and patching, use for TESTING only)') - parser.add_argument('--dldt_config', help='Specify DLDT build configuration (defaults to evaluate from DLDT commit/branch)') + parser.add_argument('--dldt_config', default=dldt_config, help='Specify DLDT build configuration (defaults to evaluate from DLDT commit/branch)') + + parser.add_argument('--override_patch_hashsum', default='', help='(script debug mode)') args = parser.parse_args() diff --git a/platforms/winrt/readme.txt b/platforms/winrt/readme.txt index 2fb4ce1f54..2d1b4e6c30 100644 --- a/platforms/winrt/readme.txt +++ b/platforms/winrt/readme.txt @@ -13,7 +13,7 @@ Install Visual Studio 2013 Community Edition http://go.microsoft.com/?linkid=9863608 Install Visual Studio Express 2012 for Windows Desktop - http://www.microsoft.com/en-us/download/details.aspx?id=34673 + https://devblogs.microsoft.com/visualstudio/visual-studio-express-2012-for-windows-desktop-is-here/ @@ -156,4 +156,4 @@ Manual build cmake -G "Visual Studio 12 2013 Win64" -DCMAKE_SYSTEM_NAME:String=WindowsStore -DCMAKE_SYSTEM_VERSION:String=8.1 -DCMAKE_VS_EFFECTIVE_PLATFORMS:String=x64 -DCMAKE_INSTALL_PREFIX:PATH=.\install\WS\8.1\x64\ .. -Return to "Running tests for Windows Store", list item 4. \ No newline at end of file +Return to "Running tests for Windows Store", list item 4. diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 0c70698ccb..9d8f588083 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -45,7 +45,12 @@ endif() if(INSTALL_PYTHON_EXAMPLES) add_subdirectory(python) endif() - +# The examples in this folder will work with a semihosting version of +# OpenCV. For more information about semihosting, see +# https://developer.arm.com/documentation/100863/latest +if(OPENCV_SEMIHOSTING) + add_subdirectory(semihosting) +endif() ocv_install_example_src("." CMakeLists.txt samples_utils.cmake) if(INSTALL_C_EXAMPLES) install(DIRECTORY data DESTINATION "${OPENCV_SAMPLES_SRC_INSTALL_PATH}" COMPONENT samples_data) @@ -98,8 +103,10 @@ if(MSVC) endforeach(flag_var) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /NODEFAULTLIB:atlthunk.lib /NODEFAULTLIB:msvcrt.lib /NODEFAULTLIB:msvcrtd.lib") - set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /NODEFAULTLIB:libcmt.lib") - set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libcmtd.lib") + if(NOT BUILD_WITH_STATIC_CRT) + set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} /NODEFAULTLIB:libcmt.lib") + set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /NODEFAULTLIB:libcmtd.lib") + endif() endif() endif() diff --git a/samples/cpp/convexhull.cpp b/samples/cpp/convexhull.cpp index bb34be2469..d839b8061f 100644 --- a/samples/cpp/convexhull.cpp +++ b/samples/cpp/convexhull.cpp @@ -38,23 +38,14 @@ int main( int argc, char** argv ) points.push_back(pt); } - vector hull; - convexHull(Mat(points), hull, true); + vector hull; + convexHull(points, hull, true); img = Scalar::all(0); for( i = 0; i < count; i++ ) circle(img, points[i], 3, Scalar(0, 0, 255), FILLED, LINE_AA); - int hullcount = (int)hull.size(); - Point pt0 = points[hull[hullcount-1]]; - - for( i = 0; i < hullcount; i++ ) - { - Point pt = points[hull[i]]; - line(img, pt0, pt, Scalar(0, 255, 0), 1,LINE_AA); - pt0 = pt; - } - + polylines(img, hull, true, Scalar(0, 255, 0), 1, LINE_AA); imshow("hull", img); char key = (char)waitKey(); diff --git a/samples/cpp/grabcut.cpp b/samples/cpp/grabcut.cpp index d3e3db49f9..25492166a7 100644 --- a/samples/cpp/grabcut.cpp +++ b/samples/cpp/grabcut.cpp @@ -107,12 +107,14 @@ void GCApplication::showImage() const Mat res; Mat binMask; - if( !isInitialized ) - image->copyTo( res ); - else - { - getBinMask( mask, binMask ); - image->copyTo( res, binMask ); + image->copyTo( res ); + if( isInitialized ){ + getBinMask( mask, binMask); + + Mat black (binMask.rows, binMask.cols, CV_8UC3, cv::Scalar(0,0,0)); + black.setTo(Scalar::all(255), binMask); + + addWeighted(black, 0.5, res, 0.5, 0.0, res); } vector::const_iterator it; @@ -201,24 +203,39 @@ void GCApplication::mouseClick( int event, int x, int y, int flags, void* ) case EVENT_LBUTTONUP: if( rectState == IN_PROCESS ) { - rect = Rect( Point(rect.x, rect.y), Point(x,y) ); - rectState = SET; - setRectInMask(); - CV_Assert( bgdPxls.empty() && fgdPxls.empty() && prBgdPxls.empty() && prFgdPxls.empty() ); + if(rect.x == x || rect.y == y){ + rectState = NOT_SET; + } + else{ + rect = Rect( Point(rect.x, rect.y), Point(x,y) ); + rectState = SET; + setRectInMask(); + CV_Assert( bgdPxls.empty() && fgdPxls.empty() && prBgdPxls.empty() && prFgdPxls.empty() ); + } showImage(); } if( lblsState == IN_PROCESS ) { setLblsInMask(flags, Point(x,y), false); lblsState = SET; + nextIter(); showImage(); } + else{ + if(rectState == SET){ + nextIter(); + showImage(); + } + } break; case EVENT_RBUTTONUP: if( prLblsState == IN_PROCESS ) { setLblsInMask(flags, Point(x,y), true); prLblsState = SET; + } + if(rectState == SET){ + nextIter(); showImage(); } break; diff --git a/samples/cpp/kalman.cpp b/samples/cpp/kalman.cpp index 501a749124..daf0ba5a71 100644 --- a/samples/cpp/kalman.cpp +++ b/samples/cpp/kalman.cpp @@ -1,6 +1,6 @@ #include "opencv2/video/tracking.hpp" #include "opencv2/highgui.hpp" - +#include "opencv2/core/cvdef.h" #include using namespace cv; @@ -14,15 +14,19 @@ static void help() { printf( "\nExample of c calls to OpenCV's Kalman filter.\n" " Tracking of rotating point.\n" -" Rotation speed is constant.\n" +" Point moves in a circle and is characterized by a 1D state.\n" +" state_k+1 = state_k + speed + process_noise N(0, 1e-5)\n" +" The speed is constant.\n" " Both state and measurements vectors are 1D (a point angle),\n" -" Measurement is the real point angle + gaussian noise.\n" -" The real and the estimated points are connected with yellow line segment,\n" -" the real and the measured points are connected with red line segment.\n" +" Measurement is the real state + gaussian noise N(0, 1e-1).\n" +" The real and the measured points are connected with red line segment,\n" +" the real and the estimated points are connected with yellow line segment,\n" +" the real and the corrected estimated points are connected with green line segment.\n" " (if Kalman filter works correctly,\n" -" the yellow segment should be shorter than the red one).\n" +" the yellow segment should be shorter than the red one and\n" +" the green segment should be shorter than the yellow one)." "\n" -" Pressing any key (except ESC) will reset the tracking with a different speed.\n" +" Pressing any key (except ESC) will reset the tracking.\n" " Pressing ESC will stop the program.\n" ); } @@ -39,7 +43,9 @@ int main(int, char**) for(;;) { - randn( state, Scalar::all(0), Scalar::all(0.1) ); + img = Scalar::all(0); + state.at(0) = 0.0f; + state.at(1) = 2.f * (float)CV_PI / 6; KF.transitionMatrix = (Mat_(2, 2) << 1, 1, 0, 1); setIdentity(KF.measurementMatrix); @@ -60,36 +66,40 @@ int main(int, char**) double predictAngle = prediction.at(0); Point predictPt = calcPoint(center, R, predictAngle); - randn( measurement, Scalar::all(0), Scalar::all(KF.measurementNoiseCov.at(0))); - // generate measurement + randn( measurement, Scalar::all(0), Scalar::all(KF.measurementNoiseCov.at(0))); measurement += KF.measurementMatrix*state; double measAngle = measurement.at(0); Point measPt = calcPoint(center, R, measAngle); + // correct the state estimates based on measurements + // updates statePost & errorCovPost + KF.correct(measurement); + double improvedAngle = KF.statePost.at(0); + Point improvedPt = calcPoint(center, R, improvedAngle); + // plot points - #define drawCross( center, color, d ) \ - line( img, Point( center.x - d, center.y - d ), \ - Point( center.x + d, center.y + d ), color, 1, LINE_AA, 0); \ - line( img, Point( center.x + d, center.y - d ), \ - Point( center.x - d, center.y + d ), color, 1, LINE_AA, 0 ) + img = img * 0.2; + drawMarker(img, measPt, Scalar(0, 0, 255), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, predictPt, Scalar(0, 255, 255), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, improvedPt, Scalar(0, 255, 0), cv::MARKER_SQUARE, 5, 2); + drawMarker(img, statePt, Scalar(255, 255, 255), cv::MARKER_STAR, 10, 1); + // forecast one step + Mat test = Mat(KF.transitionMatrix*KF.statePost); + drawMarker(img, calcPoint(center, R, Mat(KF.transitionMatrix*KF.statePost).at(0)), + Scalar(255, 255, 0), cv::MARKER_SQUARE, 12, 1); - img = Scalar::all(0); - drawCross( statePt, Scalar(255,255,255), 3 ); - drawCross( measPt, Scalar(0,0,255), 3 ); - drawCross( predictPt, Scalar(0,255,0), 3 ); - line( img, statePt, measPt, Scalar(0,0,255), 3, LINE_AA, 0 ); - line( img, statePt, predictPt, Scalar(0,255,255), 3, LINE_AA, 0 ); + line( img, statePt, measPt, Scalar(0,0,255), 1, LINE_AA, 0 ); + line( img, statePt, predictPt, Scalar(0,255,255), 1, LINE_AA, 0 ); + line( img, statePt, improvedPt, Scalar(0,255,0), 1, LINE_AA, 0 ); - if(theRNG().uniform(0,4) != 0) - KF.correct(measurement); randn( processNoise, Scalar(0), Scalar::all(sqrt(KF.processNoiseCov.at(0, 0)))); state = KF.transitionMatrix*state + processNoise; imshow( "Kalman", img ); - code = (char)waitKey(100); + code = (char)waitKey(1000); if( code > 0 ) break; diff --git a/samples/cpp/lsd_lines.cpp b/samples/cpp/lsd_lines.cpp new file mode 100644 index 0000000000..3feed9cbc2 --- /dev/null +++ b/samples/cpp/lsd_lines.cpp @@ -0,0 +1,73 @@ +#include "opencv2/imgproc.hpp" +#include "opencv2/imgcodecs.hpp" +#include "opencv2/highgui.hpp" +#include + +using namespace std; +using namespace cv; + +int main(int argc, char** argv) +{ + cv::CommandLineParser parser(argc, argv, + "{input i|building.jpg|input image}" + "{refine r|false|if true use LSD_REFINE_STD method, if false use LSD_REFINE_NONE method}" + "{canny c|false|use Canny edge detector}" + "{overlay o|false|show result on input image}" + "{help h|false|show help message}"); + + if (parser.get("help")) + { + parser.printMessage(); + return 0; + } + + parser.printMessage(); + + String filename = samples::findFile(parser.get("input")); + bool useRefine = parser.get("refine"); + bool useCanny = parser.get("canny"); + bool overlay = parser.get("overlay"); + + Mat image = imread(filename, IMREAD_GRAYSCALE); + + if( image.empty() ) + { + cout << "Unable to load " << filename; + return 1; + } + + imshow("Source Image", image); + + if (useCanny) + { + Canny(image, image, 50, 200, 3); // Apply Canny edge detector + } + + // Create and LSD detector with standard or no refinement. + Ptr ls = useRefine ? createLineSegmentDetector(LSD_REFINE_STD) : createLineSegmentDetector(LSD_REFINE_NONE); + + double start = double(getTickCount()); + vector lines_std; + + // Detect the lines + ls->detect(image, lines_std); + + double duration_ms = (double(getTickCount()) - start) * 1000 / getTickFrequency(); + std::cout << "It took " << duration_ms << " ms." << std::endl; + + // Show found lines + if (!overlay || useCanny) + { + image = Scalar(0, 0, 0); + } + + ls->drawSegments(image, lines_std); + + String window_name = useRefine ? "Result - standard refinement" : "Result - no refinement"; + window_name += useCanny ? " - Canny edge detector used" : ""; + + imshow(window_name, image); + + waitKey(); + return 0; +} diff --git a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp index 5bcc878965..f9abbae945 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/MatchTemplate_Demo.cpp @@ -89,7 +89,7 @@ void MatchingMethod( int, void* ) //! [create_result_matrix] /// Create the result matrix - int result_cols = img.cols - templ.cols + 1; + int result_cols = img.cols - templ.cols + 1; int result_rows = img.rows - templ.rows + 1; result.create( result_rows, result_cols, CV_32FC1 ); diff --git a/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp b/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp index 61b6d607ce..bcb547a2fb 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/calcBackProject_Demo1.cpp @@ -72,18 +72,18 @@ void Hist_and_Backproj(int, void* ) //! [initialize] int histSize = MAX( bins, 2 ); float hue_range[] = { 0, 180 }; - const float* ranges = { hue_range }; + const float* ranges[] = { hue_range }; //! [initialize] //! [Get the Histogram and normalize it] Mat hist; - calcHist( &hue, 1, 0, Mat(), hist, 1, &histSize, &ranges, true, false ); + calcHist( &hue, 1, 0, Mat(), hist, 1, &histSize, ranges, true, false ); normalize( hist, hist, 0, 255, NORM_MINMAX, -1, Mat() ); //! [Get the Histogram and normalize it] //! [Get Backprojection] Mat backproj; - calcBackProject( &hue, 1, 0, hist, backproj, &ranges, 1, true ); + calcBackProject( &hue, 1, 0, hist, backproj, ranges, 1, true ); //! [Get Backprojection] //! [Draw the backproj] diff --git a/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp b/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp index 86167e519a..a7582e4282 100644 --- a/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp +++ b/samples/cpp/tutorial_code/Histograms_Matching/calcHist_Demo.cpp @@ -37,7 +37,7 @@ int main(int argc, char** argv) //! [Set the ranges ( for B,G,R) )] float range[] = { 0, 256 }; //the upper boundary is exclusive - const float* histRange = { range }; + const float* histRange[] = { range }; //! [Set the ranges ( for B,G,R) )] //! [Set histogram param] @@ -46,9 +46,9 @@ int main(int argc, char** argv) //! [Compute the histograms] Mat b_hist, g_hist, r_hist; - calcHist( &bgr_planes[0], 1, 0, Mat(), b_hist, 1, &histSize, &histRange, uniform, accumulate ); - calcHist( &bgr_planes[1], 1, 0, Mat(), g_hist, 1, &histSize, &histRange, uniform, accumulate ); - calcHist( &bgr_planes[2], 1, 0, Mat(), r_hist, 1, &histSize, &histRange, uniform, accumulate ); + calcHist( &bgr_planes[0], 1, 0, Mat(), b_hist, 1, &histSize, histRange, uniform, accumulate ); + calcHist( &bgr_planes[1], 1, 0, Mat(), g_hist, 1, &histSize, histRange, uniform, accumulate ); + calcHist( &bgr_planes[2], 1, 0, Mat(), r_hist, 1, &histSize, histRange, uniform, accumulate ); //! [Compute the histograms] //! [Draw the histograms for B, G and R] diff --git a/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp b/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp index a43024e557..e2835531e1 100644 --- a/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp +++ b/samples/cpp/tutorial_code/TrackingMotion/cornerDetector_Demo.cpp @@ -62,7 +62,7 @@ int main( int argc, char** argv ) { float lambda_1 = myHarris_dst.at(i, j)[0]; float lambda_2 = myHarris_dst.at(i, j)[1]; - Mc.at(i, j) = lambda_1*lambda_2 - 0.04f*pow( ( lambda_1 + lambda_2 ), 2 ); + Mc.at(i, j) = lambda_1*lambda_2 - 0.04f*((lambda_1 + lambda_2) * (lambda_1 + lambda_2)); } } diff --git a/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp b/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp new file mode 100644 index 0000000000..cfa9d22b0d --- /dev/null +++ b/samples/cpp/tutorial_code/core/how_to_use_OpenCV_parallel_for_/how_to_use_OpenCV_parallel_for_new.cpp @@ -0,0 +1,332 @@ +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +namespace +{ +//! [convolution-sequential] +void conv_seq(Mat src, Mat &dst, Mat kernel) +{ + //![convolution-make-borders] + int rows = src.rows, cols = src.cols; + dst = Mat(rows, cols, src.type()); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + //![convolution-make-borders] + + //! [convolution-kernel-loop] + for (int i = 0; i < rows; i++) + { + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + + for (int k = -sz; k <= sz; k++) + { + // slightly faster results when we create a ptr due to more efficient memory access. + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + //! [convolution-kernel-loop] +} +//! [convolution-sequential] + +#ifdef CV_CXX11 +void conv_parallel(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-cxx11] + parallel_for_(Range(0, rows * cols), [&](const Range &range) + { + for (int r = range.start; r < range.end; r++) + { + int i = r / cols, j = r % cols; + + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dst.ptr(i)[j] = saturate_cast(value); + } + }); + //! [convolution-parallel-cxx11] +} + +void conv_parallel_row_split(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-cxx11-row-split] + parallel_for_(Range(0, rows), [&](const Range &range) + { + for (int i = range.start; i < range.end; i++) + { + + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + }); + //! [convolution-parallel-cxx11-row-split] +} +#else + +//! [convolution-parallel] +class parallelConvolution : public ParallelLoopBody +{ +private: + Mat m_src, &m_dst; + Mat m_kernel; + int sz; + +public: + parallelConvolution(Mat src, Mat &dst, Mat kernel) + : m_src(src), m_dst(dst), m_kernel(kernel) + { + sz = kernel.rows / 2; + } + + //! [overload-full] + virtual void operator()(const Range &range) const CV_OVERRIDE + { + for (int r = range.start; r < range.end; r++) + { + int i = r / m_src.cols, j = r % m_src.cols; + + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = m_src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += m_kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + m_dst.ptr(i)[j] = saturate_cast(value); + } + } + //! [overload-full] +}; +//! [convolution-parallel] + +void conv_parallel(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-function] + parallelConvolution obj(src, dst, kernel); + parallel_for_(Range(0, rows * cols), obj); + //! [convolution-parallel-function] +} + +//! [conv-parallel-row-split] +class parallelConvolutionRowSplit : public ParallelLoopBody +{ +private: + Mat m_src, &m_dst; + Mat m_kernel; + int sz; + +public: + parallelConvolutionRowSplit(Mat src, Mat &dst, Mat kernel) + : m_src(src), m_dst(dst), m_kernel(kernel) + { + sz = kernel.rows / 2; + } + + //! [overload-row-split] + virtual void operator()(const Range &range) const CV_OVERRIDE + { + for (int i = range.start; i < range.end; i++) + { + + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + { + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } + } + //! [overload-row-split] +}; +//! [conv-parallel-row-split] + +void conv_parallel_row_split(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + + dst = Mat(rows, cols, CV_8UC1, Scalar(0)); + + // Taking care of edge values + // Make border = kernel.rows / 2; + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + + //! [convolution-parallel-function-row] + parallelConvolutionRowSplit obj(src, dst, kernel); + parallel_for_(Range(0, rows), obj); + //! [convolution-parallel-function-row] +} + +#endif + +static void help(char *progName) +{ + cout << endl + << " This program shows how to use the OpenCV parallel_for_ function and \n" + << " compares the performance of the sequential and parallel implementations for a \n" + << " convolution operation\n" + << " Usage:\n " + << progName << " [image_path -- default lena.jpg] " << endl + << endl; +} +} + +int main(int argc, char *argv[]) +{ + + help(argv[0]); + const char *filepath = argc >= 2 ? argv[1] : "../../../../data/lena.jpg"; + + Mat src, dst, kernel; + src = imread(filepath, IMREAD_GRAYSCALE); + + if (src.empty()) + { + cerr << "Can't open [" << filepath << "]" << endl; + return EXIT_FAILURE; + } + namedWindow("Input", 1); + namedWindow("Output1", 1); + namedWindow("Output2", 1); + namedWindow("Output3", 1); + imshow("Input", src); + + kernel = (Mat_(3, 3) << 1, 0, -1, + 1, 0, -1, + 1, 0, -1); + + /* + Uncomment the kernels you want to use or write your own kernels to test out + performance. + */ + + /* + kernel = (Mat_(5, 5) << 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1); + kernel /= 100; + */ + + /* + kernel = (Mat_(3, 3) << 1, 1, 1, + 0, 0, 0, + -1, -1, -1); + + */ + + double t = (double)getTickCount(); + + conv_seq(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential implementation: " << t << "s" << endl; + + imshow("Output1", dst); + waitKey(0); + + t = (double)getTickCount(); + + conv_parallel(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Parallel Implementation: " << t << "s" << endl; + + imshow("Output2", dst); + waitKey(0); + + t = (double)getTickCount(); + + conv_parallel_row_split(src, dst, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Parallel Implementation(Row Split): " << t << "s" << endl + << endl; + + imshow("Output3", dst); + waitKey(0); + + // imwrite("src.png", src); + // imwrite("dst.png", dst); + + return 0; +} \ No newline at end of file diff --git a/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp b/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp new file mode 100644 index 0000000000..9be4170d7b --- /dev/null +++ b/samples/cpp/tutorial_code/core/univ_intrin/univ_intrin.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +const int N = 100005, K = 2000; + +namespace +{ + +void conv_seq(Mat src, Mat &dst, Mat kernel) +{ + int rows = src.rows, cols = src.cols; + dst = Mat(rows, cols, CV_8UC1); + + int sz = kernel.rows / 2; + copyMakeBorder(src, src, sz, sz, sz, sz, BORDER_REPLICATE); + for (int i = 0; i < rows; i++) + { + uchar *dptr = dst.ptr(i); + for (int j = 0; j < cols; j++) + { + float value = 0; + + for (int k = -sz; k <= sz; k++) + { + // slightly faster results when we create a ptr due to more efficient memory access. + uchar *sptr = src.ptr(i + sz + k); + for (int l = -sz; l <= sz; l++) + { + value += kernel.ptr(k + sz)[l + sz] * sptr[j + sz + l]; + } + } + dptr[j] = saturate_cast(value); + } + } +} + +//! [convolution-1D-scalar] +void conv1d(Mat src, Mat &dst, Mat kernel) +{ + + //! [convolution-1D-border] + int len = src.cols; + dst = Mat(1, len, CV_8UC1); + + int sz = kernel.cols / 2; + copyMakeBorder(src, src, 0, 0, sz, sz, BORDER_REPLICATE); + //! [convolution-1D-border] + + //! [convolution-1D-scalar-main] + for (int i = 0; i < len; i++) + { + double value = 0; + for (int k = -sz; k <= sz; k++) + value += src.ptr(0)[i + k + sz] * kernel.ptr(0)[k + sz]; + + dst.ptr(0)[i] = saturate_cast(value); + } + //! [convolution-1D-scalar-main] +} +//! [convolution-1D-scalar] + +//! [convolution-1D-vector] +void conv1dsimd(Mat src, Mat kernel, float *ans, int row = 0, int rowk = 0, int len = -1) +{ + if (len == -1) + len = src.cols; + + //! [convolution-1D-convert] + Mat src_32, kernel_32; + + const int alpha = 1; + src.convertTo(src_32, CV_32FC1, alpha); + + int ksize = kernel.cols, sz = kernel.cols / 2; + copyMakeBorder(src_32, src_32, 0, 0, sz, sz, BORDER_REPLICATE); + //! [convolution-1D-convert] + + + //! [convolution-1D-main] + //! [convolution-1D-main-h1] + int step = v_float32().nlanes; + float *sptr = src_32.ptr(row), *kptr = kernel.ptr(rowk); + for (int k = 0; k < ksize; k++) + { + //! [convolution-1D-main-h1] + //! [convolution-1D-main-h2] + v_float32 kernel_wide = vx_setall_f32(kptr[k]); + int i; + for (i = 0; i + step < len; i += step) + { + v_float32 window = vx_load(sptr + i + k); + v_float32 sum = vx_load(ans + i) + kernel_wide * window; + v_store(ans + i, sum); + } + //! [convolution-1D-main-h2] + + //! [convolution-1D-main-h3] + for (; i < len; i++) + { + *(ans + i) += sptr[i + k]*kptr[k]; + } + //! [convolution-1D-main-h3] + } + //! [convolution-1D-main] +} +//! [convolution-1D-vector] + +//! [convolution-2D] +void convolute_simd(Mat src, Mat &dst, Mat kernel) +{ + //! [convolution-2D-init] + int rows = src.rows, cols = src.cols; + int ksize = kernel.rows, sz = ksize / 2; + dst = Mat(rows, cols, CV_32FC1); + + copyMakeBorder(src, src, sz, sz, 0, 0, BORDER_REPLICATE); + + int step = v_float32().nlanes; + //! [convolution-2D-init] + + //! [convolution-2D-main] + for (int i = 0; i < rows; i++) + { + for (int k = 0; k < ksize; k++) + { + float ans[N] = {0}; + conv1dsimd(src, kernel, ans, i + k, k, cols); + int j; + for (j = 0; j + step < cols; j += step) + { + v_float32 sum = vx_load(&dst.ptr(i)[j]) + vx_load(&ans[j]); + v_store(&dst.ptr(i)[j], sum); + } + + for (; j < cols; j++) + dst.ptr(i)[j] += ans[j]; + } + } + //! [convolution-2D-main] + + //! [convolution-2D-conv] + const int alpha = 1; + dst.convertTo(dst, CV_8UC1, alpha); + //! [convolution-2D-conv] +} +//! [convolution-2D] + +static void help(char *progName) +{ + cout << endl + << " This program shows how to use the OpenCV parallel_for_ function and \n" + << " compares the performance of the sequential and parallel implementations for a \n" + << " convolution operation\n" + << " Usage:\n " + << progName << " [image_path -- default lena.jpg] " << endl + << endl; +} +} + +int main(int argc, char *argv[]) +{ + + // 1-D Convolution // + Mat vsrc(1, N, CV_8UC1), k(1, K, CV_32FC1), vdst; + RNG rng(time(0)); + rng.RNG::fill(vsrc, RNG::UNIFORM, Scalar(0), Scalar(255)); + rng.RNG::fill(k, RNG::UNIFORM, Scalar(-50), Scalar(50)); + + double t = (double)getTickCount(); + conv1d(vsrc, vdst, k); + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential 1-D convolution implementation: " << t << "s" << endl; + + t = (double)getTickCount(); + float ans[N] = {0}; + conv1dsimd(vsrc, k, ans); + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Vectorized 1-D convolution implementation: " << t << "s" << endl; + + // 2-D Convolution // + help(argv[0]); + + const char *filepath = argc >= 2 ? argv[1] : "../../../../data/lena.jpg"; + + Mat src, dst1, dst2, kernel; + src = imread(filepath, IMREAD_GRAYSCALE); + + if (src.empty()) + { + cerr << "Can't open [" << filepath << "]" << endl; + return EXIT_FAILURE; + } + namedWindow("Input", 1); + namedWindow("Output", 1); + imshow("Input", src); + + kernel = (Mat_(3, 3) << 1, 0, -1, + 2, 0, -2, + 1, 0, -1); + + t = (double)getTickCount(); + + conv_seq(src, dst1, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Sequential 2-D convolution implementation: " << t << "s" << endl; + + imshow("Output", dst1); + waitKey(0); + + t = (double)getTickCount(); + + convolute_simd(src, dst2, kernel); + + t = ((double)getTickCount() - t) / getTickFrequency(); + cout << " Vectorized 2-D convolution implementation: " << t << "s" << endl + << endl; + + imshow("Output", dst2); + waitKey(0); + + return 0; +} \ No newline at end of file diff --git a/samples/dnn/CMakeLists.txt b/samples/dnn/CMakeLists.txt index 209fbb586c..9a1aeed339 100644 --- a/samples/dnn/CMakeLists.txt +++ b/samples/dnn/CMakeLists.txt @@ -4,6 +4,7 @@ set(OPENCV_DNN_SAMPLES_REQUIRED_DEPS opencv_core opencv_imgproc opencv_dnn + opencv_objdetect opencv_video opencv_imgcodecs opencv_videoio diff --git a/samples/dnn/colorization.cpp b/samples/dnn/colorization.cpp index b68e0ec4d8..6d751590d1 100644 --- a/samples/dnn/colorization.cpp +++ b/samples/dnn/colorization.cpp @@ -50,7 +50,7 @@ int main(int argc, char **argv) " https://github.com/richzhang/colorization\n" "Download caffemodel and prototxt files:\n" " http://eecs.berkeley.edu/~rich.zhang/projects/2016_colorization/files/demo_v2/colorization_release_v2.caffemodel\n" - " https://raw.githubusercontent.com/richzhang/colorization/master/colorization/models/colorization_deploy_v2.prototxt\n"; + " https://raw.githubusercontent.com/richzhang/colorization/caffe/models/colorization_deploy_v2.prototxt\n"; const string keys = "{ h help | | print this help message }" "{ proto | colorization_deploy_v2.prototxt | model configuration }" diff --git a/samples/dnn/colorization.py b/samples/dnn/colorization.py index c9eb2af3b6..5bdef9793e 100644 --- a/samples/dnn/colorization.py +++ b/samples/dnn/colorization.py @@ -1,6 +1,6 @@ # Script is based on https://github.com/richzhang/colorization/blob/master/colorization/colorize.py -# To download the caffemodel and the prototxt, see: https://github.com/richzhang/colorization/tree/master/colorization/models -# To download pts_in_hull.npy, see: https://github.com/richzhang/colorization/blob/master/colorization/resources/pts_in_hull.npy +# To download the caffemodel and the prototxt, see: https://github.com/richzhang/colorization/tree/caffe/colorization/models +# To download pts_in_hull.npy, see: https://github.com/richzhang/colorization/tree/caffe/colorization/resources/pts_in_hull.npy import numpy as np import argparse import cv2 as cv diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md index 7aba491c9d..1b1a28767c 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/README.md @@ -1,6 +1,6 @@ -# Run PaddlePaddle model by OpenCV +# Run PaddlePaddle model using OpenCV -This tutorial shows how to run PaddlePaddle model by opencv. +These two demonstrations show how to inference PaddlePaddle model using OpenCV. ## Environment Setup @@ -10,16 +10,69 @@ pip install paddlehub pip install paddle2onnx ``` -## Run PaddlePaddle model demo +## 1. Run PaddlePaddle ResNet50 using OpenCV -Run the example code as below, +### Run PaddlePaddle model demo + +Run the code sample as follows: ```shell python paddle_resnet50.py ``` -there are 3 part of this execution +There are three parts to the process: -- 1. Export PaddlePaddle ResNet50 model to onnx format; -- 2. Use `cv2.dnn.readNetFromONNX` load model file; -- 3. Preprocess image file and do inference. +1. Export PaddlePaddle ResNet50 model to onnx format. +2. Use `cv2.dnn.readNetFromONNX` to load the model file. +3. Preprocess image file and do the inference. + +## 2. Run PaddleSeg Portrait Segmentation using OpenCV + +### Convert to ONNX Model + +#### 1. Get Paddle Inference model + +For more details, please refer to [PaddleSeg](https://github.com/PaddlePaddle/PaddleSeg/blob/release/2.1/contrib/HumanSeg/README.md). + +```shell +wget https://x2paddle.bj.bcebos.com/inference/models/humanseg_hrnet18_small_v1.zip +unzip humanseg_hrnet18_small_v1.zip +``` + +Notes: + +* The exported model must have a fixed input shape, as dynamic is not supported at this moment. + +#### 2. Convert to ONNX model using paddle2onnx + +To convert the model, use the following command: + +``` +paddle2onnx --model_dir humanseg_hrnet18_small_v1 \ + --model_filename model.pdmodel \ + --params_filename model.pdiparams \ + --opset_version 11 \ + --save_file humanseg_hrnet18_tiny.onnx +``` + +The converted model can be found in the current directory by the name `humanseg_hrnet18_tiny.onnx` . + +### Run PaddleSeg Portrait Segmentation demo + +Run the code sample as follows: + +```shell +python paddle_humanseg.py +``` + +There are three parts to the process: + +1. Use `cv2.dnn.readNetFromONNX` to load the model file. +2. Preprocess image file and do inference. +3. Postprocess image file and visualize. + +The resulting file can be found at `data/result_test_human.jpg` . + +### Portrait segmentation visualization + + diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg new file mode 100644 index 0000000000..652b03f3b4 Binary files /dev/null and b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/data/result_test_human.jpg differ diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py new file mode 100644 index 0000000000..e2ef62eade --- /dev/null +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_humanseg.py @@ -0,0 +1,112 @@ +import os +import paddlehub.vision.transforms as T +import numpy as np +import cv2 as cv + + +def get_color_map_list(num_classes): + """ + Returns the color map for visualizing the segmentation mask, + which can support arbitrary number of classes. + + Args: + num_classes (int): Number of classes. + + Returns: + (list). The color map. + """ + + num_classes += 1 + color_map = num_classes * [0, 0, 0] + for i in range(0, num_classes): + j = 0 + lab = i + while lab: + color_map[i * 3] |= (((lab >> 0) & 1) << (7 - j)) + color_map[i * 3 + 1] |= (((lab >> 1) & 1) << (7 - j)) + color_map[i * 3 + 2] |= (((lab >> 2) & 1) << (7 - j)) + j += 1 + lab >>= 3 + color_map = color_map[3:] + return color_map + + +def visualize(image, result, save_dir=None, weight=0.6): + """ + Convert predict result to color image, and save added image. + + Args: + image (str): The path of origin image. + result (np.ndarray): The predict result of image. + save_dir (str): The directory for saving visual image. Default: None. + weight (float): The image weight of visual image, and the result weight is (1 - weight). Default: 0.6 + + Returns: + vis_result (np.ndarray): If `save_dir` is None, return the visualized result. + """ + + color_map = get_color_map_list(256) + color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)] + color_map = np.array(color_map).astype("uint8") + # Use OpenCV LUT for color mapping + c1 = cv.LUT(result, color_map[:, 0]) + c2 = cv.LUT(result, color_map[:, 1]) + c3 = cv.LUT(result, color_map[:, 2]) + pseudo_img = np.dstack((c1, c2, c3)) + + im = cv.imread(image) + vis_result = cv.addWeighted(im, weight, pseudo_img, 1 - weight, 0) + + if save_dir is not None: + if not os.path.exists(save_dir): + os.makedirs(save_dir) + image_name = os.path.split(image)[-1] + out_path = os.path.join(save_dir, image_name) + cv.imwrite(out_path, vis_result) + else: + return vis_result + + +def preprocess(image_path): + ''' preprocess input image file to np.ndarray + + Args: + image_path(str): Path of input image file + + Returns: + ProcessedImage(numpy.ndarray): A numpy.ndarray + variable which shape is (1, 3, 192, 192) + ''' + transforms = T.Compose([ + T.Resize((192, 192)), + T.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + ], + to_rgb=True) + return np.expand_dims(transforms(image_path), axis=0) + + +if __name__ == '__main__': + img_path = "../../../../data/messi5.jpg" + # load PPSeg Model use cv.dnn + net = cv.dnn.readNetFromONNX('humanseg_hrnet18_tiny.onnx') + # read and preprocess image file + im = preprocess(img_path) + # inference + net.setInput(im) + result = net.forward(['save_infer_model/scale_0.tmp_1']) + # post process + image = cv.imread(img_path) + r, c, _ = image.shape + result = np.argmax(result[0], axis=1).astype(np.uint8) + result = cv.resize(result[0, :, :], + dsize=(c, r), + interpolation=cv.INTER_NEAREST) + + print("grid_image.shape is: ", result.shape) + folder_path = "data" + if not os.path.exists(folder_path): + os.makedirs(folder_path) + file_path = os.path.join(folder_path, '%s.jpg' % "result_test_human") + result_color = visualize(img_path, result) + cv.imwrite(file_path, result_color) + print('%s saved' % file_path) diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py index b95ce917e6..90c0d26e15 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py +++ b/samples/dnn/dnn_model_runner/dnn_conversion/paddlepaddle/paddle_resnet50.py @@ -16,15 +16,15 @@ def preprocess(image_path): variable which shape is (1, 3, 224, 224) ''' transforms = T.Compose([ - T.Resize((256, 256)), - T.CenterCrop(224), - T.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225])], - to_rgb=True) + T.Resize((256, 256)), + T.CenterCrop(224), + T.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225])], + to_rgb=True) return np.expand_dims(transforms(image_path), axis=0) -def export_onnx_mobilenetv2(save_path): +def export_onnx_resnet50(save_path): ''' export PaddlePaddle model to ONNX format Args: @@ -35,7 +35,7 @@ def export_onnx_mobilenetv2(save_path): ''' model = hub.Module(name="resnet50_vd_imagenet_ssld") input_spec = paddle.static.InputSpec( - [1, 3, 224, 224], "float32", "image") + [1, 3, 224, 224], "float32", "image") paddle.onnx.export(model, save_path, input_spec=[input_spec], opset_version=10) @@ -45,9 +45,9 @@ if __name__ == '__main__': save_path = './resnet50' image_file = './data/cat.jpg' labels = open('./data/labels.txt').read().strip().split('\n') - model = export_onnx_mobilenetv2(save_path) + model = export_onnx_resnet50(save_path) - # load mobilenetv2 use cv.dnn + # load resnet50 use cv.dnn net = cv.dnn.readNetFromONNX(save_path + '.onnx') # read and preprocess image file im = preprocess(image_file) diff --git a/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt b/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt index eb217e27df..6887c2ab2c 100644 --- a/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt +++ b/samples/dnn/dnn_model_runner/dnn_conversion/requirements.txt @@ -12,3 +12,4 @@ paddlepaddle>=2.0.0 paddlepaddle-gpu>=2.0.0 paddlehub>=2.1.0 paddle2onnx>=0.5.1 +paddleseg>=2.0.0 \ No newline at end of file diff --git a/samples/dnn/face_detect.cpp b/samples/dnn/face_detect.cpp new file mode 100644 index 0000000000..8d91a10968 --- /dev/null +++ b/samples/dnn/face_detect.cpp @@ -0,0 +1,132 @@ +#include +#include +#include +#include + +#include + +using namespace cv; +using namespace std; + +static Mat visualize(Mat input, Mat faces, int thickness=2) +{ + Mat output = input.clone(); + for (int i = 0; i < faces.rows; i++) + { + // Print results + cout << "Face " << i + << ", top-left coordinates: (" << faces.at(i, 0) << ", " << faces.at(i, 1) << "), " + << "box width: " << faces.at(i, 2) << ", box height: " << faces.at(i, 3) << ", " + << "score: " << faces.at(i, 14) << "\n"; + + // Draw bounding box + rectangle(output, Rect2i(int(faces.at(i, 0)), int(faces.at(i, 1)), int(faces.at(i, 2)), int(faces.at(i, 3))), Scalar(0, 255, 0), thickness); + // Draw landmarks + circle(output, Point2i(int(faces.at(i, 4)), int(faces.at(i, 5))), 2, Scalar(255, 0, 0), thickness); + circle(output, Point2i(int(faces.at(i, 6)), int(faces.at(i, 7))), 2, Scalar( 0, 0, 255), thickness); + circle(output, Point2i(int(faces.at(i, 8)), int(faces.at(i, 9))), 2, Scalar( 0, 255, 0), thickness); + circle(output, Point2i(int(faces.at(i, 10)), int(faces.at(i, 11))), 2, Scalar(255, 0, 255), thickness); + circle(output, Point2i(int(faces.at(i, 12)), int(faces.at(i, 13))), 2, Scalar( 0, 255, 255), thickness); + } + return output; +} + +int main(int argc, char ** argv) +{ + CommandLineParser parser(argc, argv, + "{help h | | Print this message.}" + "{input i | | Path to the input image. Omit for detecting on default camera.}" + "{model m | yunet.onnx | Path to the model. Download yunet.onnx in https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.}" + "{score_threshold | 0.9 | Filter out faces of score < score_threshold.}" + "{nms_threshold | 0.3 | Suppress bounding boxes of iou >= nms_threshold.}" + "{top_k | 5000 | Keep top_k bounding boxes before NMS.}" + "{save s | false | Set true to save results. This flag is invalid when using camera.}" + "{vis v | true | Set true to open a window for result visualization. This flag is invalid when using camera.}" + ); + if (argc == 1 || parser.has("help")) + { + parser.printMessage(); + return -1; + } + + String modelPath = parser.get("model"); + + float scoreThreshold = parser.get("score_threshold"); + float nmsThreshold = parser.get("nms_threshold"); + int topK = parser.get("top_k"); + + bool save = parser.get("save"); + bool vis = parser.get("vis"); + + // Initialize FaceDetectorYN + Ptr detector = FaceDetectorYN::create(modelPath, "", Size(320, 320), scoreThreshold, nmsThreshold, topK); + + // If input is an image + if (parser.has("input")) + { + String input = parser.get("input"); + Mat image = imread(input); + + // Set input size before inference + detector->setInputSize(image.size()); + + // Inference + Mat faces; + detector->detect(image, faces); + + // Draw results on the input image + Mat result = visualize(image, faces); + + // Save results if save is true + if(save) + { + cout << "Results saved to result.jpg\n"; + imwrite("result.jpg", result); + } + + // Visualize results + if (vis) + { + namedWindow(input, WINDOW_AUTOSIZE); + imshow(input, result); + waitKey(0); + } + } + else + { + int deviceId = 0; + VideoCapture cap; + cap.open(deviceId, CAP_ANY); + int frameWidth = int(cap.get(CAP_PROP_FRAME_WIDTH)); + int frameHeight = int(cap.get(CAP_PROP_FRAME_HEIGHT)); + detector->setInputSize(Size(frameWidth, frameHeight)); + + Mat frame; + TickMeter tm; + String msg = "FPS: "; + while(waitKey(1) < 0) // Press any key to exit + { + // Get frame + if (!cap.read(frame)) + { + cerr << "No frames grabbed!\n"; + break; + } + + // Inference + Mat faces; + tm.start(); + detector->detect(frame, faces); + tm.stop(); + + // Draw results on the input image + Mat result = visualize(frame, faces); + putText(result, msg + to_string(tm.getFPS()), Point(0, 15), FONT_HERSHEY_SIMPLEX, 0.5, Scalar(0, 255, 0)); + + // Visualize results + imshow("Live", result); + + tm.reset(); + } + } +} \ No newline at end of file diff --git a/samples/dnn/face_detect.py b/samples/dnn/face_detect.py new file mode 100644 index 0000000000..65069d6590 --- /dev/null +++ b/samples/dnn/face_detect.py @@ -0,0 +1,101 @@ +import argparse + +import numpy as np +import cv2 as cv + +def str2bool(v): + if v.lower() in ['on', 'yes', 'true', 'y', 't']: + return True + elif v.lower() in ['off', 'no', 'false', 'n', 'f']: + return False + else: + raise NotImplementedError + +parser = argparse.ArgumentParser() +parser.add_argument('--input', '-i', type=str, help='Path to the input image.') +parser.add_argument('--model', '-m', type=str, default='yunet.onnx', help='Path to the model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') +parser.add_argument('--score_threshold', type=float, default=0.9, help='Filtering out faces of score < score_threshold.') +parser.add_argument('--nms_threshold', type=float, default=0.3, help='Suppress bounding boxes of iou >= nms_threshold.') +parser.add_argument('--top_k', type=int, default=5000, help='Keep top_k bounding boxes before NMS.') +parser.add_argument('--save', '-s', type=str2bool, default=False, help='Set true to save results. This flag is invalid when using camera.') +parser.add_argument('--vis', '-v', type=str2bool, default=True, help='Set true to open a window for result visualization. This flag is invalid when using camera.') +args = parser.parse_args() + +def visualize(input, faces, thickness=2): + output = input.copy() + if faces[1] is not None: + for idx, face in enumerate(faces[1]): + print('Face {}, top-left coordinates: ({:.0f}, {:.0f}), box width: {:.0f}, box height {:.0f}, score: {:.2f}'.format(idx, face[0], face[1], face[2], face[3], face[-1])) + + coords = face[:-1].astype(np.int32) + cv.rectangle(output, (coords[0], coords[1]), (coords[0]+coords[2], coords[1]+coords[3]), (0, 255, 0), 2) + cv.circle(output, (coords[4], coords[5]), 2, (255, 0, 0), 2) + cv.circle(output, (coords[6], coords[7]), 2, (0, 0, 255), 2) + cv.circle(output, (coords[8], coords[9]), 2, (0, 255, 0), 2) + cv.circle(output, (coords[10], coords[11]), 2, (255, 0, 255), 2) + cv.circle(output, (coords[12], coords[13]), 2, (0, 255, 255), 2) + return output + +if __name__ == '__main__': + + # Instantiate FaceDetectorYN + detector = cv.FaceDetectorYN.create( + args.model, + "", + (320, 320), + args.score_threshold, + args.nms_threshold, + args.top_k + ) + + # If input is an image + if args.input is not None: + image = cv.imread(args.input) + + # Set input size before inference + detector.setInputSize((image.shape[1], image.shape[0])) + + # Inference + faces = detector.detect(image) + + # Draw results on the input image + result = visualize(image, faces) + + # Save results if save is true + if args.save: + print('Resutls saved to result.jpg\n') + cv.imwrite('result.jpg', result) + + # Visualize results in a new window + if args.vis: + cv.namedWindow(args.input, cv.WINDOW_AUTOSIZE) + cv.imshow(args.input, result) + cv.waitKey(0) + else: # Omit input to call default camera + deviceId = 0 + cap = cv.VideoCapture(deviceId) + frameWidth = int(cap.get(cv.CAP_PROP_FRAME_WIDTH)) + frameHeight = int(cap.get(cv.CAP_PROP_FRAME_HEIGHT)) + detector.setInputSize([frameWidth, frameHeight]) + + tm = cv.TickMeter() + while cv.waitKey(1) < 0: + hasFrame, frame = cap.read() + if not hasFrame: + print('No frames grabbed!') + break + + # Inference + tm.start() + faces = detector.detect(frame) # faces is a tuple + tm.stop() + + # Draw results on the input image + frame = visualize(frame, faces) + + cv.putText(frame, 'FPS: {}'.format(tm.getFPS()), (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) + + # Visualize results in a new Window + cv.imshow('Live', frame) + + tm.reset() \ No newline at end of file diff --git a/samples/dnn/face_match.cpp b/samples/dnn/face_match.cpp new file mode 100644 index 0000000000..f24134b890 --- /dev/null +++ b/samples/dnn/face_match.cpp @@ -0,0 +1,103 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "opencv2/dnn.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" + +#include + +#include "opencv2/objdetect.hpp" + + +using namespace cv; +using namespace std; + + +int main(int argc, char ** argv) +{ + if (argc != 5) + { + std::cerr << "Usage " << argv[0] << ": " + << " " + << " " + << "" + << "\n"; + return -1; + } + + String det_onnx_path = argv[1]; + String reg_onnx_path = argv[2]; + String image1_path = argv[3]; + String image2_path = argv[4]; + std::cout< faceDetector; + + faceDetector = FaceDetectorYN::create(det_onnx_path, "", image1.size(), score_thresh, nms_thresh, top_k); + Mat faces_1; + faceDetector->detect(image1, faces_1); + if (faces_1.rows < 1) + { + std::cerr << "Cannot find a face in " << image1_path << "\n"; + return -1; + } + + faceDetector = FaceDetectorYN::create(det_onnx_path, "", image2.size(), score_thresh, nms_thresh, top_k); + Mat faces_2; + faceDetector->detect(image2, faces_2); + if (faces_2.rows < 1) + { + std::cerr << "Cannot find a face in " << image2_path << "\n"; + return -1; + } + + // Initialize FaceRecognizerSF + Ptr faceRecognizer = FaceRecognizerSF::create(reg_onnx_path, ""); + + + Mat aligned_face1, aligned_face2; + faceRecognizer->alignCrop(image1, faces_1.row(0), aligned_face1); + faceRecognizer->alignCrop(image2, faces_2.row(0), aligned_face2); + + Mat feature1, feature2; + faceRecognizer->feature(aligned_face1, feature1); + feature1 = feature1.clone(); + faceRecognizer->feature(aligned_face2, feature2); + feature2 = feature2.clone(); + + double cos_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_COSINE); + double L2_score = faceRecognizer->match(feature1, feature2, FaceRecognizerSF::DisType::FR_NORM_L2); + + if(cos_score >= cosine_similar_thresh) + { + std::cout << "They have the same identity;"; + } + else + { + std::cout << "They have different identities;"; + } + std::cout << " Cosine Similarity: " << cos_score << ", threshold: " << cosine_similar_thresh << ". (higher value means higher similarity, max 1.0)\n"; + + if(L2_score <= l2norm_similar_thresh) + { + std::cout << "They have the same identity;"; + } + else + { + std::cout << "They have different identities."; + } + std::cout << " NormL2 Distance: " << L2_score << ", threshold: " << l2norm_similar_thresh << ". (lower value means higher similarity, min 0.0)\n"; + + return 0; +} diff --git a/samples/dnn/face_match.py b/samples/dnn/face_match.py new file mode 100644 index 0000000000..b36c9f6367 --- /dev/null +++ b/samples/dnn/face_match.py @@ -0,0 +1,57 @@ +import argparse + +import numpy as np +import cv2 as cv + +parser = argparse.ArgumentParser() +parser.add_argument('--input1', '-i1', type=str, help='Path to the input image1.') +parser.add_argument('--input2', '-i2', type=str, help='Path to the input image2.') +parser.add_argument('--face_detection_model', '-fd', type=str, help='Path to the face detection model. Download the model at https://github.com/ShiqiYu/libfacedetection.train/tree/master/tasks/task1/onnx.') +parser.add_argument('--face_recognition_model', '-fr', type=str, help='Path to the face recognition model. Download the model at https://drive.google.com/file/d/1ClK9WiB492c5OZFKveF3XiHCejoOxINW/view.') +args = parser.parse_args() + +# Read the input image +img1 = cv.imread(args.input1) +img2 = cv.imread(args.input2) + +# Instantiate face detector and recognizer +detector = cv.FaceDetectorYN.create( + args.face_detection_model, + "", + (img1.shape[1], img1.shape[0]) +) +recognizer = cv.FaceRecognizerSF.create( + args.face_recognition_model, + "" +) + +# Detect face +detector.setInputSize((img1.shape[1], img1.shape[0])) +face1 = detector.detect(img1) +detector.setInputSize((img2.shape[1], img2.shape[0])) +face2 = detector.detect(img2) +assert face1[1].shape[0] > 0, 'Cannot find a face in {}'.format(args.input1) +assert face2[1].shape[0] > 0, 'Cannot find a face in {}'.format(args.input2) + +# Align faces +face1_align = recognizer.alignCrop(img1, face1[1][0]) +face2_align = recognizer.alignCrop(img2, face2[1][0]) + +# Extract features +face1_feature = recognizer.faceFeature(face1_align) +face2_feature = recognizer.faceFeature(face2_align) + +# Calculate distance (0: cosine, 1: L2) +cosine_similarity_threshold = 0.363 +cosine_score = recognizer.faceMatch(face1_feature, face2_feature, 0) +msg = 'different identities' +if cosine_score >= cosine_similarity_threshold: + msg = 'the same identity' +print('They have {}. Cosine Similarity: {}, threshold: {} (higher value means higher similarity, max 1.0).'.format(msg, cosine_score, cosine_similarity_threshold)) + +l2_similarity_threshold = 1.128 +l2_score = recognizer.faceMatch(face1_feature, face2_feature, 1) +msg = 'different identities' +if l2_score <= l2_similarity_threshold: + msg = 'the same identity' +print('They have {}. NormL2 Distance: {}, threshold: {} (lower value means higher similarity, min 0.0).'.format(msg, l2_score, l2_similarity_threshold)) \ No newline at end of file diff --git a/samples/dnn/results/audrybt1.jpg b/samples/dnn/results/audrybt1.jpg new file mode 100644 index 0000000000..5829f82309 Binary files /dev/null and b/samples/dnn/results/audrybt1.jpg differ diff --git a/samples/dnn/speech_recognition.py b/samples/dnn/speech_recognition.py new file mode 100644 index 0000000000..025607edab --- /dev/null +++ b/samples/dnn/speech_recognition.py @@ -0,0 +1,506 @@ +import numpy as np +import cv2 as cv +import argparse +import os +import soundfile as sf # Temporary import to load audio files + +''' + You can download the converted onnx model from https://drive.google.com/drive/folders/1wLtxyao4ItAg8tt4Sb63zt6qXzhcQoR6?usp=sharing + or convert the model yourself. + + You can get the original pre-trained Jasper model from NVIDIA : https://ngc.nvidia.com/catalog/models/nvidia:jasper_pyt_onnx_fp16_amp/files + Download and unzip : `$ wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/jasper_pyt_onnx_fp16_amp/versions/20.10.0/zip -O jasper_pyt_onnx_fp16_amp_20.10.0.zip && unzip -o ./jasper_pyt_onnx_fp16_amp_20.10.0.zip && unzip -o ./jasper_pyt_onnx_fp16_amp.zip` + + you can get the script to convert the model here : https://gist.github.com/spazewalker/507f1529e19aea7e8417f6e935851a01 + + You can convert the model using the following steps: + 1. Import onnx and load the original model + ``` + import onnx + model = onnx.load("./jasper-onnx/1/model.onnx") + ``` + + 3. Change data type of input layer + ``` + inp = model.graph.input[0] + model.graph.input.remove(inp) + inp.type.tensor_type.elem_type = 1 + model.graph.input.insert(0,inp) + ``` + + 4. Change the data type of output layer + ``` + out = model.graph.output[0] + model.graph.output.remove(out) + out.type.tensor_type.elem_type = 1 + model.graph.output.insert(0,out) + ``` + + 5. Change the data type of every initializer and cast it's values from FP16 to FP32 + ``` + for i,init in enumerate(model.graph.initializer): + model.graph.initializer.remove(init) + init.data_type = 1 + init.raw_data = np.frombuffer(init.raw_data, count=np.product(init.dims), dtype=np.float16).astype(np.float32).tobytes() + model.graph.initializer.insert(i,init) + ``` + + 6. Add an additional reshape node to handle the inconsistant input from python and c++ of openCV. + see https://github.com/opencv/opencv/issues/19091 + Make & insert a new node with 'Reshape' operation & required initializer + ``` + tensor = numpy_helper.from_array(np.array([0,64,-1]),name='shape_reshape') + model.graph.initializer.insert(0,tensor) + node = onnx.helper.make_node(op_type='Reshape',inputs=['input__0','shape_reshape'], outputs=['input_reshaped'], name='reshape__0') + model.graph.node.insert(0,node) + model.graph.node[1].input[0] = 'input_reshaped' + ``` + + 7. Finally save the model + ``` + with open('jasper_dynamic_input_float.onnx','wb') as f: + onnx.save_model(model,f) + ``` + + Original Repo : https://github.com/NVIDIA/DeepLearningExamples/tree/master/PyTorch/SpeechRecognition/Jasper + ''' + +class FilterbankFeatures: + def __init__(self, + sample_rate=16000, window_size=0.02, window_stride=0.01, + n_fft=512, preemph=0.97, n_filt=64, lowfreq=0, + highfreq=None, log=True, dither=1e-5): + ''' + Initializes pre-processing class. Default values are the values used by the Jasper + architecture for pre-processing. For more details, refer to the paper here: + https://arxiv.org/abs/1904.03288 + ''' + self.win_length = int(sample_rate * window_size) # frame size + self.hop_length = int(sample_rate * window_stride) # stride + self.n_fft = n_fft or 2 ** np.ceil(np.log2(self.win_length)) + self.log = log + self.dither = dither + self.n_filt = n_filt + self.preemph = preemph + highfreq = highfreq or sample_rate / 2 + self.window_tensor = np.hanning(self.win_length) + + self.filterbanks = self.mel(sample_rate, self.n_fft, n_mels=n_filt, fmin=lowfreq, fmax=highfreq) + self.filterbanks.dtype=np.float32 + self.filterbanks = np.expand_dims(self.filterbanks,0) + + def normalize_batch(self, x, seq_len): + ''' + Normalizes the features. + ''' + x_mean = np.zeros((seq_len.shape[0], x.shape[1]), dtype=x.dtype) + x_std = np.zeros((seq_len.shape[0], x.shape[1]), dtype=x.dtype) + for i in range(x.shape[0]): + x_mean[i, :] = np.mean(x[i, :, :seq_len[i]],axis=1) + x_std[i, :] = np.std(x[i, :, :seq_len[i]],axis=1) + # make sure x_std is not zero + x_std += 1e-10 + return (x - np.expand_dims(x_mean,2)) / np.expand_dims(x_std,2) + + def calculate_features(self, x, seq_len): + ''' + Calculates filterbank features. + args: + x : mono channel audio + seq_len : length of the audio sample + returns: + x : filterbank features + ''' + dtype = x.dtype + + seq_len = np.ceil(seq_len / self.hop_length) + seq_len = np.array(seq_len,dtype=np.int32) + + # dither + if self.dither > 0: + x += self.dither * np.random.randn(*x.shape) + + # do preemphasis + if self.preemph is not None: + x = np.concatenate( + (np.expand_dims(x[0],-1), x[1:] - self.preemph * x[:-1]), axis=0) + + # Short Time Fourier Transform + x = self.stft(x, n_fft=self.n_fft, hop_length=self.hop_length, + win_length=self.win_length, + fft_window=self.window_tensor) + + # get power spectrum + x = (x**2).sum(-1) + + # dot with filterbank energies + x = np.matmul(np.array(self.filterbanks,dtype=x.dtype), x) + + # log features if required + if self.log: + x = np.log(x + 1e-20) + + # normalize if required + x = self.normalize_batch(x, seq_len).astype(dtype) + return x + + # Mel Frequency calculation + def hz_to_mel(self, frequencies): + ''' + Converts frequencies from hz to mel scale. Input can be a number or a vector. + ''' + frequencies = np.asanyarray(frequencies) + + f_min = 0.0 + f_sp = 200.0 / 3 + + mels = (frequencies - f_min) / f_sp + + # Fill in the log-scale part + min_log_hz = 1000.0 # beginning of log region (Hz) + min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels) + logstep = np.log(6.4) / 27.0 # step size for log region + + if frequencies.ndim: + # If we have array data, vectorize + log_t = frequencies >= min_log_hz + mels[log_t] = min_log_mel + np.log(frequencies[log_t] / min_log_hz) / logstep + elif frequencies >= min_log_hz: + # If we have scalar data, directly + mels = min_log_mel + np.log(frequencies / min_log_hz) / logstep + return mels + + def mel_to_hz(self, mels): + ''' + Converts frequencies from mel to hz scale. Input can be a number or a vector. + ''' + mels = np.asanyarray(mels) + + # Fill in the linear scale + f_min = 0.0 + f_sp = 200.0 / 3 + freqs = f_min + f_sp * mels + + # And now the nonlinear scale + min_log_hz = 1000.0 # beginning of log region (Hz) + min_log_mel = (min_log_hz - f_min) / f_sp # same (Mels) + logstep = np.log(6.4) / 27.0 # step size for log region + + if mels.ndim: + # If we have vector data, vectorize + log_t = mels >= min_log_mel + freqs[log_t] = min_log_hz * np.exp(logstep * (mels[log_t] - min_log_mel)) + elif mels >= min_log_mel: + # If we have scalar data, check directly + freqs = min_log_hz * np.exp(logstep * (mels - min_log_mel)) + + return freqs + + def mel_frequencies(self, n_mels=128, fmin=0.0, fmax=11025.0): + ''' + Calculates n mel frequencies between 2 frequencies + args: + n_mels : number of bands + fmin : min frequency + fmax : max frequency + returns: + mels : vector of mel frequencies + ''' + # 'Center freqs' of mel bands - uniformly spaced between limits + min_mel = self.hz_to_mel(fmin) + max_mel = self.hz_to_mel(fmax) + + mels = np.linspace(min_mel, max_mel, n_mels) + + return self.mel_to_hz(mels) + + def mel(self, sr, n_fft, n_mels=128, fmin=0.0, fmax=None, dtype=np.float32): + ''' + Generates mel filterbank + args: + sr : Sampling rate + n_fft : number of FFT components + n_mels : number of Mel bands to generate + fmin : lowest frequency (in Hz) + fmax : highest frequency (in Hz). sr/2.0 if None + dtype : the data type of the output basis. + returns: + mels : Mel transform matrix + ''' + # default Max freq = half of sampling rate + if fmax is None: + fmax = float(sr) / 2 + + # Initialize the weights + n_mels = int(n_mels) + weights = np.zeros((n_mels, int(1 + n_fft // 2)), dtype=dtype) + + # Center freqs of each FFT bin + fftfreqs = np.linspace(0, float(sr) / 2, int(1 + n_fft // 2), endpoint=True) + + # 'Center freqs' of mel bands - uniformly spaced between limits + mel_f = self.mel_frequencies(n_mels + 2, fmin=fmin, fmax=fmax) + + fdiff = np.diff(mel_f) + ramps = np.subtract.outer(mel_f, fftfreqs) + + for i in range(n_mels): + # lower and upper slopes for all bins + lower = -ramps[i] / fdiff[i] + upper = ramps[i + 2] / fdiff[i + 1] + + # .. then intersect them with each other and zero + weights[i] = np.maximum(0, np.minimum(lower, upper)) + + # Using Slaney-style mel which is scaled to be approx constant energy per channel + enorm = 2.0 / (mel_f[2 : n_mels + 2] - mel_f[:n_mels]) + weights *= enorm[:, np.newaxis] + return weights + + # STFT preperation + def pad_window_center(self, data, size, axis=-1, **kwargs): + ''' + Centers the data and pads. + args: + data : Vector to be padded and centered + size : Length to pad data + axis : Axis along which to pad and center the data + kwargs : arguments passed to np.pad + return : centered and padded data + ''' + kwargs.setdefault("mode", "constant") + n = data.shape[axis] + lpad = int((size - n) // 2) + lengths = [(0, 0)] * data.ndim + lengths[axis] = (lpad, int(size - n - lpad)) + if lpad < 0: + raise Exception( + ("Target size ({:d}) must be at least input size ({:d})").format(size, n) + ) + return np.pad(data, lengths, **kwargs) + + def frame(self, x, frame_length, hop_length): + ''' + Slices a data array into (overlapping) frames. + args: + x : array to frame + frame_length : length of frame + hop_length : Number of steps to advance between frames + return : A framed view of `x` + ''' + if x.shape[-1] < frame_length: + raise Exception( + "Input is too short (n={:d})" + " for frame_length={:d}".format(x.shape[-1], frame_length) + ) + x = np.asfortranarray(x) + n_frames = 1 + (x.shape[-1] - frame_length) // hop_length + strides = np.asarray(x.strides) + new_stride = np.prod(strides[strides > 0] // x.itemsize) * x.itemsize + shape = list(x.shape)[:-1] + [frame_length, n_frames] + strides = list(strides) + [hop_length * new_stride] + return np.lib.stride_tricks.as_strided(x, shape=shape, strides=strides) + + def dtype_r2c(self, d, default=np.complex64): + ''' + Find the complex numpy dtype corresponding to a real dtype. + args: + d : The real-valued dtype to convert to complex. + default : The default complex target type, if `d` does not match a known dtype + return : The complex dtype + ''' + mapping = { + np.dtype(np.float32): np.complex64, + np.dtype(np.float64): np.complex128, + } + dt = np.dtype(d) + if dt.kind == "c": + return dt + return np.dtype(mapping.get(dt, default)) + + def stft(self, y, n_fft, hop_length=None, win_length=None, fft_window=None, pad_mode='reflect', return_complex=False): + ''' + Short Time Fourier Transform. The STFT represents a signal in the time-frequency + domain by computing discrete Fourier transforms (DFT) over short overlapping windows. + args: + y : input signal + n_fft : length of the windowed signal after padding with zeros. + hop_length : number of audio samples between adjacent STFT columns. + win_length : Each frame of audio is windowed by window of length win_length and + then padded with zeros to match n_fft + fft_window : a vector or array of length `n_fft` having values computed by a + window function + pad_mode : mode while padding the singnal + return_complex : returns array with complex data type if `True` + return : Matrix of short-term Fourier transform coefficients. + ''' + if win_length is None: + win_length = n_fft + if hop_length is None: + hop_length = int(win_length // 4) + if y.ndim!=1: + raise Exception(f'Invalid input shape. Only Mono Channeled audio supported. Input must have shape (Audio,). Got {y.shape}') + + # Pad the window out to n_fft size + fft_window = self.pad_window_center(fft_window, n_fft) + + # Reshape so that the window can be broadcast + fft_window = fft_window.reshape((-1, 1)) + + # Pad the time series so that frames are centered + y = np.pad(y, int(n_fft // 2), mode=pad_mode) + + # Window the time series. + y_frames = self.frame(y, frame_length=n_fft, hop_length=hop_length) + + # Convert data type to complex + dtype = self.dtype_r2c(y.dtype) + + # Pre-allocate the STFT matrix + stft_matrix = np.empty( (int(1 + n_fft // 2), y_frames.shape[-1]), dtype=dtype, order="F") + + stft_matrix = np.fft.rfft( fft_window * y_frames, axis=0) + return stft_matrix if return_complex==True else np.stack((stft_matrix.real,stft_matrix.imag),axis=-1) + +class Decoder: + ''' + Used for decoding the output of jasper model. + ''' + def __init__(self): + labels=[' ','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',"'"] + self.labels_map = {i: label for i,label in enumerate(labels)} + self.blank_id = 28 + + def decode(self,x): + """ + Takes output of Jasper model and performs ctc decoding algorithm to + remove duplicates and special symbol. Returns prediction + """ + x = np.argmax(x,axis=-1) + hypotheses = [] + prediction = x.tolist() + # CTC decoding procedure + decoded_prediction = [] + previous = self.blank_id + for p in prediction: + if (p != previous or previous == self.blank_id) and p != self.blank_id: + decoded_prediction.append(p) + previous = p + hypothesis = ''.join([self.labels_map[c] for c in decoded_prediction]) + hypotheses.append(hypothesis) + return hypotheses + +def predict(features, net, decoder): + ''' + Passes the features through the Jasper model and decodes the output to english transcripts. + args: + features : input features, calculated using FilterbankFeatures class + net : Jasper model dnn.net object + decoder : Decoder object + return : Predicted text + ''' + # This is a workaround https://github.com/opencv/opencv/issues/19091 + # expanding 1 dimentions allows us to pass it to the network + # from python. This should be resolved in the future. + features = np.expand_dims(features,axis=3) + + # make prediction + net.setInput(features) + output = net.forward() + + # decode output to transcript + prediction = decoder.decode(output.squeeze(0)) + return prediction[0] + +if __name__ == '__main__': + + # Computation backends supported by layers + backends = (cv.dnn.DNN_BACKEND_DEFAULT, cv.dnn.DNN_BACKEND_INFERENCE_ENGINE, cv.dnn.DNN_BACKEND_OPENCV) + # Target Devices for computation + targets = (cv.dnn.DNN_TARGET_CPU, cv.dnn.DNN_TARGET_OPENCL, cv.dnn.DNN_TARGET_OPENCL_FP16) + + parser = argparse.ArgumentParser(description='This script runs Jasper Speech recognition model', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument('--input_audio', type=str, required=True, help='Path to input audio file. OR Path to a txt file with relative path to multiple audio files in different lines') + parser.add_argument('--show_spectrogram', action='store_true', help='Whether to show a spectrogram of the input audio.') + parser.add_argument('--model', type=str, default='jasper.onnx', help='Path to the onnx file of Jasper. default="jasper.onnx"') + parser.add_argument('--output', type=str, help='Path to file where recognized audio transcript must be saved. Leave this to print on console.') + parser.add_argument('--backend', choices=backends, default=cv.dnn.DNN_BACKEND_DEFAULT, type=int, + help='Select a computation backend: ' + "%d: automatically (by default) " + "%d: OpenVINO Inference Engine " + "%d: OpenCV Implementation " % backends) + parser.add_argument('--target', choices=targets, default=cv.dnn.DNN_TARGET_CPU, type=int, + help='Select a target device: ' + "%d: CPU target (by default) " + "%d: OpenCL " + "%d: OpenCL FP16 " % targets) + + args, _ = parser.parse_known_args() + + if args.input_audio and not os.path.isfile(args.input_audio): + raise OSError("Input audio file does not exist") + if not os.path.isfile(args.model): + raise OSError("Jasper model file does not exist") + if args.input_audio.endswith('.txt'): + with open(args.input_audio) as f: + content = f.readlines() + content = [x.strip() for x in content] + audio_file_paths = content + for audio_file_path in audio_file_paths: + if not os.path.isfile(audio_file_path): + raise OSError("Audio file({audio_file_path}) does not exist") + else: + audio_file_paths = [args.input_audio] + audio_file_paths = [os.path.abspath(x) for x in audio_file_paths] + + # Read audio Files + features = [] + try: + for audio_file_path in audio_file_paths: + audio = sf.read(audio_file_path) + # If audio is stereo, just take one channel. + X = audio[0] if audio[0].ndim==1 else audio[0][:,0] + features.append(X) + except: + raise Exception(f"Soundfile cannot read {args.input_audio}. Try a different format") + + # Get Filterbank Features + feature_extractor = FilterbankFeatures() + for i in range(len(features)): + X = features[i] + seq_len = np.array([X.shape[0]], dtype=np.int32) + features[i] = feature_extractor.calculate_features(x=X, seq_len=seq_len) + + # Load Network + net = cv.dnn.readNetFromONNX(args.model) + net.setPreferableBackend(args.backend) + net.setPreferableTarget(args.target) + + # Show spectogram if required + if args.show_spectrogram and not args.input_audio.endswith('.txt'): + img = cv.normalize(src=features[0][0], dst=None, alpha=0, beta=255, norm_type=cv.NORM_MINMAX, dtype=cv.CV_8U) + img = cv.applyColorMap(img, cv.COLORMAP_JET) + cv.imshow('spectogram', img) + cv.waitKey(0) + + # Initialize decoder + decoder = Decoder() + + # Make prediction + prediction = [] + print("Predicting...") + for feature in features: + print(f"\rAudio file {len(prediction)+1}/{len(features)}", end='') + prediction.append(predict(feature, net, decoder)) + print("") + + # save transcript if required + if args.output: + with open(args.output,'w') as f: + for pred in prediction: + f.write(pred+'\n') + print("Transcript was written to {}".format(args.output)) + else: + print(prediction) + cv.destroyAllWindows() diff --git a/samples/dnn/text_detection.py b/samples/dnn/text_detection.py index d86c03d251..6fb1e90901 100644 --- a/samples/dnn/text_detection.py +++ b/samples/dnn/text_detection.py @@ -222,8 +222,8 @@ def main(): 0.5, (255, 0, 0)) for j in range(4): - p1 = (vertices[j][0], vertices[j][1]) - p2 = (vertices[(j + 1) % 4][0], vertices[(j + 1) % 4][1]) + p1 = (int(vertices[j][0]), int(vertices[j][1])) + p2 = (int(vertices[(j + 1) % 4][0]), int(vertices[(j + 1) % 4][1])) cv.line(frame, p1, p2, (0, 255, 0), 1) # Put efficiency information diff --git a/samples/gdb/gdbinit b/samples/gdb/gdbinit new file mode 100644 index 0000000000..228e8f7023 --- /dev/null +++ b/samples/gdb/gdbinit @@ -0,0 +1,23 @@ +set auto-load local-gdbinit on +set print elements 0 +add-auto-load-safe-path / + +python +# Update GDB's Python paths with the `sys.path` values of the local +# Python installation, whether that is brew'ed Python, a virtualenv, +# or another system python. + +# Convert GDB to interpret in Python + +import os, subprocess, sys + +# Execute a Python using the user's shell and pull out the sys.path (for site-packages) +paths = subprocess.check_output('/usr/bin/python3 -c "import os,sys;print(os.linesep.join(sys.path).strip())"',shell=True).decode("utf-8").split() + +# Extend GDB's Python's search path +sys.path.extend(paths) + +end + + +source /your/path/to/mat_pretty_printer.py diff --git a/samples/gdb/mat_pretty_printer.py b/samples/gdb/mat_pretty_printer.py new file mode 100644 index 0000000000..e6ad2cbde2 --- /dev/null +++ b/samples/gdb/mat_pretty_printer.py @@ -0,0 +1,212 @@ +import gdb +import numpy as np +from enum import Enum + +np.set_printoptions(suppress=True) # prevent numpy exponential notation on print, default False +# np.set_printoptions(threshold=sys.maxsize) + + +def conv(obj, t): + return gdb.parse_and_eval(f'({t})({obj})') + + +def booli(obj): + return conv(str(obj).lower(), 'bool') + + +def stri(obj): + s = f'"{obj}"' + return conv(s.translate(s.maketrans('\n', ' ')), 'char*') + + +class MagicValues(Enum): + MAGIC_VAL = 0x42FF0000 + AUTO_STEP = 0 + CONTINUOUS_FLAG = 1 << 14 + SUBMATRIX_FLAG = 1 << 15 + + +class MagicMasks(Enum): + MAGIC_MASK = 0xFFFF0000 + TYPE_MASK = 0x00000FFF + DEPTH_MASK = 7 + + +class Depth(Enum): + CV_8U = 0 + CV_8S = 1 + CV_16U = 2 + CV_16S = 3 + CV_32S = 4 + CV_32F = 5 + CV_64F = 6 + CV_16F = 7 + + +def create_enum(n): + def make_type(depth, cn): + return depth.value + ((cn - 1) << 3) + defs = [(f'{depth.name}C{i}', make_type(depth, i)) for depth in Depth for i in range(1, n + 1)] + return Enum('Type', defs) + + +Type = create_enum(512) + + +class Flags: + def depth(self): + return Depth(self.flags & MagicMasks.DEPTH_MASK.value) + + def dtype(self): + depth = self.depth() + ret = None + + if depth == Depth.CV_8U: + ret = (np.uint8, 'uint8_t') + elif depth == Depth.CV_8S: + ret = (np.int8, 'int8_t') + elif depth == Depth.CV_16U: + ret = (np.uint16, 'uint16_t') + elif depth == Depth.CV_16S: + ret = (np.int16, 'int16_t') + elif depth == Depth.CV_32S: + ret = (np.int32, 'int32_t') + elif depth == Depth.CV_32F: + ret = (np.float32, 'float') + elif depth == Depth.CV_64F: + ret = (np.float64, 'double') + elif depth == Depth.CV_16F: + ret = (np.float16, 'float16') + + return ret + + def type(self): + return Type(self.flags & MagicMasks.TYPE_MASK.value) + + def channels(self): + return ((self.flags & (511 << 3)) >> 3) + 1 + + def is_continuous(self): + return (self.flags & MagicValues.CONTINUOUS_FLAG.value) != 0 + + def is_submatrix(self): + return (self.flags & MagicValues.SUBMATRIX_FLAG.value) != 0 + + def __init__(self, flags): + self.flags = flags + + def __iter__(self): + return iter({ + 'type': stri(self.type().name), + 'is_continuous': booli(self.is_continuous()), + 'is_submatrix': booli(self.is_submatrix()) + }.items()) + + +class Size: + def __init__(self, ptr): + self.ptr = ptr + + def dims(self): + return int((self.ptr - 1).dereference()) + + def to_numpy(self): + return np.array([int(self.ptr[i]) for i in range(self.dims())], dtype=np.int64) + + def __iter__(self): + return iter({'size': stri(self.to_numpy())}.items()) + + +class Mat: + def __init__(self, m, size, flags): + (dtype, ctype) = flags.dtype() + elsize = np.dtype(dtype).itemsize + + ptr = m['data'] + dataptr = int(ptr) + length = (int(m['dataend']) - dataptr) // elsize + start = (int(m['datastart']) - dataptr) // elsize + + if length == 0: + self.mat = np.array([]) + self.view = self.mat + return + + if dtype != np.float16: + ctype = gdb.lookup_type(ctype) + ptr = ptr.cast(ctype.array(length - 1).pointer()).dereference() + self.mat = np.array([ptr[i] for i in range(length)], dtype=dtype) + else: + u16 = gdb.lookup_type('uint16_t') + ptr = ptr.cast(u16.array(length - 1).pointer()).dereference() + self.mat = np.array([ptr[i] for i in range(length)], dtype=np.uint16) + self.mat = self.mat.view(np.float16) + + steps = np.asarray([int(m['step']['p'][i]) for i in range(size.dims())], dtype=np.int64) + self.view = np.lib.stride_tricks.as_strided(self.mat[start:], shape=size.to_numpy(), strides=steps) + + def __iter__(self): + return iter({'data': stri(self.view)}.items()) + + +class MatPrinter: + """Print a cv::Mat""" + + def __init__(self, mat): + self.mat = mat + + def views(self): + m = self.mat + + flags = Flags(int(m['flags'])) + size = Size(m['size']['p']) + data = Mat(m, size, flags) + + for x in [flags, size, data]: + for k, v in x: + yield 'view_' + k, v + + def real(self): + m = self.mat + + for field in m.type.fields(): + k = field.name + v = m[k] + yield k, v + + # TODO: add an enum in interface.h with all cv::Mat element types and use that instead + # yield 'test', gdb.parse_and_eval(f'(cv::MatTypes)0') + + def children(self): # TODO: hide real members under new child somehow + yield from self.views() + yield from self.real() + + +def get_type(val): + # Get the type. + vtype = val.type + + # If it points to a reference, get the reference. + if vtype.code == gdb.TYPE_CODE_REF: + vtype = vtype.target() + + # Get the unqualified type, stripped of typedefs. + vtype = vtype.unqualified().strip_typedefs() + + # Get the type name. + typename = vtype.tag + + return typename + + +def mat_printer(val): + typename = get_type(val) + + if typename is None: + return None + + if str(typename) == 'cv::Mat': + return MatPrinter(val) + + +gdb.pretty_printers.append(mat_printer) diff --git a/samples/python/camera_calibration_show_extrinsics.py b/samples/python/camera_calibration_show_extrinsics.py index 0118b5b913..d676691f15 100755 --- a/samples/python/camera_calibration_show_extrinsics.py +++ b/samples/python/camera_calibration_show_extrinsics.py @@ -188,7 +188,7 @@ def main(): fig = plt.figure() ax = fig.gca(projection='3d') - ax.set_aspect("equal") + ax.set_aspect("auto") cam_width = args.cam_width cam_height = args.cam_height diff --git a/samples/python/gaussian_mix.py b/samples/python/gaussian_mix.py index 5f2dfcc440..4c1f86794c 100755 --- a/samples/python/gaussian_mix.py +++ b/samples/python/gaussian_mix.py @@ -28,11 +28,11 @@ def make_gaussians(cluster_n, img_size): return points, ref_distrs def draw_gaussain(img, mean, cov, color): - x, y = np.int32(mean) + x, y = mean w, u, _vt = cv.SVDecomp(cov) ang = np.arctan2(u[1, 0], u[0, 0])*(180/np.pi) s1, s2 = np.sqrt(w)*3.0 - cv.ellipse(img, (x, y), (s1, s2), ang, 0, 360, color, 1, cv.LINE_AA) + cv.ellipse(img, (int(x), int(y)), (int(s1), int(s2)), ang, 0, 360, color, 1, cv.LINE_AA) def main(): diff --git a/samples/python/hist.py b/samples/python/hist.py index 4c2c1ad395..8c1f4546a8 100755 --- a/samples/python/hist.py +++ b/samples/python/hist.py @@ -46,9 +46,9 @@ def hist_lines(im): im = cv.cvtColor(im,cv.COLOR_BGR2GRAY) hist_item = cv.calcHist([im],[0],None,[256],[0,256]) cv.normalize(hist_item,hist_item,0,255,cv.NORM_MINMAX) - hist=np.int32(np.around(hist_item)) + hist = np.int32(np.around(hist_item)) for x,y in enumerate(hist): - cv.line(h,(x,0),(x,y),(255,255,255)) + cv.line(h,(x,0),(x,y[0]),(255,255,255)) y = np.flipud(h) return y diff --git a/samples/python/kalman.py b/samples/python/kalman.py index 654e3de3da..cf152a8700 100755 --- a/samples/python/kalman.py +++ b/samples/python/kalman.py @@ -1,14 +1,18 @@ #!/usr/bin/env python """ Tracking of rotating point. - Rotation speed is constant. + Point moves in a circle and is characterized by a 1D state. + state_k+1 = state_k + speed + process_noise N(0, 1e-5) + The speed is constant. Both state and measurements vectors are 1D (a point angle), - Measurement is the real point angle + gaussian noise. - The real and the estimated points are connected with yellow line segment, - the real and the measured points are connected with red line segment. + Measurement is the real state + gaussian noise N(0, 1e-1). + The real and the measured points are connected with red line segment, + the real and the estimated points are connected with yellow line segment, + the real and the corrected estimated points are connected with green line segment. (if Kalman filter works correctly, - the yellow segment should be shorter than the red one). - Pressing any key (except ESC) will reset the tracking with a different speed. + the yellow segment should be shorter than the red one and + the green segment should be shorter than the yellow one). + Pressing any key (except ESC) will reset the tracking. Pressing ESC will stop the program. """ # Python 2/3 compatibility @@ -21,8 +25,7 @@ if PY3: import numpy as np import cv2 as cv -from math import cos, sin, sqrt -import numpy as np +from math import cos, sin, sqrt, pi def main(): img_height = 500 @@ -30,64 +33,62 @@ def main(): kalman = cv.KalmanFilter(2, 1, 0) code = long(-1) - - cv.namedWindow("Kalman") - + num_circle_steps = 12 while True: - state = 0.1 * np.random.randn(2, 1) - - kalman.transitionMatrix = np.array([[1., 1.], [0., 1.]]) - kalman.measurementMatrix = 1. * np.ones((1, 2)) - kalman.processNoiseCov = 1e-5 * np.eye(2) - kalman.measurementNoiseCov = 1e-1 * np.ones((1, 1)) - kalman.errorCovPost = 1. * np.ones((2, 2)) - kalman.statePost = 0.1 * np.random.randn(2, 1) + img = np.zeros((img_height, img_width, 3), np.uint8) + state = np.array([[0.0],[(2 * pi) / num_circle_steps]]) # start state + kalman.transitionMatrix = np.array([[1., 1.], [0., 1.]]) # F. input + kalman.measurementMatrix = 1. * np.eye(1, 2) # H. input + kalman.processNoiseCov = 1e-5 * np.eye(2) # Q. input + kalman.measurementNoiseCov = 1e-1 * np.ones((1, 1)) # R. input + kalman.errorCovPost = 1. * np.eye(2, 2) # P._k|k KF state var + kalman.statePost = 0.1 * np.random.randn(2, 1) # x^_k|k KF state var while True: def calc_point(angle): - return (np.around(img_width/2 + img_width/3*cos(angle), 0).astype(int), - np.around(img_height/2 - img_width/3*sin(angle), 1).astype(int)) - + return (np.around(img_width / 2. + img_width / 3.0 * cos(angle), 0).astype(int), + np.around(img_height / 2. - img_width / 3.0 * sin(angle), 1).astype(int)) + img = img * 1e-3 state_angle = state[0, 0] state_pt = calc_point(state_angle) - + # advance Kalman filter to next timestep + # updates statePre, statePost, errorCovPre, errorCovPost + # k-> k+1, x'(k) = A*x(k) + # P'(k) = temp1*At + Q prediction = kalman.predict() - predict_angle = prediction[0, 0] - predict_pt = calc_point(predict_angle) - - measurement = kalman.measurementNoiseCov * np.random.randn(1, 1) + predict_pt = calc_point(prediction[0, 0]) # equivalent to calc_point(kalman.statePre[0,0]) # generate measurement + measurement = kalman.measurementNoiseCov * np.random.randn(1, 1) measurement = np.dot(kalman.measurementMatrix, state) + measurement measurement_angle = measurement[0, 0] measurement_pt = calc_point(measurement_angle) - # plot points - def draw_cross(center, color, d): - cv.line(img, - (center[0] - d, center[1] - d), (center[0] + d, center[1] + d), - color, 1, cv.LINE_AA, 0) - cv.line(img, - (center[0] + d, center[1] - d), (center[0] - d, center[1] + d), - color, 1, cv.LINE_AA, 0) - - img = np.zeros((img_height, img_width, 3), np.uint8) - draw_cross(np.int32(state_pt), (255, 255, 255), 3) - draw_cross(np.int32(measurement_pt), (0, 0, 255), 3) - draw_cross(np.int32(predict_pt), (0, 255, 0), 3) - - cv.line(img, state_pt, measurement_pt, (0, 0, 255), 3, cv.LINE_AA, 0) - cv.line(img, state_pt, predict_pt, (0, 255, 255), 3, cv.LINE_AA, 0) - + # correct the state estimates based on measurements + # updates statePost & errorCovPost kalman.correct(measurement) + improved_pt = calc_point(kalman.statePost[0, 0]) - process_noise = sqrt(kalman.processNoiseCov[0,0]) * np.random.randn(2, 1) - state = np.dot(kalman.transitionMatrix, state) + process_noise + # plot points + cv.drawMarker(img, measurement_pt, (0, 0, 255), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, predict_pt, (0, 255, 255), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, improved_pt, (0, 255, 0), cv.MARKER_SQUARE, 5, 2) + cv.drawMarker(img, state_pt, (255, 255, 255), cv.MARKER_STAR, 10, 1) + # forecast one step + cv.drawMarker(img, calc_point(np.dot(kalman.transitionMatrix, kalman.statePost)[0, 0]), + (255, 255, 0), cv.MARKER_SQUARE, 12, 1) + + cv.line(img, state_pt, measurement_pt, (0, 0, 255), 1, cv.LINE_AA, 0) # red measurement error + cv.line(img, state_pt, predict_pt, (0, 255, 255), 1, cv.LINE_AA, 0) # yellow pre-meas error + cv.line(img, state_pt, improved_pt, (0, 255, 0), 1, cv.LINE_AA, 0) # green post-meas error + + # update the real process + process_noise = sqrt(kalman.processNoiseCov[0, 0]) * np.random.randn(2, 1) + state = np.dot(kalman.transitionMatrix, state) + process_noise # x_k+1 = F x_k + w_k cv.imshow("Kalman", img) - - code = cv.waitKey(100) + code = cv.waitKey(1000) if code != -1: break diff --git a/samples/python/lk_homography.py b/samples/python/lk_homography.py index 808f30965f..38a05f63b6 100755 --- a/samples/python/lk_homography.py +++ b/samples/python/lk_homography.py @@ -77,8 +77,8 @@ class App: for (x0, y0), (x1, y1), good in zip(self.p0[:,0], self.p1[:,0], status[:,0]): if good: - cv.line(vis, (x0, y0), (x1, y1), (0, 128, 0)) - cv.circle(vis, (x1, y1), 2, (red, green)[good], -1) + cv.line(vis, (int(x0), int(y0)), (int(x1), int(y1)), (0, 128, 0)) + cv.circle(vis, (int(x1), int(y1)), 2, (red, green)[good], -1) draw_str(vis, (20, 20), 'track count: %d' % len(self.p1)) if self.use_ransac: draw_str(vis, (20, 40), 'RANSAC') @@ -86,7 +86,7 @@ class App: p = cv.goodFeaturesToTrack(frame_gray, **feature_params) if p is not None: for x, y in p[:,0]: - cv.circle(vis, (x, y), 2, green, -1) + cv.circle(vis, (int(x), int(y)), 2, green, -1) draw_str(vis, (20, 20), 'feature count: %d' % len(p)) cv.imshow('lk_homography', vis) diff --git a/samples/python/lk_track.py b/samples/python/lk_track.py index 7b77f1b335..97a8c40241 100755 --- a/samples/python/lk_track.py +++ b/samples/python/lk_track.py @@ -65,7 +65,7 @@ class App: if len(tr) > self.track_len: del tr[0] new_tracks.append(tr) - cv.circle(vis, (x, y), 2, (0, 255, 0), -1) + cv.circle(vis, (int(x), int(y)), 2, (0, 255, 0), -1) self.tracks = new_tracks cv.polylines(vis, [np.int32(tr) for tr in self.tracks], False, (0, 255, 0)) draw_str(vis, (20, 20), 'track count: %d' % len(self.tracks)) diff --git a/samples/python/morphology.py b/samples/python/morphology.py index 9ecf5b0682..183f5e8288 100755 --- a/samples/python/morphology.py +++ b/samples/python/morphology.py @@ -50,8 +50,11 @@ def main(): cur_str_mode = str_modes.next() def update(dummy=None): - sz = cv.getTrackbarPos('op/size', 'morphology') - iters = cv.getTrackbarPos('iters', 'morphology') + try: # do not get trackbar position while trackbar is not created + sz = cv.getTrackbarPos('op/size', 'morphology') + iters = cv.getTrackbarPos('iters', 'morphology') + except: + return opers = cur_mode.split('/') if len(opers) > 1: sz = sz - 10 diff --git a/samples/python/stitching_detailed.py b/samples/python/stitching_detailed.py index a7e316105e..4ee29048d1 100644 --- a/samples/python/stitching_detailed.py +++ b/samples/python/stitching_detailed.py @@ -450,7 +450,8 @@ def main(): cameras[i].focal *= compose_work_aspect cameras[i].ppx *= compose_work_aspect cameras[i].ppy *= compose_work_aspect - sz = (full_img_sizes[i][0] * compose_scale, full_img_sizes[i][1] * compose_scale) + sz = (int(round(full_img_sizes[i][0] * compose_scale)), + int(round(full_img_sizes[i][1] * compose_scale))) K = cameras[i].K().astype(np.float32) roi = warper.warpRoi(sz, K, cameras[i].R) corners.append(roi[0:2]) diff --git a/samples/python/video_v4l2.py b/samples/python/video_v4l2.py index 61b1e35804..abebb2a2ca 100644 --- a/samples/python/video_v4l2.py +++ b/samples/python/video_v4l2.py @@ -30,7 +30,7 @@ def main(): color = (0, 255, 0) cap = cv.VideoCapture(0) - cap.set(cv.CAP_PROP_AUTOFOCUS, False) # Known bug: https://github.com/opencv/opencv/pull/5474 + cap.set(cv.CAP_PROP_AUTOFOCUS, 0) # Known bug: https://github.com/opencv/opencv/pull/5474 cv.namedWindow("Video") @@ -67,7 +67,7 @@ def main(): break elif k == ord('g'): convert_rgb = not convert_rgb - cap.set(cv.CAP_PROP_CONVERT_RGB, convert_rgb) + cap.set(cv.CAP_PROP_CONVERT_RGB, 1 if convert_rgb else 0) print('Done') diff --git a/samples/semihosting/CMakeLists.txt b/samples/semihosting/CMakeLists.txt new file mode 100644 index 0000000000..9fddb0587b --- /dev/null +++ b/samples/semihosting/CMakeLists.txt @@ -0,0 +1,10 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(SEMIHOSTING_SUFFIX semihosting) + +add_subdirectory(include) +set(RAW_PIXEL_INCLUDE ${CMAKE_CURRENT_BINARY_DIR}/include) +add_subdirectory(histogram) +add_subdirectory(norm) diff --git a/samples/semihosting/README.md b/samples/semihosting/README.md new file mode 100644 index 0000000000..881b09b735 --- /dev/null +++ b/samples/semihosting/README.md @@ -0,0 +1,27 @@ +# Arm semihosting + +This folder contain a toolchain file and a couple of examples for +building OpenCV based applications that can run in an [Arm +semihosting](https://developer.arm.com/documentation/100863/latest) +setup. + +OpenCV can be compiled to target a semihosting platform as follows: + +``` +cmake ../opencv/ \ + -DCMAKE_TOOLCHAIN_FILE=../opencv/platforms/semihosting/aarch64-semihosting.toolchain.cmake \ + -DSEMIHOSTING_TOOLCHAIN_PATH=/path/to/baremetal-toolchain/bin/ \ + -DBUILD_EXAMPLES=ON -GNinja +``` + +A barematel toolchain for targeting aarch64 semihosting can be found +[here](https://developer.arm.com/tools-and-software/open-source-software/developer-tools/gnu-toolchain/gnu-a/downloads), +under `aarch64-none-elf`. + +The code of the examples in the `norm` and `histogram` folders can be +executed with qemu in Linux userspace: + +``` + qemu-aarch64 ./bin/example_semihosting_histogram + qemu-aarch64 ./bin/example_semihosting_norm +``` diff --git a/samples/semihosting/histogram/CMakeLists.txt b/samples/semihosting/histogram/CMakeLists.txt new file mode 100644 index 0000000000..d2f065d1b9 --- /dev/null +++ b/samples/semihosting/histogram/CMakeLists.txt @@ -0,0 +1,26 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(PROJECT_NAME histogram) +project(${PROJECT_NAME}) + +ocv_install_example_src(histogram *.cpp *.hpp CMakeLists.txt) + +set(LOCAL_DEPS + opencv_core + opencv_imgproc + ${OPENCV_MODULES_PUBLIC} + ${OpenCV_LIB_COMPONENTS}) +ocv_check_dependencies(${LOCAL_DEPS}) + +if(NOT OCV_DEPENDENCIES_FOUND) + return() +endif() + +ocv_define_sample(histogram histogram.cpp ${SEMIHOSTING_SUFFIX}) +ocv_include_modules_recurse(${LOCAL_DEPS}) +target_include_directories(${histogram} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(${histogram} PRIVATE ${RAW_PIXEL_INCLUDE}) +ocv_target_link_libraries(${histogram} PRIVATE ${OPENCV_LINKER_LIBS} + ${LOCAL_DEPS}) diff --git a/samples/semihosting/histogram/histogram.cpp b/samples/semihosting/histogram/histogram.cpp new file mode 100644 index 0000000000..daa568d0bb --- /dev/null +++ b/samples/semihosting/histogram/histogram.cpp @@ -0,0 +1,43 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include +#include + +#include +#include +#include +#include "raw_pixels.hpp" + +#define IMG_ROWS 100 +#define IMG_COLS 100 + +static_assert(IMG_ROWS * IMG_COLS <= RAW_PIXELS_SIZE, "Incompatible size"); + +int main(void) +{ + // Number of experiment runs + int no_runs = 2; + + // https://docs.opencv.org/master/d3/d63/classcv_1_1Mat.html + cv::Mat src_new(IMG_ROWS, IMG_COLS, CV_8UC1, (void *)raw_pixels); + + // Set parameters + int imgCount = 1; + const int channels[] = {0}; + cv::Mat mask = cv::Mat(); + cv::Mat hist; + int dims = 1; + const int hist_sizes[] = {256}; + float Range[] = {0,256}; + const float *ranges[] = {Range}; + + // Run calc Hist + for(int i=0; i < no_runs; i++){ + std::cout << "Running iteration # "<< i << std::endl; + cv::calcHist(&src_new, imgCount, channels, mask, hist, dims, hist_sizes, ranges); + } + + return 0; +} diff --git a/samples/semihosting/include/CMakeLists.txt b/samples/semihosting/include/CMakeLists.txt new file mode 100644 index 0000000000..3c429b8adf --- /dev/null +++ b/samples/semihosting/include/CMakeLists.txt @@ -0,0 +1,16 @@ +# Populate a C array with random data. +set(RAW_PIXELS_SIZE 102400) +set(RAW_PIXELS_HEADER ${CMAKE_CURRENT_BINARY_DIR}/raw_pixels.hpp) +set(RAW_PIXELS_HEADER_IN ${CMAKE_CURRENT_SOURCE_DIR}/raw_pixels.hpp.in) + +set(RAW_PIXEL_VALUES "") +# Seed the random number generator. +string(RANDOM LENGTH 8 ALPHABET 0123456789abcdf RANDOM_SEED 314 number) +math(EXPR LOOP_RANGE "${RAW_PIXELS_SIZE} - 1") + +foreach(i RANGE ${LOOP_RANGE}) + string(RANDOM LENGTH 8 ALPHABET 0123456789abcdf number) + string(CONCAT RAW_PIXEL_VALUES ${RAW_PIXEL_VALUES} "0x${number}, \\\n") +endforeach() + +configure_file(${RAW_PIXELS_HEADER_IN} ${RAW_PIXELS_HEADER}) diff --git a/samples/semihosting/include/raw_pixels.hpp.in b/samples/semihosting/include/raw_pixels.hpp.in new file mode 100644 index 0000000000..6ee98222cc --- /dev/null +++ b/samples/semihosting/include/raw_pixels.hpp.in @@ -0,0 +1,11 @@ +#ifndef RAW_PIXELS_HPP +#define RAW_PIXELS_HP +#include + +#cmakedefine RAW_PIXEL_VALUES @RAW_PIXEL_VALUES@ +#cmakedefine RAW_PIXELS_SIZE @RAW_PIXELS_SIZE@ + +static std::uint32_t raw_pixels[RAW_PIXELS_SIZE] = { + RAW_PIXEL_VALUES +}; +#endif //RAW_PIXELS_HPP diff --git a/samples/semihosting/norm/CMakeLists.txt b/samples/semihosting/norm/CMakeLists.txt new file mode 100644 index 0000000000..6f23d74627 --- /dev/null +++ b/samples/semihosting/norm/CMakeLists.txt @@ -0,0 +1,25 @@ +# This file is part of OpenCV project. +# It is subject to the license terms in the LICENSE file found in the top-level directory +# of this distribution and at http://opencv.org/license.html + +set(PROJECT_NAME norm) +project(${PROJECT_NAME}) + +ocv_install_example_src(norm *.cpp *.hpp CMakeLists.txt) + +set(LOCAL_DEPS + opencv_core + ${OPENCV_MODULES_PUBLIC} + ${OpenCV_LIB_COMPONENTS}) +ocv_check_dependencies(${LOCAL_DEPS}) + +if(NOT OCV_DEPENDENCIES_FOUND) + return() +endif() + +ocv_define_sample(norm norm.cpp ${SEMIHOSTING_SUFFIX}) +ocv_include_modules_recurse(${LOCAL_DEPS}) +target_include_directories(${norm} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(${norm} PRIVATE ${RAW_PIXEL_INCLUDE}) +ocv_target_link_libraries(${norm} PRIVATE ${OPENCV_LINKER_LIBS} + ${LOCAL_DEPS}) diff --git a/samples/semihosting/norm/norm.cpp b/samples/semihosting/norm/norm.cpp new file mode 100644 index 0000000000..f911754be1 --- /dev/null +++ b/samples/semihosting/norm/norm.cpp @@ -0,0 +1,33 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html + +#include +#include + +#include +#include +#include +#include "raw_pixels.hpp" + +#define IMG_ROWS 100 +#define IMG_COLS 100 + +static_assert(IMG_ROWS * IMG_COLS <= RAW_PIXELS_SIZE, "Incompatible size"); + +int main(void) +{ + // Number of experiment runs + int no_runs = 2; + + // https://docs.opencv.org/master/d3/d63/classcv_1_1Mat.html + cv::Mat src(IMG_ROWS, IMG_COLS, CV_8UC1, (void *)raw_pixels); + + // Run calc Hist + for(int i=0; i < no_runs; i++){ + std::cout << "Running iteration # "<< i << std::endl; + cv::norm(src); + } + + return 0; +}