diff --git a/CMakeLists.txt b/CMakeLists.txt index 5523d3848a..872874b73c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -390,9 +390,19 @@ OCV_OPTION(WITH_ANDROID_MEDIANDK "Use Android Media NDK for Video I/O (Android)" OCV_OPTION(WITH_ANDROID_NATIVE_CAMERA "Use Android NDK for Camera I/O (Android)" (ANDROID_NATIVE_API_LEVEL GREATER 23) VISIBLE_IF ANDROID VERIFY HAVE_ANDROID_NATIVE_CAMERA) -OCV_OPTION(WITH_ONNX "Include Microsoft ONNX Runtime support" OFF +OCV_OPTION(WITH_ONNXRUNTIME "Include Microsoft ONNX Runtime support" OFF VISIBLE_IF TRUE VERIFY HAVE_ONNX) +OCV_OPTION(DOWNLOAD_ONNXRUNTIME "Download ONNX Runtime prebuilt binaries" OFF + VISIBLE_IF WITH_ONNXRUNTIME) +OCV_OPTION(ONNXRUNTIME_PREFER_STATIC "Prefer static ONNX Runtime library when available" ON + VISIBLE_IF WITH_ONNXRUNTIME) +set(ONNXRUNTIME_VERSION "1.24.2" CACHE STRING "ONNX Runtime version to download (prebuilt binaries)") + +# Backward compatibility for previous option name. +if(DEFINED WITH_ONNX AND WITH_ONNX AND NOT WITH_ONNXRUNTIME) + set(WITH_ONNXRUNTIME ON CACHE BOOL "Include Microsoft ONNX Runtime support" FORCE) +endif() OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF VISIBLE_IF TRUE VERIFY HAVE_TIMVX) @@ -849,7 +859,7 @@ if(WITH_VTK) include(cmake/OpenCVDetectVTK.cmake) endif() -if(WITH_ONNX) +if(WITH_ONNXRUNTIME) include(cmake/FindONNX.cmake) endif() @@ -1848,10 +1858,10 @@ if(WITH_OPENCL OR HAVE_OPENCL) endif() endif() -if(WITH_ONNX OR HAVE_ONNX) +if(DEFINED WITH_ONNXRUNTIME OR DEFINED HAVE_ONNXRUNTIME) status("") - status(" ONNX:" HAVE_ONNX THEN "YES" ELSE "NO") - if(HAVE_ONNX) + status(" ONNX Runtime:" HAVE_ONNXRUNTIME THEN "YES (ver ${ONNX_VERSION})" ELSE "NO") + if(HAVE_ONNXRUNTIME) status(" Include path:" ONNX_INCLUDE_DIR THEN "${ONNX_INCLUDE_DIR}" ELSE "NO") status(" Link libraries:" ONNX_LIBRARIES THEN "${ONNX_LIBRARIES}" ELSE "NO") endif() diff --git a/cmake/FindONNX.cmake b/cmake/FindONNX.cmake index 0b61185824..e162cdeebb 100644 --- a/cmake/FindONNX.cmake +++ b/cmake/FindONNX.cmake @@ -1,4 +1,4 @@ -ocv_clear_vars(HAVE_ONNX) +ocv_clear_vars(HAVE_ONNX ORT_LIB ORT_INCLUDE ONNX_LIBRARIES ONNX_INCLUDE_DIR ONNX_VERSION) set(ONNXRT_ROOT_DIR "" CACHE PATH "ONNX Runtime install directory") @@ -7,17 +7,74 @@ if(ORT_INSTALL_DIR AND NOT ONNXRT_ROOT_DIR) set(ONNXRT_ROOT_DIR ${ORT_INSTALL_DIR}) endif() +if(NOT ONNXRT_ROOT_DIR AND DEFINED OpenCV_BINARY_DIR) + if(EXISTS "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime") + file(GLOB __ort_candidates LIST_DIRECTORIES true + "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime/onnxruntime-*") + list(LENGTH __ort_candidates __ort_candidates_len) + if(__ort_candidates_len GREATER 0) + list(GET __ort_candidates 0 ONNXRT_ROOT_DIR) + endif() + unset(__ort_candidates) + unset(__ort_candidates_len) + endif() +endif() + +set(__ort_hint_roots "") if(ONNXRT_ROOT_DIR) - find_library(ORT_LIB onnxruntime - ${ONNXRT_ROOT_DIR}/lib - CMAKE_FIND_ROOT_PATH_BOTH) - # The location of headers varies across different versions of ONNX Runtime - find_path(ORT_INCLUDE onnxruntime_cxx_api.h - ${ONNXRT_ROOT_DIR}/include/onnxruntime/ - ${ONNXRT_ROOT_DIR}/include/onnxruntime/core/session + list(APPEND __ort_hint_roots "${ONNXRT_ROOT_DIR}") +endif() + +# Prefer CMake config packages if present i.e system-installed ORT +find_package(onnxruntime CONFIG QUIET) +find_package(ONNXRuntime CONFIG QUIET) + +set(__ort_target "") +foreach(t + onnxruntime::onnxruntime + ONNXRuntime::onnxruntime + ONNXRuntime::onnxruntime_shared +) + if(TARGET ${t}) + set(__ort_target "${t}") + break() + endif() +endforeach() + +if(__ort_target) + get_target_property(ORT_INCLUDE ${__ort_target} INTERFACE_INCLUDE_DIRECTORIES) + if(ORT_INCLUDE AND NOT IS_DIRECTORY "${ORT_INCLUDE}") + list(GET ORT_INCLUDE 0 ORT_INCLUDE) + endif() + get_target_property(ORT_LIB ${__ort_target} IMPORTED_LOCATION) + if(NOT ORT_LIB) + get_target_property(ORT_LIB ${__ort_target} IMPORTED_LOCATION_RELEASE) + endif() + if(NOT ORT_LIB) + get_target_property(ORT_LIB ${__ort_target} LOCATION) + endif() +endif() + +# Locate headers and libraries via find_* in system paths and/or ONNXRT_ROOT_DIR. +if(NOT ORT_LIB) + find_library(ORT_LIB NAMES onnxruntime + HINTS ${__ort_hint_roots} + PATH_SUFFIXES lib lib64 CMAKE_FIND_ROOT_PATH_BOTH) endif() +if(NOT ORT_INCLUDE) + find_path(ORT_INCLUDE NAMES onnxruntime_cxx_api.h + HINTS ${__ort_hint_roots} + PATH_SUFFIXES + include + include/onnxruntime + include/onnxruntime/core/session + CMAKE_FIND_ROOT_PATH_BOTH) +endif() + +unset(__ort_hint_roots) + macro(detect_onxxrt_ep filename dir have_ep_var) find_path(ORT_EP_INCLUDE ${filename} ${dir} CMAKE_FIND_ROOT_PATH_BOTH) if(ORT_EP_INCLUDE) @@ -26,8 +83,14 @@ macro(detect_onxxrt_ep filename dir have_ep_var) endmacro() if(ORT_LIB AND ORT_INCLUDE) + set(__ort_root_for_ep "${ONNXRT_ROOT_DIR}") + if(NOT __ort_root_for_ep) + set(__ort_root_for_ep "${ORT_INCLUDE}") + string(REGEX REPLACE "(/include/onnxruntime.*)$" "" __ort_root_for_ep "${__ort_root_for_ep}") + endif() + # Check DirectML Execution Provider availability - get_filename_component(dml_dir ${ONNXRT_ROOT_DIR}/include/onnxruntime/core/providers/dml ABSOLUTE) + get_filename_component(dml_dir ${__ort_root_for_ep}/include/onnxruntime/core/providers/dml ABSOLUTE) detect_onxxrt_ep( dml_provider_factory.h ${dml_dir} @@ -35,7 +98,7 @@ if(ORT_LIB AND ORT_INCLUDE) ) # Check CoreML Execution Provider availability - get_filename_component(coreml_dir ${ONNXRT_ROOT_DIR}/include/onnxruntime/core/providers/coreml ABSOLUTE) + get_filename_component(coreml_dir ${__ort_root_for_ep}/include/onnxruntime/core/providers/coreml ABSOLUTE) detect_onxxrt_ep( coreml_provider_factory.h ${coreml_dir} @@ -43,19 +106,73 @@ if(ORT_LIB AND ORT_INCLUDE) ) set(HAVE_ONNX TRUE) + # Try to report a human-readable ONNX Runtime version in diagnostics/build info. + if(DEFINED onnxruntime_VERSION AND onnxruntime_VERSION) + set(ONNX_VERSION "${onnxruntime_VERSION}") + elseif(DEFINED ONNXRuntime_VERSION AND ONNXRuntime_VERSION) + set(ONNX_VERSION "${ONNXRuntime_VERSION}") + elseif(DEFINED ONNXRUNTIME_VERSION AND ONNXRUNTIME_VERSION) + set(ONNX_VERSION "${ONNXRUNTIME_VERSION}") + else() + set(__ort_version_candidates "") + if(ONNXRT_ROOT_DIR) + list(APPEND __ort_version_candidates "${ONNXRT_ROOT_DIR}") + endif() + if(ORT_LIB) + list(APPEND __ort_version_candidates "${ORT_LIB}") + endif() + foreach(__ort_version_candidate ${__ort_version_candidates}) + string(REGEX MATCH "([0-9]+\\.[0-9]+\\.[0-9]+([.-][0-9A-Za-z]+)?)" __ort_version_match "${__ort_version_candidate}") + if(__ort_version_match) + set(ONNX_VERSION "${__ort_version_match}") + break() + endif() + endforeach() + unset(__ort_version_candidate) + unset(__ort_version_candidates) + unset(__ort_version_match) + endif() + # For CMake output only set(ONNX_LIBRARIES "${ORT_LIB}" CACHE STRING "ONNX Runtime libraries") set(ONNX_INCLUDE_DIR "${ORT_INCLUDE}" CACHE STRING "ONNX Runtime include path") + if(NOT ONNX_VERSION) + set(ONNX_VERSION "unknown") + endif() + set(ONNX_VERSION "${ONNX_VERSION}" CACHE STRING "ONNX Runtime version") # Link target with associated interface headers set(ONNX_LIBRARY "onnxruntime" CACHE STRING "ONNX Link Target") - ocv_add_library(${ONNX_LIBRARY} SHARED IMPORTED) - set_target_properties(${ONNX_LIBRARY} PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${ORT_INCLUDE} - IMPORTED_LOCATION ${ORT_LIB} - IMPORTED_IMPLIB ${ORT_LIB}) + if(NOT TARGET ${ONNX_LIBRARY}) + ocv_add_library(${ONNX_LIBRARY} SHARED IMPORTED) + endif() + + if(WIN32) + # ORT_LIB is typically the import library (.lib). Prefer matching runtime DLL if available. + set(__ort_dll "") + if(ONNXRT_ROOT_DIR) + find_file(__ort_dll NAMES onnxruntime.dll HINTS "${ONNXRT_ROOT_DIR}" PATH_SUFFIXES bin) + endif() + if(__ort_dll) + set_target_properties(${ONNX_LIBRARY} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${ORT_INCLUDE}" + IMPORTED_LOCATION "${__ort_dll}" + IMPORTED_IMPLIB "${ORT_LIB}") + else() + set_target_properties(${ONNX_LIBRARY} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${ORT_INCLUDE}" + IMPORTED_LOCATION "${ORT_LIB}" + IMPORTED_IMPLIB "${ORT_LIB}") + endif() + unset(__ort_dll) + else() + set_target_properties(${ONNX_LIBRARY} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${ORT_INCLUDE}" + IMPORTED_LOCATION "${ORT_LIB}") + endif() + unset(__ort_root_for_ep) endif() if(NOT HAVE_ONNX) - ocv_clear_vars(HAVE_ONNX ORT_LIB ORT_INCLUDE_DIR) + ocv_clear_vars(HAVE_ONNX ORT_LIB ORT_INCLUDE ONNX_LIBRARIES ONNX_INCLUDE_DIR ONNX_VERSION) endif() diff --git a/doc/tutorials/introduction/config_reference/config_reference.markdown b/doc/tutorials/introduction/config_reference/config_reference.markdown index d2267c206d..4b966627f5 100644 --- a/doc/tutorials/introduction/config_reference/config_reference.markdown +++ b/doc/tutorials/introduction/config_reference/config_reference.markdown @@ -524,6 +524,10 @@ OpenCV have own DNN inference module which have own build-in engine, but can als | `INF_ENGINE_RELEASE` | _2020040000_ | **Deprecated since OpenVINO 2022.1** Defines version of Inference Engine library which is tied to OpenVINO toolkit version. Must be a 10-digit string, e.g. _2020040000_ for OpenVINO 2020.4. | | `WITH_NGRAPH` | _OFF_ | **Deprecated since OpenVINO 2022.1** Enables Intel NGraph library support. This library is part of Inference Engine backend which allows executing arbitrary networks read from files in multiple formats supported by OpenCV: Caffe, TensorFlow, PyTorch, Darknet, etc.. NGraph library must be installed, it is included into Inference Engine. | | `WITH_OPENVINO` | _OFF_ | Enable Intel OpenVINO Toolkit support. Should be used for OpenVINO>=2022.1 instead of `WITH_INF_ENGINE` and `WITH_NGRAPH`. | +| `WITH_ONNXRUNTIME` | _OFF_ | Enable Microsoft ONNX Runtime backend support for OpenCV DNN. | +| `DOWNLOAD_ONNXRUNTIME` | _OFF_ | Download official ONNX Runtime prebuilt binaries when enabled (or when ONNX Runtime is not available in system paths). | +| `ONNXRUNTIME_PREFER_STATIC` | _ON_ | Prefer static `libonnxruntime.a` when both static and shared ONNX Runtime libraries are available. | +| `ONNXRUNTIME_VERSION` | _1.24.2_ | ONNX Runtime version to download for prebuilt packages. | | `OPENCV_DNN_CUDA` | _OFF_ | Enable CUDA backend. [CUDA](https://en.wikipedia.org/wiki/CUDA), CUBLAS and [CUDNN](https://developer.nvidia.com/cudnn) must be installed. | | `WITH_VULKAN` | _OFF_ | Enable experimental [Vulkan](https://en.wikipedia.org/wiki/Vulkan_(API)) backend. Does not require additional dependencies, but can use external Vulkan headers (`VULKAN_INCLUDE_DIRS`). | diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 62c769a4ab..51ed14ea24 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -110,6 +110,178 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS set(include_dirs "") set(libs "") +if(WITH_ONNXRUNTIME) + include("${OpenCV_SOURCE_DIR}/cmake/FindONNX.cmake") + + set(_ort_download_requested OFF) + set(_ort_download_forced OFF) + if(DOWNLOAD_ONNXRUNTIME) + set(_ort_download_requested ON) + set(_ort_download_forced ON) + elseif(NOT HAVE_ONNXRUNTIME) + set(_ort_download_requested ON) + message(STATUS "DNN: ONNX Runtime was not found in system paths, attempting to download prebuilt package") + endif() + + if(_ort_download_requested) + set(_ort_filename "") + set(_ort_md5 "") + + if(WIN32) + if(X86_64 OR CMAKE_SIZEOF_VOID_P EQUAL 8) + set(_ort_filename "onnxruntime-win-x64-${ONNXRUNTIME_VERSION}.zip") + set(_ort_md5 "c535ec65cf6f850f2a70b4029acdc5d7") + elseif(ARM64 OR AARCH64) + set(_ort_filename "onnxruntime-win-arm64-${ONNXRUNTIME_VERSION}.zip") + set(_ort_md5 "01df64c14cf9285a6b23744f1b2ddf63") + endif() + elseif(APPLE) + if(ARM64 OR AARCH64 OR CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + set(_ort_filename "onnxruntime-osx-arm64-${ONNXRUNTIME_VERSION}.tgz") + set(_ort_md5 "44302ee5651926ba6b0a3e92a3a303db") + endif() + elseif(UNIX) + if(X86_64) + set(_ort_filename "onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz") + set(_ort_md5 "8e444d3ed1ea286013e339132ed4f08e") + elseif(AARCH64 OR ARM64) + set(_ort_filename "onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}.tgz") + set(_ort_md5 "857b0fbb35f9d4cc117033242fc2a3da") + endif() + endif() + + if(NOT _ort_filename) + if(_ort_download_forced OR NOT HAVE_ONNXRUNTIME) + message(FATAL_ERROR + "DNN: there is no official ONNX Runtime prebuilt package for " + "CMAKE_SYSTEM_NAME='${CMAKE_SYSTEM_NAME}', CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}'. " + "Provide an installed ORT via ONNXRT_ROOT_DIR, or use a supported platform." + ) + endif() + else() + set(_ort_url "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/${_ort_filename}") + string(REGEX REPLACE "\\.(tgz|zip)$" "" _ort_unpack_dirname "${_ort_filename}") + + ocv_download( + FILENAME ${_ort_filename} + URL ${_ort_url} + DESTINATION_DIR "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime" + ID "ORT" + HASH ${_ort_md5} + UNPACK + ) + + set(ONNXRT_ROOT_DIR "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime/${_ort_unpack_dirname}" + CACHE PATH "ONNX Runtime install directory" FORCE) + + if(NOT APPLE AND NOT WIN32) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,'$ORIGIN/../3rdparty/onnxruntime/${_ort_unpack_dirname}/lib'") + endif() + + include("${OpenCV_SOURCE_DIR}/cmake/FindONNX.cmake") + endif() + endif() # _ort_download_requested + + if(HAVE_ONNX) + set(HAVE_ONNXRUNTIME 1 CACHE INTERNAL "ONNX Runtime availability") + + if(ONNXRUNTIME_PREFER_STATIC) + set(_ort_static_lib "") + if(ONNXRT_ROOT_DIR) + find_file(_ort_static_lib + NAMES libonnxruntime.a + HINTS "${ONNXRT_ROOT_DIR}" + PATH_SUFFIXES lib lib64 + NO_DEFAULT_PATH + ) + endif() + if(NOT _ort_static_lib AND ONNX_LIBRARIES) + foreach(_ort_lib ${ONNX_LIBRARIES}) + get_filename_component(_ort_lib_dir "${_ort_lib}" DIRECTORY) + find_file(_ort_static_candidate + NAMES libonnxruntime.a + HINTS "${_ort_lib_dir}" + NO_DEFAULT_PATH + ) + if(_ort_static_candidate AND _ort_static_candidate MATCHES "\\.a$") + set(_ort_static_lib "${_ort_static_candidate}") + break() + endif() + endforeach() + endif() + if(_ort_static_lib AND _ort_static_lib MATCHES "\\.a$") + set(ONNX_LIBRARIES "${_ort_static_lib}" CACHE STRING "ONNX Runtime libraries" FORCE) + message(STATUS "DNN: ONNX Runtime static library selected: ${_ort_static_lib}") + endif() + unset(_ort_static_candidate) + unset(_ort_static_lib) + endif() + + if(ONNX_INCLUDE_DIR) + list(APPEND include_dirs "${ONNX_INCLUDE_DIR}") + endif() + if(ONNX_LIBRARIES) + list(APPEND libs "${ONNX_LIBRARIES}") + endif() + + add_definitions(-DHAVE_ONNXRUNTIME=1) + message(STATUS "DNN: ONNX Runtime enabled") + + # Ensure runtime ORT binaries are deployed into OpenCV install tree + set(_ort_runtime_libs "") + if(WIN32) + if(ONNXRT_ROOT_DIR) + file(GLOB _ort_runtime_libs "${ONNXRT_ROOT_DIR}/bin/onnxruntime*.dll") + endif() + if(_ort_runtime_libs) + install(FILES ${_ort_runtime_libs} DESTINATION ${OPENCV_BIN_INSTALL_PATH} COMPONENT libs) + endif() + elseif(APPLE) + if(ONNXRT_ROOT_DIR) + file(GLOB _ort_runtime_libs "${ONNXRT_ROOT_DIR}/lib/libonnxruntime*.dylib") + endif() + if(NOT _ort_runtime_libs AND ONNX_LIBRARIES) + foreach(_ort_lib ${ONNX_LIBRARIES}) + get_filename_component(_ort_lib_dir "${_ort_lib}" DIRECTORY) + file(GLOB _ort_runtime_libs "${_ort_lib_dir}/libonnxruntime*.dylib") + if(_ort_runtime_libs) + break() + endif() + endforeach() + endif() + if(_ort_runtime_libs) + install(FILES ${_ort_runtime_libs} DESTINATION ${OPENCV_LIB_INSTALL_PATH} COMPONENT libs) + endif() + else() + if(ONNXRT_ROOT_DIR) + file(GLOB _ort_runtime_libs "${ONNXRT_ROOT_DIR}/lib/libonnxruntime.so*") + endif() + if(NOT _ort_runtime_libs AND ONNX_LIBRARIES) + foreach(_ort_lib ${ONNX_LIBRARIES}) + get_filename_component(_ort_lib_dir "${_ort_lib}" DIRECTORY) + file(GLOB _ort_runtime_libs "${_ort_lib_dir}/libonnxruntime.so*") + if(_ort_runtime_libs) + break() + endif() + endforeach() + endif() + if(_ort_runtime_libs) + install(FILES ${_ort_runtime_libs} DESTINATION ${OPENCV_LIB_INSTALL_PATH} COMPONENT libs) + endif() + endif() + unset(_ort_runtime_libs) + else() + message(SEND_ERROR + "DNN: ONNX Runtime support was requested (WITH_ONNXRUNTIME=ON), but it was not found. " + "Set ONNXRT_ROOT_DIR to an existing installation. " + "OpenCV attempts to download supported prebuilt ONNX Runtime packages automatically when needed." + ) + endif() + + unset(_ort_download_requested) + unset(_ort_download_forced) +endif() + if(HAVE_PROTOBUF) ocv_target_compile_definitions(${the_module} PRIVATE "HAVE_PROTOBUF=1") diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 2c3b5ff9e6..5ea2ce27a5 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -1020,7 +1020,8 @@ CV__DNN_INLINE_NS_BEGIN { ENGINE_CLASSIC=1, //!< Force use the old dnn engine similar to 4.x branch ENGINE_NEW=2, //!< Force use the new dnn engine. The engine does not support non CPU back-ends for now. - ENGINE_AUTO=3 //!< Try to use the new engine and then fall back to the classic version. + ENGINE_AUTO=3, //!< Try to use the new engine and then fall back to the classic version. + ENGINE_ORT=4 //!< Try to use ONNX Runtime wrapper (ONNX only, requires build with WITH_ONNXRUNTIME=ON). }; /** @brief Reads a network model stored in Darknet model files. diff --git a/modules/dnn/src/model.cpp b/modules/dnn/src/model.cpp index e6a259d0d8..404dafd250 100644 --- a/modules/dnn/src/model.cpp +++ b/modules/dnn/src/model.cpp @@ -732,19 +732,39 @@ struct TextRecognitionModel_Impl : public Model::Impl return decodeSeq; } + static Mat ensureFloat32Prediction(const Mat& prediction) + { + CV_Assert(!prediction.empty()); + if (prediction.type() == CV_32FC1) + return prediction; + + const int depth = prediction.depth(); + if (depth == CV_16F || depth == CV_16BF) + { + Mat prediction32f; + prediction.convertTo(prediction32f, CV_32F); + return prediction32f; + } + + CV_CheckType(prediction.type(), CV_32FC1, ""); + return prediction; + } + virtual std::string ctcGreedyDecode(const Mat& prediction) { + Mat prediction32f = ensureFloat32Prediction(prediction); + const Mat& probs = prediction32f; + std::string decodeSeq; - CV_CheckEQ(prediction.dims, 3, ""); - CV_CheckType(prediction.type(), CV_32FC1, ""); + CV_CheckEQ(probs.dims, 3, ""); const int vocLength = (int)(vocabulary.size()); - CV_CheckLE(prediction.size[1], vocLength, ""); + CV_CheckLE(probs.size[1], vocLength, ""); bool ctcFlag = true; int lastLoc = 0; - for (int i = 0; i < prediction.size[0]; i++) + for (int i = 0; i < probs.size[0]; i++) { - const float* pred = prediction.ptr(i); + const float* pred = probs.ptr(i); int maxLoc = 0; float maxScore = pred[0]; for (int j = 1; j < vocLength + 1; j++) @@ -853,6 +873,9 @@ struct TextRecognitionModel_Impl : public Model::Impl virtual std::string ctcPrefixBeamSearchDecode(const Mat& prediction) { + Mat prediction32f = ensureFloat32Prediction(prediction); + const Mat& probs = prediction32f; + // CTC prefix beam search decode. // For more detail, refer to: // https://distill.pub/2017/ctc/#inference @@ -860,18 +883,17 @@ struct TextRecognitionModel_Impl : public Model::Impl using Beam = std::vector, PrefixScore>>; using BeamInDict = std::unordered_map, PrefixScore, PrefixHash>; - CV_CheckType(prediction.type(), CV_32FC1, ""); - CV_CheckEQ(prediction.dims, 3, ""); - CV_CheckEQ(prediction.size[1], 1, ""); - CV_CheckEQ(prediction.size[2], (int)vocabulary.size() + 1, ""); // Length add 1 for ctc blank + CV_CheckEQ(probs.dims, 3, ""); + CV_CheckEQ(probs.size[1], 1, ""); + CV_CheckEQ(probs.size[2], (int)vocabulary.size() + 1, ""); // Length add 1 for ctc blank std::string decodeSeq; Beam beam = {std::make_pair(std::vector(), PrefixScore(0.0, kNegativeInfinity))}; - for (int i = 0; i < prediction.size[0]; i++) + for (int i = 0; i < probs.size[0]; i++) { // Loop over time BeamInDict nextBeam; - const float* pred = prediction.ptr(i); + const float* pred = probs.ptr(i); std::vector> topkPreds = TopK(pred, vocabulary.size() + 1, vocPruneSize); for (const auto& each : topkPreds) diff --git a/modules/dnn/src/net_impl.hpp b/modules/dnn/src/net_impl.hpp index e6573604a0..af610ad271 100644 --- a/modules/dnn/src/net_impl.hpp +++ b/modules/dnn/src/net_impl.hpp @@ -27,6 +27,14 @@ #include +#ifdef HAVE_ONNXRUNTIME +namespace Ort { + class Env; + class Session; + class SessionOptions; +} +#endif + namespace cv { namespace dnn { CV__DNN_INLINE_NS_BEGIN @@ -36,6 +44,10 @@ using std::string; typedef std::unordered_map NamesHash; +#ifdef HAVE_ONNXRUNTIME +struct OrtNamesCache; +#endif + // NB: Implementation is divided between of multiple .cpp files struct Net::Impl : public detail::NetImplBase { @@ -232,6 +244,13 @@ struct Net::Impl : public detail::NetImplBase void initCUDABackend(const std::vector& blobsToKeep_); #endif + #ifdef HAVE_ONNXRUNTIME + std::shared_ptr ort_env; + std::shared_ptr ort_session; + std::shared_ptr ort_session_options; + std::shared_ptr ort_names_cache; +#endif + void allocateLayer(int lid, const LayersShapesMap& layersShapes); // TODO add getter @@ -373,6 +392,11 @@ struct Net::Impl : public detail::NetImplBase void forwardGraph(Ptr& graph, InputArrayOfArrays inputs, OutputArrayOfArrays outputs, bool isMainGraph); // run the whole model void forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays outputs); +#ifdef HAVE_ONNXRUNTIME + // Run inference through ONNX Runtime session (if configured). + // If outIdxs is empty, returns all ORT outputs in ORT-defined order. + std::vector runOrtSession(std::vector inputBlobs, const std::vector& outIdxs); +#endif // run the whole model, convenience wrapper Mat forwardWithSingleOutput(const std::string& outname); // run the whole model, convenience wrapper @@ -427,6 +451,9 @@ inline Net::Impl* getNetImpl(const Layer* layer) Net readNetFromONNX2(const String&); Net readNetFromONNX2(const char*, size_t); Net readNetFromONNX2(const std::vector&); +#ifdef HAVE_ONNXRUNTIME +Net readNetFromONNX2_ORT(const String& onnxFile); +#endif CV__DNN_INLINE_NS_END }} // namespace cv::dnn diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 1dfe373de0..5c1fa37236 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -6,10 +6,209 @@ #include "net_impl.hpp" +#include + +#ifdef HAVE_ONNXRUNTIME +#include +#endif + namespace cv { namespace dnn { CV__DNN_INLINE_NS_BEGIN +#ifdef HAVE_ONNXRUNTIME +struct OrtNamesCache +{ + std::vector input_names; + std::vector output_names; + std::unordered_map input_name_to_index; + std::unordered_map output_name_to_index; + + explicit OrtNamesCache(Ort::Session& session) + { + Ort::AllocatorWithDefaultOptions allocator; + + const size_t ninputs = session.GetInputCount(); + input_names.reserve(ninputs); + for (size_t i = 0; i < ninputs; ++i) + { + Ort::AllocatedStringPtr in = session.GetInputNameAllocated(i, allocator); + std::string n = in ? std::string(in.get()) : std::string(); + input_name_to_index[n] = (int)i; + input_names.push_back(std::move(n)); + } + + const size_t noutputs = session.GetOutputCount(); + output_names.reserve(noutputs); + for (size_t i = 0; i < noutputs; ++i) + { + Ort::AllocatedStringPtr out = session.GetOutputNameAllocated(i, allocator); + std::string n = out ? std::string(out.get()) : std::string(); + output_name_to_index[n] = (int)i; + output_names.push_back(std::move(n)); + } + } +}; +#endif + +#ifdef HAVE_ONNXRUNTIME +static int cvTypeFromONNXElemType(const ONNXTensorElementDataType t) +{ + switch (t) + { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return CV_32F; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return CV_8U; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8: return CV_8S; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16: return CV_16U; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16: return CV_16S; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: return CV_32S; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return CV_64S; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL: return CV_8U; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE: return CV_64F; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return CV_16F; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16:return CV_16BF; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32: return CV_32U; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64: return CV_64U; + default: + return -1; + } +} + +static Ort::Value createOrtTensorFromMat(Ort::Session& session, + size_t inputIdx, + const Ort::MemoryInfo& memory_info, + Mat& inputBlob, + std::vector& inputDims, + ONNXTensorElementDataType& in_elem_type) +{ + Ort::TypeInfo in_type_info = session.GetInputTypeInfo(inputIdx); + Ort::ConstTensorTypeAndShapeInfo in_tensor_info = in_type_info.GetTensorTypeAndShapeInfo(); + in_elem_type = in_tensor_info.GetElementType(); + + const int cvInType = cvTypeFromONNXElemType(in_elem_type); + if (cvInType < 0) + CV_Error_(Error::StsNotImplemented, ("DNN/ORT: unsupported ORT input element type: %d", (int)in_elem_type)); + + if (inputBlob.type() != cvInType) + inputBlob.convertTo(inputBlob, cvInType); + + if (!inputBlob.isContinuous()) + inputBlob = inputBlob.clone(); + + inputDims.clear(); + inputDims.reserve((size_t)inputBlob.dims); + for (int i = 0; i < inputBlob.dims; i++) + inputDims.push_back((int64_t)inputBlob.size[i]); + + const size_t nbytes = (size_t)inputBlob.total() * inputBlob.elemSize(); + OrtValue* input_tensor_raw = nullptr; + Ort::ThrowOnError(Ort::GetApi().CreateTensorWithDataAsOrtValue( + memory_info, + inputBlob.data, + nbytes, + inputDims.data(), + inputDims.size(), + in_elem_type, + &input_tensor_raw)); + return Ort::Value(input_tensor_raw); +} + +std::vector Net::Impl::runOrtSession(std::vector inputBlobs, const std::vector& outIdxs) +{ + CV_Assert(this->ort_session); + Ort::Session& session = *this->ort_session; + + if (!this->ort_names_cache) + this->ort_names_cache = std::make_shared(session); + + OrtNamesCache& names = *this->ort_names_cache; + if (names.input_names.empty()) + CV_Error(Error::StsError, "DNN/ORT: ORT session has no inputs"); + if (names.output_names.empty()) + CV_Error(Error::StsError, "DNN/ORT: ORT session has no outputs"); + + const size_t ninputs = names.input_names.size(); + if (inputBlobs.size() != ninputs) + CV_Error_(Error::StsBadArg, ("DNN/ORT: expected %zu inputs, but got %zu", ninputs, inputBlobs.size())); + + std::vector in_names; + in_names.reserve(ninputs); + for (size_t i = 0; i < ninputs; ++i) + in_names.push_back(names.input_names[i].c_str()); + + std::vector out_names; + if (outIdxs.empty()) + { + out_names.reserve(names.output_names.size()); + for (const std::string& n : names.output_names) + out_names.push_back(n.c_str()); + } + else + { + out_names.reserve(outIdxs.size()); + for (int idx : outIdxs) + { + CV_CheckGE(idx, 0, "DNN/ORT: output index must be non-negative"); + CV_CheckLT((size_t)idx, names.output_names.size(), "DNN/ORT: output index is out of range"); + out_names.push_back(names.output_names[(size_t)idx].c_str()); + } + } + + static const Ort::MemoryInfo memory_info = + Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + + std::vector input_tensors; + input_tensors.reserve(ninputs); + for (size_t i = 0; i < ninputs; ++i) + { + if (inputBlobs[i].empty()) + CV_Error_(Error::StsError, ("DNN/ORT: input '%s' is empty", names.input_names[i].c_str())); + + std::vector inputDims; + ONNXTensorElementDataType in_elem_type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED; + input_tensors.push_back(createOrtTensorFromMat(session, i, memory_info, inputBlobs[i], inputDims, in_elem_type)); + } + + std::vector output_tensors = session.Run( + Ort::RunOptions{nullptr}, + in_names.data(), input_tensors.data(), input_tensors.size(), + out_names.data(), out_names.size()); + + CV_CheckEQ(output_tensors.size(), out_names.size(), "DNN/ORT: ORT returned unexpected number of outputs"); + + std::vector results; + results.reserve(output_tensors.size()); + + for (Ort::Value& outv : output_tensors) + { + Ort::TensorTypeAndShapeInfo shape_info = outv.GetTensorTypeAndShapeInfo(); + std::vector out_shape = shape_info.GetShape(); + + std::vector out_dims; + out_dims.reserve(out_shape.size()); + for (int64_t d : out_shape) + { + if (d < 0) + CV_Error(Error::StsError, "DNN/ORT: dynamic output shapes are not supported at runtime"); + if (d > (int64_t)std::numeric_limits::max()) + CV_Error(Error::StsError, "DNN/ORT: output shape dimension is too large"); + out_dims.push_back((int)d); + } + + const ONNXTensorElementDataType out_elem_type = shape_info.GetElementType(); + const int cvOutType = cvTypeFromONNXElemType(out_elem_type); + if (cvOutType < 0) + CV_Error_(Error::StsNotImplemented, ("DNN/ORT: unsupported ORT output element type: %d", (int)out_elem_type)); + + uint8_t* out_bytes = outv.GetTensorMutableData(); + Mat view(out_dims, cvOutType, out_bytes); + results.push_back(view.clone()); // detach from ORT-owned memory + } + + return results; +} +#endif + std::string modelFormatToString(ModelFormat modelFormat) { return @@ -288,6 +487,15 @@ Ptr Net::Impl::newGraph(const std::string& name_, const std::vector& void Net::Impl::prepareForInference() { +#ifdef HAVE_ONNXRUNTIME + if (this->ort_session) + { + prepared = true; + finalizeLayers = false; + return; + } +#endif + if (!prepared) { constFold(); //inferTypes(); @@ -364,6 +572,42 @@ void Net::Impl::allocateLayerOutputs( void Net::Impl::forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays outputs) { +#ifdef HAVE_ONNXRUNTIME + if (this->ort_session) + { + if (!netInputLayer || netInputLayer->blobs.empty()) + CV_Error(Error::StsError, "DNN/ORT: No input data found"); + + std::vector ortOuts = runOrtSession(netInputLayer->blobs, std::vector()); + + std::vector* outMats = nullptr; + std::vector* outUMats = nullptr; + _InputArray::KindFlag outKind = outputs.kind(); + if (outKind == _InputArray::STD_VECTOR_MAT) + { + outMats = &outputs.getMatVecRef(); + *outMats = ortOuts; + } + else if (outKind == _InputArray::STD_VECTOR_UMAT) + { + outUMats = &outputs.getUMatVecRef(); + outUMats->resize(ortOuts.size()); + for (size_t i = 0; i < ortOuts.size(); ++i) + ortOuts[i].copyTo(outUMats->at(i)); + } + else if (outKind == _InputArray::MAT || outKind == _InputArray::UMAT) + { + CV_CheckEQ((int)ortOuts.size(), 1, "DNN/ORT: single Mat/UMat output requires exactly one ORT output"); + ortOuts[0].copyTo(outputs); + } + else + { + CV_Error(Error::StsBadArg, "DNN/ORT: outputs must be Mat, UMat, a vector of Mat's or a vector of UMat's"); + } + return; + } + +#endif if (!mainGraph) { CV_Error(Error::StsNullPtr, "the model was not loaded"); } @@ -385,22 +629,115 @@ void Net::Impl::forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays Mat Net::Impl::forwardWithSingleOutput(const std::string& outname) { - if (!mainGraph) { - CV_Error(Error::StsNullPtr, "the model was not loaded"); +#ifdef HAVE_ONNXRUNTIME + if (this->ort_session) + { + if (!netInputLayer || netInputLayer->blobs.empty()) + CV_Error(Error::StsError, "DNN/ORT: No input data found"); + + if (!this->ort_names_cache) + this->ort_names_cache = std::make_shared(*this->ort_session); + + int outIdx = 0; + if (!outname.empty()) + { + OrtNamesCache& names = *this->ort_names_cache; + auto it = names.output_name_to_index.find(outname); + if (it == names.output_name_to_index.end()) + CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: output '%s' is not found", outname.c_str())); + outIdx = it->second; + } + + std::vector outIdxs(1, outIdx); + std::vector outs = runOrtSession(netInputLayer->blobs, outIdxs); + CV_Assert(outs.size() == 1); + return outs[0]; } - const std::vector& outargs = mainGraph->outputs(); - CV_Assert(outargs.size() > 0); - if (!outname.empty()) { - const ArgData& outdata = args.at(outargs[0].idx); - CV_Assert(outdata.name == outname); +#endif + { + if (!mainGraph) { + CV_Error(Error::StsNullPtr, "the model was not loaded"); + } + const std::vector& outargs = mainGraph->outputs(); + CV_Assert(outargs.size() > 0); + if (!outname.empty()) { + const ArgData& outdata = args.at(outargs[0].idx); + CV_Assert(outdata.name == outname); + } } - std::vector inps={}, outs; + + std::vector inps, outs; forwardMainGraph(inps, outs); + CV_Assert(!outs.empty()); return outs[0]; } void Net::Impl::forwardWithMultipleOutputs(OutputArrayOfArrays outblobs, const std::vector& outnames) { +#ifdef HAVE_ONNXRUNTIME + if (this->ort_session) + { + if (!netInputLayer || netInputLayer->blobs.empty()) + CV_Error(Error::StsError, "DNN/ORT: No input data found"); + + if (!this->ort_names_cache) + this->ort_names_cache = std::make_shared(*this->ort_session); + + OrtNamesCache& names = *this->ort_names_cache; + const int totalOutputs = (int)names.output_names.size(); + if (totalOutputs <= 0) + CV_Error(Error::StsError, "DNN/ORT: ORT session has no outputs"); + + std::vector outIdxs; + if (outnames.empty()) + { + outIdxs.resize((size_t)totalOutputs); + for (int i = 0; i < totalOutputs; ++i) + outIdxs[(size_t)i] = i; + } + else + { + outIdxs.reserve(outnames.size()); + for (const std::string& n : outnames) + { + auto it = names.output_name_to_index.find(n); + if (it == names.output_name_to_index.end()) + CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: output '%s' is not found", n.c_str())); + outIdxs.push_back(it->second); + } + } + + std::vector outs = runOrtSession(netInputLayer->blobs, outIdxs); + + std::vector* outMats = nullptr; + std::vector* outUMats = nullptr; + _InputArray::KindFlag outKind = outblobs.kind(); + if (outKind == _InputArray::STD_VECTOR_MAT) { + outMats = &outblobs.getMatVecRef(); + outMats->resize(outs.size()); + } else if (outKind == _InputArray::STD_VECTOR_UMAT) { + outUMats = &outblobs.getUMatVecRef(); + outUMats->resize(outs.size()); + } else if (outKind == _InputArray::MAT || outKind == _InputArray::UMAT) { + CV_CheckEQ((int)outs.size(), 1, "DNN/ORT: Mat/UMat output requires exactly one output"); + } else { + CV_Error(Error::StsBadArg, "outputs must be Mat, UMat, a vector of Mat's or a vector of UMat's"); + } + + for (size_t i = 0; i < outs.size(); ++i) { + Mat src = outs[i]; + if (outMats) { + src.copyTo(outMats->at(i)); + } else if (outUMats) { + src.copyTo(outUMats->at(i)); + } else { + src.copyTo(outblobs); + } + } + return; + } +#endif + if (!mainGraph) { CV_Error(Error::StsNullPtr, "the model was not loaded"); } @@ -515,6 +852,50 @@ void Net::Impl::traceArg(std::ostream& strm_, const char* prefix, size_t i, Arg void Net::Impl::setMainGraphInput(InputArray m, const std::string& inpname) { +#ifdef HAVE_ONNXRUNTIME + if (this->ort_session) + { + if (!this->ort_names_cache) + this->ort_names_cache = std::make_shared(*this->ort_session); + + OrtNamesCache& names = *this->ort_names_cache; + const size_t ninputs = names.input_names.size(); + if (ninputs == 0) + CV_Error(Error::StsError, "DNN/ORT: ORT session has no inputs"); + + if (!netInputLayer) { + netInputLayer = Ptr(new DataLayer()); + netInputLayer->name = "ort_data_layer"; + netInputLayer->type = "Data"; + } + + Mat inputMat = m.getMat(); + if (inputMat.empty()) + CV_Error(Error::StsBadArg, "DNN/ORT: Input blob is empty"); + + if (netInputLayer->blobs.size() != ninputs) + netInputLayer->blobs.resize(ninputs); + + size_t inputIdx = 0; + if (inpname.empty()) + { + if (ninputs != 1) + CV_Error(Error::StsBadArg, "DNN/ORT: input name must be specified for models with multiple inputs"); + inputIdx = 0; + } + else + { + auto it = names.input_name_to_index.find(inpname); + if (it == names.input_name_to_index.end()) + CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: input '%s' is not found", inpname.c_str())); + inputIdx = (size_t)it->second; + } + + inputMat.copyTo(netInputLayer->blobs[inputIdx]); + return; + } +#endif + CV_Assert(mainGraph); const std::vector& gr_inputs = mainGraph->inputs(); size_t i, ninputs = gr_inputs.size(); diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index 84d0f6572f..38ee54589b 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -4131,6 +4131,20 @@ Net readNetFromONNX(const String& onnxFile, int engine) return detail::readNetDiagnostic(onnxFile.c_str()); case ENGINE_NEW: return readNetFromONNX2(onnxFile); + case ENGINE_ORT: + { +#ifdef HAVE_ONNXRUNTIME + Net net = readNetFromONNX2_ORT(onnxFile); + if (net.empty()) + CV_Error(Error::StsError, "DNN/ONNX/ORT: failed to load model"); + if (!net.getImpl() || !net.getImpl()->ort_session) + CV_Error(Error::StsError, "DNN/ONNX/ORT: ONNX Runtime session was not initialized"); + return net; +#else + CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: OpenCV was built without ONNX Runtime (WITH_ONNXRUNTIME=OFF). Falling back to ENGINE_AUTO."); +#endif + } + /* fall through */ case ENGINE_AUTO: { Net net = readNetFromONNX2(onnxFile); @@ -4156,6 +4170,13 @@ Net readNetFromONNX(const char* buffer, size_t sizeBuffer, int engine) return detail::readNetDiagnostic(buffer, sizeBuffer); case ENGINE_NEW: return readNetFromONNX2(buffer, sizeBuffer); + case ENGINE_ORT: +#ifdef HAVE_ONNXRUNTIME + CV_Error(Error::StsNotImplemented, "DNN/ONNX/ORT: loading from memory buffer is not supported"); +#else + CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: OpenCV was built without ONNX Runtime (WITH_ONNXRUNTIME=OFF). Falling back to ENGINE_AUTO."); +#endif + /* fall through */ case ENGINE_AUTO: { Net net = readNetFromONNX2(buffer, sizeBuffer); @@ -4181,6 +4202,13 @@ Net readNetFromONNX(const std::vector& buffer, int engine) return readNetFromONNX(reinterpret_cast(buffer.data()), buffer.size()); case ENGINE_NEW: return readNetFromONNX2(buffer); + case ENGINE_ORT: +#ifdef HAVE_ONNXRUNTIME + CV_Error(Error::StsNotImplemented, "DNN/ONNX/ORT: loading from memory buffer is not supported"); +#else + CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: OpenCV was built without ONNX Runtime (WITH_ONNXRUNTIME=OFF). Falling back to ENGINE_AUTO."); +#endif + /* fall through */ case ENGINE_AUTO: { Net net = readNetFromONNX2(buffer); diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index 5d24197717..db6b9e9dac 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -5,6 +5,10 @@ #include "../precomp.hpp" #include "../net_impl.hpp" +#ifdef HAVE_ONNXRUNTIME +#include +#endif + #include #include #include @@ -2827,6 +2831,62 @@ Net readNetFromONNX2(const String& onnxFile) return net; } +#ifdef HAVE_ONNXRUNTIME +Net readNetFromONNX2_ORT(const String& onnxFile) +{ + try + { + static auto s_ort_env = std::make_shared(ORT_LOGGING_LEVEL_WARNING, "OpenCV_DNN_ORT"); + + Net net; + auto impl = net.getImpl(); + + impl->ort_env = s_ort_env; + impl->ort_session_options = std::make_shared(); +#ifdef _WIN32 + std::wstring w_onnxFile(onnxFile.begin(), onnxFile.end()); + impl->ort_session = std::make_shared(*s_ort_env, w_onnxFile.c_str(), *impl->ort_session_options); +#else + impl->ort_session = std::make_shared(*s_ort_env, onnxFile.c_str(), *impl->ort_session_options); +#endif + impl->modelFileName = onnxFile; + impl->modelFormat = DNN_MODEL_ONNX; + Ptr g = impl->newGraph("ort_session_active", {}, true); + + Ort::Session& session = *impl->ort_session; + Ort::AllocatorWithDefaultOptions allocator; + const size_t noutputs = session.GetOutputCount(); + if (noutputs == 0) + CV_Error(Error::StsError, "DNN/ONNX/ORT: ORT session has no outputs"); + + std::vector outs; + outs.reserve(noutputs); + for (size_t i = 0; i < noutputs; ++i) + { + Ort::AllocatedStringPtr out = session.GetOutputNameAllocated(i, allocator); + std::string n = out ? std::string(out.get()) : std::string(); + if (n.empty()) + n = format("output_%zu", i); + + if (impl->haveArg(n)) + n = format("%s_%zu", n.c_str(), i); + + outs.push_back(impl->newArg(n, DNN_ARG_OUTPUT)); + } + if (g) + g->setOutputs(outs); + + CV_LOG_INFO(NULL, "DNN/ONNX: Successfully initialized ORT Session for " << onnxFile); + return net; + } + catch (const std::exception& e) + { + CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: ORT initialization failed (" << e.what() << ")"); + return Net(); + } +} +#endif // HAVE_ONNXRUNTIME + Net readNetFromONNX2(const char* buffer, size_t size) { ONNXImporter2 importer; diff --git a/modules/dnn/test/test_onnx_ort_wrapper.cpp b/modules/dnn/test/test_onnx_ort_wrapper.cpp new file mode 100644 index 0000000000..8dc1ab5b89 --- /dev/null +++ b/modules/dnn/test/test_onnx_ort_wrapper.cpp @@ -0,0 +1,90 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "npy_blob.hpp" + +namespace opencv_test { namespace { + +#ifdef HAVE_ONNXRUNTIME + +static std::string _tf(const std::string& filename, bool required = true) +{ + return findDataFile(std::string("dnn/onnx/") + filename, required); +} + +static cv::dnn::Net readNetFromONNX_ORT(const std::string& onnxModelPath) +{ + cv::dnn::Net net = cv::dnn::readNetFromONNX(onnxModelPath, cv::dnn::ENGINE_ORT); + EXPECT_FALSE(net.empty()); + return net; +} + +TEST(Test_ONNX_ORT_Wrapper, SingleInputSingleOutput) +{ + const std::string basename = "convolution"; + const std::string onnxmodel = _tf("models/" + basename + ".onnx", true); + + cv::Mat input = blobFromNPY(_tf("data/input_" + basename + ".npy")); + cv::Mat ref = blobFromNPY(_tf("data/output_" + basename + ".npy")); + + cv::dnn::Net net = readNetFromONNX_ORT(onnxmodel); + net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); + + net.setInput(input); + cv::Mat out = net.forward(); + + normAssert(ref, out, "ORT 1in/1out convolution", 1e-5, 1e-4); +} + +TEST(Test_ONNX_ORT_Wrapper, MultipleInputSingleOutput) +{ + const std::string basename = "min"; + const std::string onnxmodel = _tf("models/" + basename + ".onnx", true); + + cv::Mat inp0 = blobFromNPY(_tf("data/input_" + basename + "_0.npy")); + cv::Mat inp1 = blobFromNPY(_tf("data/input_" + basename + "_1.npy")); + cv::Mat ref = blobFromNPY(_tf("data/output_" + basename + ".npy")); + + cv::dnn::Net net = readNetFromONNX_ORT(onnxmodel); + net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); + + net.setInput(inp0, "0"); + net.setInput(inp1, "1"); + cv::Mat out = net.forward(); + + normAssert(ref, out, "ORT 2in/1out min", 1e-5, 1e-4); +} + +TEST(Test_ONNX_ORT_Wrapper, SingleInputMultipleOutput) +{ + const std::string basename = "top_k"; + const std::string onnxmodel = _tf("models/" + basename + ".onnx", true); + + cv::Mat input = cv::dnn::readTensorFromONNX(_tf("data/input_" + basename + ".pb")); + cv::Mat ref_val = cv::dnn::readTensorFromONNX(_tf("data/output_" + basename + "_0.pb")); + cv::Mat ref_ind = cv::dnn::readTensorFromONNX(_tf("data/output_" + basename + "_1.pb")); + + cv::dnn::Net net = readNetFromONNX_ORT(onnxmodel); + net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV); + net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU); + + net.setInput(input); + std::vector outputs; + net.forward(outputs, std::vector{"values", "indices"}); + ASSERT_EQ(outputs.size(), 2u); + + normAssert(ref_val, outputs[0], "ORT top_k values", 1e-5, 1e-4); + normAssert(ref_ind, outputs[1], "ORT top_k indices", 0.0, 0.0); +} + +#else // HAVE_ONNXRUNTIME + +TEST(Test_ONNX_ORT_Wrapper, DISABLED_NoONNXRuntime) {} + +#endif + +}} // namespace diff --git a/samples/dnn/segmentation.cpp b/samples/dnn/segmentation.cpp index 853c69e941..9797013270 100644 --- a/samples/dnn/segmentation.cpp +++ b/samples/dnn/segmentation.cpp @@ -257,6 +257,7 @@ int main(int argc, char **argv) //! [Set input blob] net.setInput(blob); //! [Set input blob] + int64 t0 = getTickCount(); if (modelName == "u2netp") { @@ -291,9 +292,7 @@ int main(int argc, char **argv) } // Put efficiency information. - vector layersTimes; - double freq = getTickFrequency() / 1000; - double t = net.getPerfProfile(layersTimes) / freq; + double t = (getTickCount() - t0) * 1000.0 / getTickFrequency(); string label = format("Inference time: %.2f ms", t); Rect r = getTextSize(Size(), label, Point(), fontFace, fontSize, fontWeight); r.height += fontSize; // padding diff --git a/samples/dnn/segmentation.py b/samples/dnn/segmentation.py index c3934e8191..348c75ad0f 100644 --- a/samples/dnn/segmentation.py +++ b/samples/dnn/segmentation.py @@ -135,6 +135,7 @@ def main(func_args=None): blob = cv.dnn.blobFromImage(frame, args.scale, (inpWidth, inpHeight), args.mean, args.rgb, crop=False) net.setInput(blob) + t0 = cv.getTickCount() if args.alias == 'u2netp': output = net.forward(net.getUnconnectedOutLayersNames()) pred = output[0][0, 0, :, :] @@ -167,9 +168,7 @@ def main(func_args=None): showLegend(labels, colors, legend) - # Put efficiency information. - t, _ = net.getPerfProfile() - label = 'Inference time: %.2f ms' % (t * 1000.0 / cv.getTickFrequency()) + label = 'Inference time: %.2f ms' % ((cv.getTickCount() - t0) * 1000.0 / cv.getTickFrequency()) labelSize, _ = cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, fontSize, fontThickness) cv.rectangle(frame, (0, 0), (labelSize[0]+10, labelSize[1]), (255,255,255), cv.FILLED) cv.putText(frame, label, (10, int(25*fontSize)), cv.FONT_HERSHEY_SIMPLEX, fontSize, (0, 0, 0), fontThickness)