Merge pull request #28444 from abhishek-gola:added_ORT_wrapper

Added ONNX Runtime as an optional wrapper #28444

This PR adds ONNXRuntime (ORT) as an _optional_ wrapper, which can be enabled by adding **WITH_ONNXRUNTIME** flag in CMake command.

Using ORT wrapper the inference time for _resnet50.onnx model_ has come to _**~7ms**_ from _**~14ms**_.
Also, we are able to run models like `ssd_mobilenet_v1.onnx`.
### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake

CMakeLists.txt

@@ -390,9 +390,19 @@ OCV_OPTION(WITH_ANDROID_MEDIANDK "Use Android Media NDK for Video I/O (Android)"
 OCV_OPTION(WITH_ANDROID_NATIVE_CAMERA "Use Android NDK for Camera I/O (Android)" (ANDROID_NATIVE_API_LEVEL GREATER 23)
   VISIBLE_IF ANDROID
   VERIFY HAVE_ANDROID_NATIVE_CAMERA)
-OCV_OPTION(WITH_ONNX "Include Microsoft ONNX Runtime support" OFF
+OCV_OPTION(WITH_ONNXRUNTIME "Include Microsoft ONNX Runtime support" OFF
   VISIBLE_IF TRUE
   VERIFY HAVE_ONNX)
+OCV_OPTION(DOWNLOAD_ONNXRUNTIME "Download ONNX Runtime prebuilt binaries" OFF
+  VISIBLE_IF WITH_ONNXRUNTIME)
+OCV_OPTION(ONNXRUNTIME_PREFER_STATIC "Prefer static ONNX Runtime library when available" ON
+  VISIBLE_IF WITH_ONNXRUNTIME)
+set(ONNXRUNTIME_VERSION "1.24.2" CACHE STRING "ONNX Runtime version to download (prebuilt binaries)")
+
+# Backward compatibility for previous option name.
+if(DEFINED WITH_ONNX AND WITH_ONNX AND NOT WITH_ONNXRUNTIME)
+  set(WITH_ONNXRUNTIME ON CACHE BOOL "Include Microsoft ONNX Runtime support" FORCE)
+endif()
 OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF
   VISIBLE_IF TRUE
   VERIFY HAVE_TIMVX)
@@ -849,7 +859,7 @@ if(WITH_VTK)
   include(cmake/OpenCVDetectVTK.cmake)
 endif()
 
-if(WITH_ONNX)
+if(WITH_ONNXRUNTIME)
   include(cmake/FindONNX.cmake)
 endif()
 
@@ -1848,10 +1858,10 @@ if(WITH_OPENCL OR HAVE_OPENCL)
   endif()
 endif()
 
-if(WITH_ONNX OR HAVE_ONNX)
+if(DEFINED WITH_ONNXRUNTIME OR DEFINED HAVE_ONNXRUNTIME)
   status("")
-  status("  ONNX:"     HAVE_ONNX THEN "YES" ELSE "NO")
-  if(HAVE_ONNX)
+  status("  ONNX Runtime:" HAVE_ONNXRUNTIME THEN "YES (ver ${ONNX_VERSION})" ELSE "NO")
+  if(HAVE_ONNXRUNTIME)
     status("    Include path:"  ONNX_INCLUDE_DIR THEN "${ONNX_INCLUDE_DIR}" ELSE "NO")
     status("    Link libraries:" ONNX_LIBRARIES THEN "${ONNX_LIBRARIES}" ELSE "NO")
   endif()

cmake/FindONNX.cmake

@@ -1,4 +1,4 @@
-ocv_clear_vars(HAVE_ONNX)
+ocv_clear_vars(HAVE_ONNX ORT_LIB ORT_INCLUDE ONNX_LIBRARIES ONNX_INCLUDE_DIR ONNX_VERSION)
 
 set(ONNXRT_ROOT_DIR "" CACHE PATH "ONNX Runtime install directory")
 
@@ -7,17 +7,74 @@ if(ORT_INSTALL_DIR AND NOT ONNXRT_ROOT_DIR)
   set(ONNXRT_ROOT_DIR ${ORT_INSTALL_DIR})
 endif()
 
+if(NOT ONNXRT_ROOT_DIR AND DEFINED OpenCV_BINARY_DIR)
+  if(EXISTS "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime")
+    file(GLOB __ort_candidates LIST_DIRECTORIES true
+      "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime/onnxruntime-*")
+    list(LENGTH __ort_candidates __ort_candidates_len)
+    if(__ort_candidates_len GREATER 0)
+      list(GET __ort_candidates 0 ONNXRT_ROOT_DIR)
+    endif()
+    unset(__ort_candidates)
+    unset(__ort_candidates_len)
+  endif()
+endif()
+
+set(__ort_hint_roots "")
 if(ONNXRT_ROOT_DIR)
-  find_library(ORT_LIB onnxruntime
-    ${ONNXRT_ROOT_DIR}/lib
-    CMAKE_FIND_ROOT_PATH_BOTH)
-  # The location of headers varies across different versions of ONNX Runtime
-  find_path(ORT_INCLUDE onnxruntime_cxx_api.h
-    ${ONNXRT_ROOT_DIR}/include/onnxruntime/
-    ${ONNXRT_ROOT_DIR}/include/onnxruntime/core/session
+  list(APPEND __ort_hint_roots "${ONNXRT_ROOT_DIR}")
+endif()
+
+# Prefer CMake config packages if present i.e system-installed ORT
+find_package(onnxruntime CONFIG QUIET)
+find_package(ONNXRuntime CONFIG QUIET)
+
+set(__ort_target "")
+foreach(t
+    onnxruntime::onnxruntime
+    ONNXRuntime::onnxruntime
+    ONNXRuntime::onnxruntime_shared
+)
+  if(TARGET ${t})
+    set(__ort_target "${t}")
+    break()
+  endif()
+endforeach()
+
+if(__ort_target)
+  get_target_property(ORT_INCLUDE ${__ort_target} INTERFACE_INCLUDE_DIRECTORIES)
+  if(ORT_INCLUDE AND NOT IS_DIRECTORY "${ORT_INCLUDE}")
+    list(GET ORT_INCLUDE 0 ORT_INCLUDE)
+  endif()
+  get_target_property(ORT_LIB ${__ort_target} IMPORTED_LOCATION)
+  if(NOT ORT_LIB)
+    get_target_property(ORT_LIB ${__ort_target} IMPORTED_LOCATION_RELEASE)
+  endif()
+  if(NOT ORT_LIB)
+    get_target_property(ORT_LIB ${__ort_target} LOCATION)
+  endif()
+endif()
+
+# Locate headers and libraries via find_* in system paths and/or ONNXRT_ROOT_DIR.
+if(NOT ORT_LIB)
+  find_library(ORT_LIB NAMES onnxruntime
+    HINTS ${__ort_hint_roots}
+    PATH_SUFFIXES lib lib64
     CMAKE_FIND_ROOT_PATH_BOTH)
 endif()
 
+if(NOT ORT_INCLUDE)
+  find_path(ORT_INCLUDE NAMES onnxruntime_cxx_api.h
+    HINTS ${__ort_hint_roots}
+    PATH_SUFFIXES
+      include
+      include/onnxruntime
+      include/onnxruntime/core/session
+    CMAKE_FIND_ROOT_PATH_BOTH)
+endif()
+
+unset(__ort_hint_roots)
+
 macro(detect_onxxrt_ep filename dir have_ep_var)
     find_path(ORT_EP_INCLUDE ${filename} ${dir} CMAKE_FIND_ROOT_PATH_BOTH)
     if(ORT_EP_INCLUDE)
@@ -26,8 +83,14 @@ macro(detect_onxxrt_ep filename dir have_ep_var)
 endmacro()
 
 if(ORT_LIB AND ORT_INCLUDE)
+  set(__ort_root_for_ep "${ONNXRT_ROOT_DIR}")
+  if(NOT __ort_root_for_ep)
+    set(__ort_root_for_ep "${ORT_INCLUDE}")
+    string(REGEX REPLACE "(/include/onnxruntime.*)$" "" __ort_root_for_ep "${__ort_root_for_ep}")
+  endif()
+
   # Check DirectML Execution Provider availability
-  get_filename_component(dml_dir ${ONNXRT_ROOT_DIR}/include/onnxruntime/core/providers/dml ABSOLUTE)
+  get_filename_component(dml_dir ${__ort_root_for_ep}/include/onnxruntime/core/providers/dml ABSOLUTE)
   detect_onxxrt_ep(
       dml_provider_factory.h
       ${dml_dir}
@@ -35,7 +98,7 @@ if(ORT_LIB AND ORT_INCLUDE)
   )
 
   # Check CoreML Execution Provider availability
-  get_filename_component(coreml_dir ${ONNXRT_ROOT_DIR}/include/onnxruntime/core/providers/coreml ABSOLUTE)
+  get_filename_component(coreml_dir ${__ort_root_for_ep}/include/onnxruntime/core/providers/coreml ABSOLUTE)
   detect_onxxrt_ep(
       coreml_provider_factory.h
       ${coreml_dir}
@@ -43,19 +106,73 @@ if(ORT_LIB AND ORT_INCLUDE)
   )
 
   set(HAVE_ONNX TRUE)
+  # Try to report a human-readable ONNX Runtime version in diagnostics/build info.
+  if(DEFINED onnxruntime_VERSION AND onnxruntime_VERSION)
+    set(ONNX_VERSION "${onnxruntime_VERSION}")
+  elseif(DEFINED ONNXRuntime_VERSION AND ONNXRuntime_VERSION)
+    set(ONNX_VERSION "${ONNXRuntime_VERSION}")
+  elseif(DEFINED ONNXRUNTIME_VERSION AND ONNXRUNTIME_VERSION)
+    set(ONNX_VERSION "${ONNXRUNTIME_VERSION}")
+  else()
+    set(__ort_version_candidates "")
+    if(ONNXRT_ROOT_DIR)
+      list(APPEND __ort_version_candidates "${ONNXRT_ROOT_DIR}")
+    endif()
+    if(ORT_LIB)
+      list(APPEND __ort_version_candidates "${ORT_LIB}")
+    endif()
+    foreach(__ort_version_candidate ${__ort_version_candidates})
+      string(REGEX MATCH "([0-9]+\\.[0-9]+\\.[0-9]+([.-][0-9A-Za-z]+)?)" __ort_version_match "${__ort_version_candidate}")
+      if(__ort_version_match)
+        set(ONNX_VERSION "${__ort_version_match}")
+        break()
+      endif()
+    endforeach()
+    unset(__ort_version_candidate)
+    unset(__ort_version_candidates)
+    unset(__ort_version_match)
+  endif()
+
   # For CMake output only
   set(ONNX_LIBRARIES "${ORT_LIB}" CACHE STRING "ONNX Runtime libraries")
   set(ONNX_INCLUDE_DIR "${ORT_INCLUDE}" CACHE STRING "ONNX Runtime include path")
+  if(NOT ONNX_VERSION)
+    set(ONNX_VERSION "unknown")
+  endif()
+  set(ONNX_VERSION "${ONNX_VERSION}" CACHE STRING "ONNX Runtime version")
 
   # Link target with associated interface headers
   set(ONNX_LIBRARY "onnxruntime" CACHE STRING "ONNX Link Target")
-  ocv_add_library(${ONNX_LIBRARY} SHARED IMPORTED)
-  set_target_properties(${ONNX_LIBRARY} PROPERTIES
-                        INTERFACE_INCLUDE_DIRECTORIES ${ORT_INCLUDE}
-                        IMPORTED_LOCATION ${ORT_LIB}
-                        IMPORTED_IMPLIB ${ORT_LIB})
+  if(NOT TARGET ${ONNX_LIBRARY})
+    ocv_add_library(${ONNX_LIBRARY} SHARED IMPORTED)
+  endif()
+
+  if(WIN32)
+    # ORT_LIB is typically the import library (.lib). Prefer matching runtime DLL if available.
+    set(__ort_dll "")
+    if(ONNXRT_ROOT_DIR)
+      find_file(__ort_dll NAMES onnxruntime.dll HINTS "${ONNXRT_ROOT_DIR}" PATH_SUFFIXES bin)
+    endif()
+    if(__ort_dll)
+      set_target_properties(${ONNX_LIBRARY} PROPERTIES
+                            INTERFACE_INCLUDE_DIRECTORIES "${ORT_INCLUDE}"
+                            IMPORTED_LOCATION "${__ort_dll}"
+                            IMPORTED_IMPLIB "${ORT_LIB}")
+    else()
+      set_target_properties(${ONNX_LIBRARY} PROPERTIES
+                            INTERFACE_INCLUDE_DIRECTORIES "${ORT_INCLUDE}"
+                            IMPORTED_LOCATION "${ORT_LIB}"
+                            IMPORTED_IMPLIB "${ORT_LIB}")
+    endif()
+    unset(__ort_dll)
+  else()
+    set_target_properties(${ONNX_LIBRARY} PROPERTIES
+                          INTERFACE_INCLUDE_DIRECTORIES "${ORT_INCLUDE}"
+                          IMPORTED_LOCATION "${ORT_LIB}")
+  endif()
+  unset(__ort_root_for_ep)
 endif()
 
 if(NOT HAVE_ONNX)
-  ocv_clear_vars(HAVE_ONNX ORT_LIB ORT_INCLUDE_DIR)
+  ocv_clear_vars(HAVE_ONNX ORT_LIB ORT_INCLUDE ONNX_LIBRARIES ONNX_INCLUDE_DIR ONNX_VERSION)
 endif()

doc/tutorials/introduction/config_reference/config_reference.markdown

@@ -524,6 +524,10 @@ OpenCV have own DNN inference module which have own build-in engine, but can als
 | `INF_ENGINE_RELEASE` | _2020040000_ | **Deprecated since OpenVINO 2022.1** Defines version of Inference Engine library which is tied to OpenVINO toolkit version. Must be a 10-digit string, e.g. _2020040000_ for OpenVINO 2020.4. |
 | `WITH_NGRAPH` | _OFF_ | **Deprecated since OpenVINO 2022.1** Enables Intel NGraph library support. This library is part of Inference Engine backend which allows executing arbitrary networks read from files in multiple formats supported by OpenCV: Caffe, TensorFlow, PyTorch, Darknet, etc.. NGraph library must be installed, it is included into Inference Engine. |
 | `WITH_OPENVINO` | _OFF_ | Enable Intel OpenVINO Toolkit support. Should be used for OpenVINO>=2022.1 instead of `WITH_INF_ENGINE` and `WITH_NGRAPH`. |
+| `WITH_ONNXRUNTIME` | _OFF_ | Enable Microsoft ONNX Runtime backend support for OpenCV DNN. |
+| `DOWNLOAD_ONNXRUNTIME` | _OFF_ | Download official ONNX Runtime prebuilt binaries when enabled (or when ONNX Runtime is not available in system paths). |
+| `ONNXRUNTIME_PREFER_STATIC` | _ON_ | Prefer static `libonnxruntime.a` when both static and shared ONNX Runtime libraries are available. |
+| `ONNXRUNTIME_VERSION` | _1.24.2_ | ONNX Runtime version to download for prebuilt packages. |
 | `OPENCV_DNN_CUDA` | _OFF_ | Enable CUDA backend. [CUDA](https://en.wikipedia.org/wiki/CUDA), CUBLAS and [CUDNN](https://developer.nvidia.com/cudnn) must be installed. |
 | `WITH_VULKAN` | _OFF_ | Enable experimental [Vulkan](https://en.wikipedia.org/wiki/Vulkan_(API)) backend. Does not require additional dependencies, but can use external Vulkan headers (`VULKAN_INCLUDE_DIRS`). |
 

modules/dnn/CMakeLists.txt

@@ -110,6 +110,178 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS
 set(include_dirs "")
 set(libs "")
 
+if(WITH_ONNXRUNTIME)
+  include("${OpenCV_SOURCE_DIR}/cmake/FindONNX.cmake")
+
+  set(_ort_download_requested OFF)
+  set(_ort_download_forced OFF)
+  if(DOWNLOAD_ONNXRUNTIME)
+    set(_ort_download_requested ON)
+    set(_ort_download_forced ON)
+  elseif(NOT HAVE_ONNXRUNTIME)
+    set(_ort_download_requested ON)
+    message(STATUS "DNN: ONNX Runtime was not found in system paths, attempting to download prebuilt package")
+  endif()
+
+  if(_ort_download_requested)
+    set(_ort_filename "")
+    set(_ort_md5 "")
+
+    if(WIN32)
+      if(X86_64 OR CMAKE_SIZEOF_VOID_P EQUAL 8)
+        set(_ort_filename "onnxruntime-win-x64-${ONNXRUNTIME_VERSION}.zip")
+        set(_ort_md5 "c535ec65cf6f850f2a70b4029acdc5d7")
+      elseif(ARM64 OR AARCH64)
+        set(_ort_filename "onnxruntime-win-arm64-${ONNXRUNTIME_VERSION}.zip")
+        set(_ort_md5 "01df64c14cf9285a6b23744f1b2ddf63")
+      endif()
+    elseif(APPLE)
+      if(ARM64 OR AARCH64 OR CMAKE_OSX_ARCHITECTURES MATCHES "arm64")
+        set(_ort_filename "onnxruntime-osx-arm64-${ONNXRUNTIME_VERSION}.tgz")
+        set(_ort_md5 "44302ee5651926ba6b0a3e92a3a303db")
+      endif()
+    elseif(UNIX)
+      if(X86_64)
+        set(_ort_filename "onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz")
+        set(_ort_md5 "8e444d3ed1ea286013e339132ed4f08e")
+      elseif(AARCH64 OR ARM64)
+        set(_ort_filename "onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}.tgz")
+        set(_ort_md5 "857b0fbb35f9d4cc117033242fc2a3da")
+      endif()
+    endif()
+
+    if(NOT _ort_filename)
+      if(_ort_download_forced OR NOT HAVE_ONNXRUNTIME)
+        message(FATAL_ERROR
+          "DNN: there is no official ONNX Runtime prebuilt package for "
+          "CMAKE_SYSTEM_NAME='${CMAKE_SYSTEM_NAME}', CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}'. "
+          "Provide an installed ORT via ONNXRT_ROOT_DIR, or use a supported platform."
+        )
+      endif()
+    else()
+      set(_ort_url "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/${_ort_filename}")
+      string(REGEX REPLACE "\\.(tgz|zip)$" "" _ort_unpack_dirname "${_ort_filename}")
+
+      ocv_download(
+        FILENAME ${_ort_filename}
+        URL ${_ort_url}
+        DESTINATION_DIR "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime"
+        ID "ORT"
+        HASH ${_ort_md5}
+        UNPACK
+      )
+
+      set(ONNXRT_ROOT_DIR "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime/${_ort_unpack_dirname}"
+          CACHE PATH "ONNX Runtime install directory" FORCE)
+
+      if(NOT APPLE AND NOT WIN32)
+        set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,'$ORIGIN/../3rdparty/onnxruntime/${_ort_unpack_dirname}/lib'")
+      endif()
+
+      include("${OpenCV_SOURCE_DIR}/cmake/FindONNX.cmake")
+    endif()
+  endif()  # _ort_download_requested
+
+  if(HAVE_ONNX)
+    set(HAVE_ONNXRUNTIME 1 CACHE INTERNAL "ONNX Runtime availability")
+
+    if(ONNXRUNTIME_PREFER_STATIC)
+      set(_ort_static_lib "")
+      if(ONNXRT_ROOT_DIR)
+        find_file(_ort_static_lib
+          NAMES libonnxruntime.a
+          HINTS "${ONNXRT_ROOT_DIR}"
+          PATH_SUFFIXES lib lib64
+          NO_DEFAULT_PATH
+        )
+      endif()
+      if(NOT _ort_static_lib AND ONNX_LIBRARIES)
+        foreach(_ort_lib ${ONNX_LIBRARIES})
+          get_filename_component(_ort_lib_dir "${_ort_lib}" DIRECTORY)
+          find_file(_ort_static_candidate
+            NAMES libonnxruntime.a
+            HINTS "${_ort_lib_dir}"
+            NO_DEFAULT_PATH
+          )
+          if(_ort_static_candidate AND _ort_static_candidate MATCHES "\\.a$")
+            set(_ort_static_lib "${_ort_static_candidate}")
+            break()
+          endif()
+        endforeach()
+      endif()
+      if(_ort_static_lib AND _ort_static_lib MATCHES "\\.a$")
+        set(ONNX_LIBRARIES "${_ort_static_lib}" CACHE STRING "ONNX Runtime libraries" FORCE)
+        message(STATUS "DNN: ONNX Runtime static library selected: ${_ort_static_lib}")
+      endif()
+      unset(_ort_static_candidate)
+      unset(_ort_static_lib)
+    endif()
+
+    if(ONNX_INCLUDE_DIR)
+      list(APPEND include_dirs "${ONNX_INCLUDE_DIR}")
+    endif()
+    if(ONNX_LIBRARIES)
+      list(APPEND libs "${ONNX_LIBRARIES}")
+    endif()
+
+    add_definitions(-DHAVE_ONNXRUNTIME=1)
+    message(STATUS "DNN: ONNX Runtime enabled")
+
+    # Ensure runtime ORT binaries are deployed into OpenCV install tree
+    set(_ort_runtime_libs "")
+    if(WIN32)
+      if(ONNXRT_ROOT_DIR)
+        file(GLOB _ort_runtime_libs "${ONNXRT_ROOT_DIR}/bin/onnxruntime*.dll")
+      endif()
+      if(_ort_runtime_libs)
+        install(FILES ${_ort_runtime_libs} DESTINATION ${OPENCV_BIN_INSTALL_PATH} COMPONENT libs)
+      endif()
+    elseif(APPLE)
+      if(ONNXRT_ROOT_DIR)
+        file(GLOB _ort_runtime_libs "${ONNXRT_ROOT_DIR}/lib/libonnxruntime*.dylib")
+      endif()
+      if(NOT _ort_runtime_libs AND ONNX_LIBRARIES)
+        foreach(_ort_lib ${ONNX_LIBRARIES})
+          get_filename_component(_ort_lib_dir "${_ort_lib}" DIRECTORY)
+          file(GLOB _ort_runtime_libs "${_ort_lib_dir}/libonnxruntime*.dylib")
+          if(_ort_runtime_libs)
+            break()
+          endif()
+        endforeach()
+      endif()
+      if(_ort_runtime_libs)
+        install(FILES ${_ort_runtime_libs} DESTINATION ${OPENCV_LIB_INSTALL_PATH} COMPONENT libs)
+      endif()
+    else()
+      if(ONNXRT_ROOT_DIR)
+        file(GLOB _ort_runtime_libs "${ONNXRT_ROOT_DIR}/lib/libonnxruntime.so*")
+      endif()
+      if(NOT _ort_runtime_libs AND ONNX_LIBRARIES)
+        foreach(_ort_lib ${ONNX_LIBRARIES})
+          get_filename_component(_ort_lib_dir "${_ort_lib}" DIRECTORY)
+          file(GLOB _ort_runtime_libs "${_ort_lib_dir}/libonnxruntime.so*")
+          if(_ort_runtime_libs)
+            break()
+          endif()
+        endforeach()
+      endif()
+      if(_ort_runtime_libs)
+        install(FILES ${_ort_runtime_libs} DESTINATION ${OPENCV_LIB_INSTALL_PATH} COMPONENT libs)
+      endif()
+    endif()
+    unset(_ort_runtime_libs)
+  else()
+    message(SEND_ERROR
+      "DNN: ONNX Runtime support was requested (WITH_ONNXRUNTIME=ON), but it was not found. "
+      "Set ONNXRT_ROOT_DIR to an existing installation. "
+      "OpenCV attempts to download supported prebuilt ONNX Runtime packages automatically when needed."
+    )
+  endif()
+
+  unset(_ort_download_requested)
+  unset(_ort_download_forced)
+endif()
+
 if(HAVE_PROTOBUF)
   ocv_target_compile_definitions(${the_module} PRIVATE "HAVE_PROTOBUF=1")
 

modules/dnn/include/opencv2/dnn/dnn.hpp

@@ -1020,7 +1020,8 @@ CV__DNN_INLINE_NS_BEGIN
     {
         ENGINE_CLASSIC=1, //!< Force use the old dnn engine similar to 4.x branch
         ENGINE_NEW=2,     //!< Force use the new dnn engine. The engine does not support non CPU back-ends for now.
-        ENGINE_AUTO=3     //!< Try to use the new engine and then fall back to the classic version.
+        ENGINE_AUTO=3,    //!< Try to use the new engine and then fall back to the classic version.
+        ENGINE_ORT=4      //!< Try to use ONNX Runtime wrapper (ONNX only, requires build with WITH_ONNXRUNTIME=ON).
     };
 
     /** @brief Reads a network model stored in <a href="https://pjreddie.com/darknet/">Darknet</a> model files.

modules/dnn/src/model.cpp

@@ -732,19 +732,39 @@ struct TextRecognitionModel_Impl : public Model::Impl
         return decodeSeq;
     }
 
+    static Mat ensureFloat32Prediction(const Mat& prediction)
+    {
+        CV_Assert(!prediction.empty());
+        if (prediction.type() == CV_32FC1)
+            return prediction;
+
+        const int depth = prediction.depth();
+        if (depth == CV_16F || depth == CV_16BF)
+        {
+            Mat prediction32f;
+            prediction.convertTo(prediction32f, CV_32F);
+            return prediction32f;
+        }
+
+        CV_CheckType(prediction.type(), CV_32FC1, "");
+        return prediction;
+    }
+
     virtual
     std::string ctcGreedyDecode(const Mat& prediction)
     {
+        Mat prediction32f = ensureFloat32Prediction(prediction);
+        const Mat& probs = prediction32f;
+
         std::string decodeSeq;
-        CV_CheckEQ(prediction.dims, 3, "");
-        CV_CheckType(prediction.type(), CV_32FC1, "");
+        CV_CheckEQ(probs.dims, 3, "");
         const int vocLength = (int)(vocabulary.size());
-        CV_CheckLE(prediction.size[1], vocLength, "");
+        CV_CheckLE(probs.size[1], vocLength, "");
         bool ctcFlag = true;
         int lastLoc = 0;
-        for (int i = 0; i < prediction.size[0]; i++)
+        for (int i = 0; i < probs.size[0]; i++)
         {
-            const float* pred = prediction.ptr<float>(i);
+            const float* pred = probs.ptr<float>(i);
             int maxLoc = 0;
             float maxScore = pred[0];
             for (int j = 1; j < vocLength + 1; j++)
@@ -853,6 +873,9 @@ struct TextRecognitionModel_Impl : public Model::Impl
 
     virtual
     std::string ctcPrefixBeamSearchDecode(const Mat& prediction) {
+          Mat prediction32f = ensureFloat32Prediction(prediction);
+          const Mat& probs = prediction32f;
+
           // CTC prefix beam search decode.
           // For more detail, refer to:
           // https://distill.pub/2017/ctc/#inference
@@ -860,18 +883,17 @@ struct TextRecognitionModel_Impl : public Model::Impl
           using Beam = std::vector<std::pair<std::vector<int>, PrefixScore>>;
           using BeamInDict = std::unordered_map<std::vector<int>, PrefixScore, PrefixHash>;
 
-          CV_CheckType(prediction.type(), CV_32FC1, "");
-          CV_CheckEQ(prediction.dims, 3, "");
-          CV_CheckEQ(prediction.size[1], 1, "");
-          CV_CheckEQ(prediction.size[2], (int)vocabulary.size() + 1, "");  // Length add 1 for ctc blank
+          CV_CheckEQ(probs.dims, 3, "");
+          CV_CheckEQ(probs.size[1], 1, "");
+          CV_CheckEQ(probs.size[2], (int)vocabulary.size() + 1, "");  // Length add 1 for ctc blank
 
           std::string decodeSeq;
           Beam beam = {std::make_pair(std::vector<int>(), PrefixScore(0.0, kNegativeInfinity))};
-          for (int i = 0; i < prediction.size[0]; i++)
+          for (int i = 0; i < probs.size[0]; i++)
           {
               // Loop over time
               BeamInDict nextBeam;
-              const float* pred = prediction.ptr<float>(i);
+              const float* pred = probs.ptr<float>(i);
               std::vector<std::pair<float, int>> topkPreds =
                   TopK(pred, vocabulary.size() + 1, vocPruneSize);
               for (const auto& each : topkPreds)

modules/dnn/src/net_impl.hpp

@@ -27,6 +27,14 @@
 
 #include <unordered_map>
 
+#ifdef HAVE_ONNXRUNTIME
+namespace Ort {
+    class Env;
+    class Session;
+    class SessionOptions;
+}
+#endif
+
 namespace cv {
 namespace dnn {
 CV__DNN_INLINE_NS_BEGIN
@@ -36,6 +44,10 @@ using std::string;
 
 typedef std::unordered_map<std::string, int64_t> NamesHash;
 
+#ifdef HAVE_ONNXRUNTIME
+struct OrtNamesCache;
+#endif
+
 // NB: Implementation is divided between of multiple .cpp files
 struct Net::Impl : public detail::NetImplBase
 {
@@ -232,6 +244,13 @@ struct Net::Impl : public detail::NetImplBase
     void initCUDABackend(const std::vector<LayerPin>& blobsToKeep_);
 #endif
 
+    #ifdef HAVE_ONNXRUNTIME
+    std::shared_ptr<Ort::Env> ort_env;
+    std::shared_ptr<Ort::Session> ort_session;
+    std::shared_ptr<Ort::SessionOptions> ort_session_options;
+    std::shared_ptr<OrtNamesCache> ort_names_cache;
+#endif
+
     void allocateLayer(int lid, const LayersShapesMap& layersShapes);
 
     // TODO add getter
@@ -373,6 +392,11 @@ struct Net::Impl : public detail::NetImplBase
     void forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs, OutputArrayOfArrays outputs, bool isMainGraph);
     // run the whole model
     void forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays outputs);
+#ifdef HAVE_ONNXRUNTIME
+    // Run inference through ONNX Runtime session (if configured).
+    // If outIdxs is empty, returns all ORT outputs in ORT-defined order.
+    std::vector<Mat> runOrtSession(std::vector<Mat> inputBlobs, const std::vector<int>& outIdxs);
+#endif
     // run the whole model, convenience wrapper
     Mat forwardWithSingleOutput(const std::string& outname);
     // run the whole model, convenience wrapper
@@ -427,6 +451,9 @@ inline Net::Impl* getNetImpl(const Layer* layer)
 Net readNetFromONNX2(const String&);
 Net readNetFromONNX2(const char*, size_t);
 Net readNetFromONNX2(const std::vector<uchar>&);
+#ifdef HAVE_ONNXRUNTIME
+Net readNetFromONNX2_ORT(const String& onnxFile);
+#endif
 
 CV__DNN_INLINE_NS_END
 }}  // namespace cv::dnn

modules/dnn/src/net_impl2.cpp

@@ -6,10 +6,209 @@
 
 #include "net_impl.hpp"
 
+#include <limits>
+
+#ifdef HAVE_ONNXRUNTIME
+#include <onnxruntime_cxx_api.h>
+#endif
+
 namespace cv {
 namespace dnn {
 CV__DNN_INLINE_NS_BEGIN
 
+#ifdef HAVE_ONNXRUNTIME
+struct OrtNamesCache
+{
+    std::vector<std::string> input_names;
+    std::vector<std::string> output_names;
+    std::unordered_map<std::string, int> input_name_to_index;
+    std::unordered_map<std::string, int> output_name_to_index;
+
+    explicit OrtNamesCache(Ort::Session& session)
+    {
+        Ort::AllocatorWithDefaultOptions allocator;
+
+        const size_t ninputs = session.GetInputCount();
+        input_names.reserve(ninputs);
+        for (size_t i = 0; i < ninputs; ++i)
+        {
+            Ort::AllocatedStringPtr in = session.GetInputNameAllocated(i, allocator);
+            std::string n = in ? std::string(in.get()) : std::string();
+            input_name_to_index[n] = (int)i;
+            input_names.push_back(std::move(n));
+        }
+
+        const size_t noutputs = session.GetOutputCount();
+        output_names.reserve(noutputs);
+        for (size_t i = 0; i < noutputs; ++i)
+        {
+            Ort::AllocatedStringPtr out = session.GetOutputNameAllocated(i, allocator);
+            std::string n = out ? std::string(out.get()) : std::string();
+            output_name_to_index[n] = (int)i;
+            output_names.push_back(std::move(n));
+        }
+    }
+};
+#endif
+
+#ifdef HAVE_ONNXRUNTIME
+static int cvTypeFromONNXElemType(const ONNXTensorElementDataType t)
+{
+    switch (t)
+    {
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:   return CV_32F;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8:   return CV_8U;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8:    return CV_8S;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16:  return CV_16U;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16:   return CV_16S;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32:   return CV_32S;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64:   return CV_64S;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL:    return CV_8U;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE:  return CV_64F;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16: return CV_16F;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16:return CV_16BF;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32:  return CV_32U;
+    case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64:  return CV_64U;
+    default:
+        return -1;
+    }
+}
+
+static Ort::Value createOrtTensorFromMat(Ort::Session& session,
+                                        size_t inputIdx,
+                                        const Ort::MemoryInfo& memory_info,
+                                        Mat& inputBlob,
+                                        std::vector<int64_t>& inputDims,
+                                        ONNXTensorElementDataType& in_elem_type)
+{
+    Ort::TypeInfo in_type_info = session.GetInputTypeInfo(inputIdx);
+    Ort::ConstTensorTypeAndShapeInfo in_tensor_info = in_type_info.GetTensorTypeAndShapeInfo();
+    in_elem_type = in_tensor_info.GetElementType();
+
+    const int cvInType = cvTypeFromONNXElemType(in_elem_type);
+    if (cvInType < 0)
+        CV_Error_(Error::StsNotImplemented, ("DNN/ORT: unsupported ORT input element type: %d", (int)in_elem_type));
+
+    if (inputBlob.type() != cvInType)
+        inputBlob.convertTo(inputBlob, cvInType);
+
+    if (!inputBlob.isContinuous())
+        inputBlob = inputBlob.clone();
+
+    inputDims.clear();
+    inputDims.reserve((size_t)inputBlob.dims);
+    for (int i = 0; i < inputBlob.dims; i++)
+        inputDims.push_back((int64_t)inputBlob.size[i]);
+
+    const size_t nbytes = (size_t)inputBlob.total() * inputBlob.elemSize();
+    OrtValue* input_tensor_raw = nullptr;
+    Ort::ThrowOnError(Ort::GetApi().CreateTensorWithDataAsOrtValue(
+        memory_info,
+        inputBlob.data,
+        nbytes,
+        inputDims.data(),
+        inputDims.size(),
+        in_elem_type,
+        &input_tensor_raw));
+    return Ort::Value(input_tensor_raw);
+}
+
+std::vector<Mat> Net::Impl::runOrtSession(std::vector<Mat> inputBlobs, const std::vector<int>& outIdxs)
+{
+    CV_Assert(this->ort_session);
+    Ort::Session& session = *this->ort_session;
+
+    if (!this->ort_names_cache)
+        this->ort_names_cache = std::make_shared<OrtNamesCache>(session);
+
+    OrtNamesCache& names = *this->ort_names_cache;
+    if (names.input_names.empty())
+        CV_Error(Error::StsError, "DNN/ORT: ORT session has no inputs");
+    if (names.output_names.empty())
+        CV_Error(Error::StsError, "DNN/ORT: ORT session has no outputs");
+
+    const size_t ninputs = names.input_names.size();
+    if (inputBlobs.size() != ninputs)
+        CV_Error_(Error::StsBadArg, ("DNN/ORT: expected %zu inputs, but got %zu", ninputs, inputBlobs.size()));
+
+    std::vector<const char*> in_names;
+    in_names.reserve(ninputs);
+    for (size_t i = 0; i < ninputs; ++i)
+        in_names.push_back(names.input_names[i].c_str());
+
+    std::vector<const char*> out_names;
+    if (outIdxs.empty())
+    {
+        out_names.reserve(names.output_names.size());
+        for (const std::string& n : names.output_names)
+            out_names.push_back(n.c_str());
+    }
+    else
+    {
+        out_names.reserve(outIdxs.size());
+        for (int idx : outIdxs)
+        {
+            CV_CheckGE(idx, 0, "DNN/ORT: output index must be non-negative");
+            CV_CheckLT((size_t)idx, names.output_names.size(), "DNN/ORT: output index is out of range");
+            out_names.push_back(names.output_names[(size_t)idx].c_str());
+        }
+    }
+
+    static const Ort::MemoryInfo memory_info =
+        Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
+
+    std::vector<Ort::Value> input_tensors;
+    input_tensors.reserve(ninputs);
+    for (size_t i = 0; i < ninputs; ++i)
+    {
+        if (inputBlobs[i].empty())
+            CV_Error_(Error::StsError, ("DNN/ORT: input '%s' is empty", names.input_names[i].c_str()));
+
+        std::vector<int64_t> inputDims;
+        ONNXTensorElementDataType in_elem_type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED;
+        input_tensors.push_back(createOrtTensorFromMat(session, i, memory_info, inputBlobs[i], inputDims, in_elem_type));
+    }
+
+    std::vector<Ort::Value> output_tensors = session.Run(
+        Ort::RunOptions{nullptr},
+        in_names.data(), input_tensors.data(), input_tensors.size(),
+        out_names.data(), out_names.size());
+
+    CV_CheckEQ(output_tensors.size(), out_names.size(), "DNN/ORT: ORT returned unexpected number of outputs");
+
+    std::vector<Mat> results;
+    results.reserve(output_tensors.size());
+
+    for (Ort::Value& outv : output_tensors)
+    {
+        Ort::TensorTypeAndShapeInfo shape_info = outv.GetTensorTypeAndShapeInfo();
+        std::vector<int64_t> out_shape = shape_info.GetShape();
+
+        std::vector<int> out_dims;
+        out_dims.reserve(out_shape.size());
+        for (int64_t d : out_shape)
+        {
+            if (d < 0)
+                CV_Error(Error::StsError, "DNN/ORT: dynamic output shapes are not supported at runtime");
+            if (d > (int64_t)std::numeric_limits<int>::max())
+                CV_Error(Error::StsError, "DNN/ORT: output shape dimension is too large");
+            out_dims.push_back((int)d);
+        }
+
+        const ONNXTensorElementDataType out_elem_type = shape_info.GetElementType();
+        const int cvOutType = cvTypeFromONNXElemType(out_elem_type);
+        if (cvOutType < 0)
+            CV_Error_(Error::StsNotImplemented, ("DNN/ORT: unsupported ORT output element type: %d", (int)out_elem_type));
+
+        uint8_t* out_bytes = outv.GetTensorMutableData<uint8_t>();
+        Mat view(out_dims, cvOutType, out_bytes);
+        results.push_back(view.clone());  // detach from ORT-owned memory
+    }
+
+    return results;
+}
+#endif
+
 std::string modelFormatToString(ModelFormat modelFormat)
 {
     return
@@ -288,6 +487,15 @@ Ptr<Graph> Net::Impl::newGraph(const std::string& name_, const std::vector<Arg>&
 
 void Net::Impl::prepareForInference()
 {
+#ifdef HAVE_ONNXRUNTIME
+    if (this->ort_session)
+    {
+        prepared = true;
+        finalizeLayers = false;
+        return;
+    }
+#endif
+
     if (!prepared) {
         constFold();
         //inferTypes();
@@ -364,6 +572,42 @@ void Net::Impl::allocateLayerOutputs(
 
 void Net::Impl::forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays outputs)
 {
+#ifdef HAVE_ONNXRUNTIME
+    if (this->ort_session)
+    {
+        if (!netInputLayer || netInputLayer->blobs.empty())
+            CV_Error(Error::StsError, "DNN/ORT: No input data found");
+
+        std::vector<Mat> ortOuts = runOrtSession(netInputLayer->blobs, std::vector<int>());
+
+        std::vector<Mat>* outMats = nullptr;
+        std::vector<UMat>* outUMats = nullptr;
+        _InputArray::KindFlag outKind = outputs.kind();
+        if (outKind == _InputArray::STD_VECTOR_MAT)
+        {
+            outMats = &outputs.getMatVecRef();
+            *outMats = ortOuts;
+        }
+        else if (outKind == _InputArray::STD_VECTOR_UMAT)
+        {
+            outUMats = &outputs.getUMatVecRef();
+            outUMats->resize(ortOuts.size());
+            for (size_t i = 0; i < ortOuts.size(); ++i)
+                ortOuts[i].copyTo(outUMats->at(i));
+        }
+        else if (outKind == _InputArray::MAT || outKind == _InputArray::UMAT)
+        {
+            CV_CheckEQ((int)ortOuts.size(), 1, "DNN/ORT: single Mat/UMat output requires exactly one ORT output");
+            ortOuts[0].copyTo(outputs);
+        }
+        else
+        {
+            CV_Error(Error::StsBadArg, "DNN/ORT: outputs must be Mat, UMat, a vector of Mat's or a vector of UMat's");
+        }
+        return;
+    }
+
+#endif
     if (!mainGraph) {
         CV_Error(Error::StsNullPtr, "the model was not loaded");
     }
@@ -385,22 +629,115 @@ void Net::Impl::forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays
 
 Mat Net::Impl::forwardWithSingleOutput(const std::string& outname)
 {
-    if (!mainGraph) {
-        CV_Error(Error::StsNullPtr, "the model was not loaded");
+#ifdef HAVE_ONNXRUNTIME
+    if (this->ort_session)
+    {
+        if (!netInputLayer || netInputLayer->blobs.empty())
+            CV_Error(Error::StsError, "DNN/ORT: No input data found");
+
+        if (!this->ort_names_cache)
+            this->ort_names_cache = std::make_shared<OrtNamesCache>(*this->ort_session);
+
+        int outIdx = 0;
+        if (!outname.empty())
+        {
+            OrtNamesCache& names = *this->ort_names_cache;
+            auto it = names.output_name_to_index.find(outname);
+            if (it == names.output_name_to_index.end())
+                CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: output '%s' is not found", outname.c_str()));
+            outIdx = it->second;
+        }
+
+        std::vector<int> outIdxs(1, outIdx);
+        std::vector<Mat> outs = runOrtSession(netInputLayer->blobs, outIdxs);
+        CV_Assert(outs.size() == 1);
+        return outs[0];
     }
-    const std::vector<Arg>& outargs = mainGraph->outputs();
-    CV_Assert(outargs.size() > 0);
-    if (!outname.empty()) {
-        const ArgData& outdata = args.at(outargs[0].idx);
-        CV_Assert(outdata.name == outname);
+#endif
+    {
+        if (!mainGraph) {
+            CV_Error(Error::StsNullPtr, "the model was not loaded");
+        }
+        const std::vector<Arg>& outargs = mainGraph->outputs();
+        CV_Assert(outargs.size() > 0);
+        if (!outname.empty()) {
+            const ArgData& outdata = args.at(outargs[0].idx);
+            CV_Assert(outdata.name == outname);
+        }
     }
-    std::vector<Mat> inps={}, outs;
+
+    std::vector<Mat> inps, outs;
     forwardMainGraph(inps, outs);
+    CV_Assert(!outs.empty());
     return outs[0];
 }
 
 void Net::Impl::forwardWithMultipleOutputs(OutputArrayOfArrays outblobs, const std::vector<std::string>& outnames)
 {
+#ifdef HAVE_ONNXRUNTIME
+    if (this->ort_session)
+    {
+        if (!netInputLayer || netInputLayer->blobs.empty())
+            CV_Error(Error::StsError, "DNN/ORT: No input data found");
+
+        if (!this->ort_names_cache)
+            this->ort_names_cache = std::make_shared<OrtNamesCache>(*this->ort_session);
+
+        OrtNamesCache& names = *this->ort_names_cache;
+        const int totalOutputs = (int)names.output_names.size();
+        if (totalOutputs <= 0)
+            CV_Error(Error::StsError, "DNN/ORT: ORT session has no outputs");
+
+        std::vector<int> outIdxs;
+        if (outnames.empty())
+        {
+            outIdxs.resize((size_t)totalOutputs);
+            for (int i = 0; i < totalOutputs; ++i)
+                outIdxs[(size_t)i] = i;
+        }
+        else
+        {
+            outIdxs.reserve(outnames.size());
+            for (const std::string& n : outnames)
+            {
+                auto it = names.output_name_to_index.find(n);
+                if (it == names.output_name_to_index.end())
+                    CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: output '%s' is not found", n.c_str()));
+                outIdxs.push_back(it->second);
+            }
+        }
+
+        std::vector<Mat> outs = runOrtSession(netInputLayer->blobs, outIdxs);
+
+        std::vector<Mat>* outMats = nullptr;
+        std::vector<UMat>* outUMats = nullptr;
+        _InputArray::KindFlag outKind = outblobs.kind();
+        if (outKind == _InputArray::STD_VECTOR_MAT) {
+            outMats = &outblobs.getMatVecRef();
+            outMats->resize(outs.size());
+        } else if (outKind == _InputArray::STD_VECTOR_UMAT) {
+            outUMats = &outblobs.getUMatVecRef();
+            outUMats->resize(outs.size());
+        } else if (outKind == _InputArray::MAT || outKind == _InputArray::UMAT) {
+            CV_CheckEQ((int)outs.size(), 1, "DNN/ORT: Mat/UMat output requires exactly one output");
+        } else {
+            CV_Error(Error::StsBadArg, "outputs must be Mat, UMat, a vector of Mat's or a vector of UMat's");
+        }
+
+        for (size_t i = 0; i < outs.size(); ++i) {
+            Mat src = outs[i];
+            if (outMats) {
+                src.copyTo(outMats->at(i));
+            } else if (outUMats) {
+                src.copyTo(outUMats->at(i));
+            } else {
+                src.copyTo(outblobs);
+            }
+        }
+        return;
+    }
+#endif
+
     if (!mainGraph) {
         CV_Error(Error::StsNullPtr, "the model was not loaded");
     }
@@ -515,6 +852,50 @@ void Net::Impl::traceArg(std::ostream& strm_, const char* prefix, size_t i, Arg
 
 void Net::Impl::setMainGraphInput(InputArray m, const std::string& inpname)
 {
+#ifdef HAVE_ONNXRUNTIME
+    if (this->ort_session)
+    {
+        if (!this->ort_names_cache)
+            this->ort_names_cache = std::make_shared<OrtNamesCache>(*this->ort_session);
+
+        OrtNamesCache& names = *this->ort_names_cache;
+        const size_t ninputs = names.input_names.size();
+        if (ninputs == 0)
+            CV_Error(Error::StsError, "DNN/ORT: ORT session has no inputs");
+
+        if (!netInputLayer) {
+            netInputLayer = Ptr<DataLayer>(new DataLayer());
+            netInputLayer->name = "ort_data_layer";
+            netInputLayer->type = "Data";
+        }
+
+        Mat inputMat = m.getMat();
+        if (inputMat.empty())
+            CV_Error(Error::StsBadArg, "DNN/ORT: Input blob is empty");
+
+        if (netInputLayer->blobs.size() != ninputs)
+            netInputLayer->blobs.resize(ninputs);
+
+        size_t inputIdx = 0;
+        if (inpname.empty())
+        {
+            if (ninputs != 1)
+                CV_Error(Error::StsBadArg, "DNN/ORT: input name must be specified for models with multiple inputs");
+            inputIdx = 0;
+        }
+        else
+        {
+            auto it = names.input_name_to_index.find(inpname);
+            if (it == names.input_name_to_index.end())
+                CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: input '%s' is not found", inpname.c_str()));
+            inputIdx = (size_t)it->second;
+        }
+
+        inputMat.copyTo(netInputLayer->blobs[inputIdx]);
+        return;
+    }
+#endif
+
     CV_Assert(mainGraph);
     const std::vector<Arg>& gr_inputs = mainGraph->inputs();
     size_t i, ninputs = gr_inputs.size();

modules/dnn/src/onnx/onnx_importer.cpp

@@ -4131,6 +4131,20 @@ Net readNetFromONNX(const String& onnxFile, int engine)
             return detail::readNetDiagnostic<ONNXImporter>(onnxFile.c_str());
         case ENGINE_NEW:
             return readNetFromONNX2(onnxFile);
+        case ENGINE_ORT:
+        {
+#ifdef HAVE_ONNXRUNTIME
+            Net net = readNetFromONNX2_ORT(onnxFile);
+            if (net.empty())
+                CV_Error(Error::StsError, "DNN/ONNX/ORT: failed to load model");
+            if (!net.getImpl() || !net.getImpl()->ort_session)
+                CV_Error(Error::StsError, "DNN/ONNX/ORT: ONNX Runtime session was not initialized");
+            return net;
+#else
+            CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: OpenCV was built without ONNX Runtime (WITH_ONNXRUNTIME=OFF). Falling back to ENGINE_AUTO.");
+#endif
+        }
+        /* fall through */
         case ENGINE_AUTO:
         {
             Net net = readNetFromONNX2(onnxFile);
@@ -4156,6 +4170,13 @@ Net readNetFromONNX(const char* buffer, size_t sizeBuffer, int engine)
             return detail::readNetDiagnostic<ONNXImporter>(buffer, sizeBuffer);
         case ENGINE_NEW:
             return readNetFromONNX2(buffer, sizeBuffer);
+        case ENGINE_ORT:
+#ifdef HAVE_ONNXRUNTIME
+            CV_Error(Error::StsNotImplemented, "DNN/ONNX/ORT: loading from memory buffer is not supported");
+#else
+            CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: OpenCV was built without ONNX Runtime (WITH_ONNXRUNTIME=OFF). Falling back to ENGINE_AUTO.");
+#endif
+            /* fall through */
         case ENGINE_AUTO:
         {
             Net net = readNetFromONNX2(buffer, sizeBuffer);
@@ -4181,6 +4202,13 @@ Net readNetFromONNX(const std::vector<uchar>& buffer, int engine)
             return readNetFromONNX(reinterpret_cast<const char*>(buffer.data()), buffer.size());
         case ENGINE_NEW:
             return readNetFromONNX2(buffer);
+        case ENGINE_ORT:
+#ifdef HAVE_ONNXRUNTIME
+            CV_Error(Error::StsNotImplemented, "DNN/ONNX/ORT: loading from memory buffer is not supported");
+#else
+            CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: OpenCV was built without ONNX Runtime (WITH_ONNXRUNTIME=OFF). Falling back to ENGINE_AUTO.");
+#endif
+            /* fall through */
         case ENGINE_AUTO:
         {
             Net net = readNetFromONNX2(buffer);

modules/dnn/src/onnx/onnx_importer2.cpp

@@ -5,6 +5,10 @@
 #include "../precomp.hpp"
 #include "../net_impl.hpp"
 
+#ifdef HAVE_ONNXRUNTIME
+#include <onnxruntime_cxx_api.h>
+#endif
+
 #include <opencv2/dnn/shape_utils.hpp>
 #include <opencv2/dnn/layer_reg.private.hpp>
 #include <opencv2/core/utils/filesystem.hpp>
@@ -2827,6 +2831,62 @@ Net readNetFromONNX2(const String& onnxFile)
     return net;
 }
 
+#ifdef HAVE_ONNXRUNTIME
+Net readNetFromONNX2_ORT(const String& onnxFile)
+{
+    try
+    {
+        static auto s_ort_env = std::make_shared<Ort::Env>(ORT_LOGGING_LEVEL_WARNING, "OpenCV_DNN_ORT");
+
+        Net net;
+        auto impl = net.getImpl();
+
+        impl->ort_env = s_ort_env;
+        impl->ort_session_options = std::make_shared<Ort::SessionOptions>();
+#ifdef _WIN32
+        std::wstring w_onnxFile(onnxFile.begin(), onnxFile.end());
+        impl->ort_session = std::make_shared<Ort::Session>(*s_ort_env, w_onnxFile.c_str(), *impl->ort_session_options);
+#else
+        impl->ort_session = std::make_shared<Ort::Session>(*s_ort_env, onnxFile.c_str(), *impl->ort_session_options);
+#endif
+        impl->modelFileName = onnxFile;
+        impl->modelFormat = DNN_MODEL_ONNX;
+        Ptr<Graph> g = impl->newGraph("ort_session_active", {}, true);
+
+        Ort::Session& session = *impl->ort_session;
+        Ort::AllocatorWithDefaultOptions allocator;
+        const size_t noutputs = session.GetOutputCount();
+        if (noutputs == 0)
+            CV_Error(Error::StsError, "DNN/ONNX/ORT: ORT session has no outputs");
+
+        std::vector<Arg> outs;
+        outs.reserve(noutputs);
+        for (size_t i = 0; i < noutputs; ++i)
+        {
+            Ort::AllocatedStringPtr out = session.GetOutputNameAllocated(i, allocator);
+            std::string n = out ? std::string(out.get()) : std::string();
+            if (n.empty())
+                n = format("output_%zu", i);
+
+            if (impl->haveArg(n))
+                n = format("%s_%zu", n.c_str(), i);
+
+            outs.push_back(impl->newArg(n, DNN_ARG_OUTPUT));
+        }
+        if (g)
+            g->setOutputs(outs);
+
+        CV_LOG_INFO(NULL, "DNN/ONNX: Successfully initialized ORT Session for " << onnxFile);
+        return net;
+    }
+    catch (const std::exception& e)
+    {
+        CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: ORT initialization failed (" << e.what() << ")");
+        return Net();
+    }
+}
+#endif  // HAVE_ONNXRUNTIME
+
 Net readNetFromONNX2(const char* buffer, size_t size)
 {
     ONNXImporter2 importer;

modules/dnn/test/test_onnx_ort_wrapper.cpp

@@ -0,0 +1,90 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "test_precomp.hpp"
+#include "npy_blob.hpp"
+
+namespace opencv_test { namespace {
+
+#ifdef HAVE_ONNXRUNTIME
+
+static std::string _tf(const std::string& filename, bool required = true)
+{
+    return findDataFile(std::string("dnn/onnx/") + filename, required);
+}
+
+static cv::dnn::Net readNetFromONNX_ORT(const std::string& onnxModelPath)
+{
+    cv::dnn::Net net = cv::dnn::readNetFromONNX(onnxModelPath, cv::dnn::ENGINE_ORT);
+    EXPECT_FALSE(net.empty());
+    return net;
+}
+
+TEST(Test_ONNX_ORT_Wrapper, SingleInputSingleOutput)
+{
+    const std::string basename = "convolution";
+    const std::string onnxmodel = _tf("models/" + basename + ".onnx", true);
+
+    cv::Mat input = blobFromNPY(_tf("data/input_" + basename + ".npy"));
+    cv::Mat ref = blobFromNPY(_tf("data/output_" + basename + ".npy"));
+
+    cv::dnn::Net net = readNetFromONNX_ORT(onnxmodel);
+    net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV);
+    net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
+
+    net.setInput(input);
+    cv::Mat out = net.forward();
+
+    normAssert(ref, out, "ORT 1in/1out convolution", 1e-5, 1e-4);
+}
+
+TEST(Test_ONNX_ORT_Wrapper, MultipleInputSingleOutput)
+{
+    const std::string basename = "min";
+    const std::string onnxmodel = _tf("models/" + basename + ".onnx", true);
+
+    cv::Mat inp0 = blobFromNPY(_tf("data/input_" + basename + "_0.npy"));
+    cv::Mat inp1 = blobFromNPY(_tf("data/input_" + basename + "_1.npy"));
+    cv::Mat ref = blobFromNPY(_tf("data/output_" + basename + ".npy"));
+
+    cv::dnn::Net net = readNetFromONNX_ORT(onnxmodel);
+    net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV);
+    net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
+
+    net.setInput(inp0, "0");
+    net.setInput(inp1, "1");
+    cv::Mat out = net.forward();
+
+    normAssert(ref, out, "ORT 2in/1out min", 1e-5, 1e-4);
+}
+
+TEST(Test_ONNX_ORT_Wrapper, SingleInputMultipleOutput)
+{
+    const std::string basename = "top_k";
+    const std::string onnxmodel = _tf("models/" + basename + ".onnx", true);
+
+    cv::Mat input = cv::dnn::readTensorFromONNX(_tf("data/input_" + basename + ".pb"));
+    cv::Mat ref_val = cv::dnn::readTensorFromONNX(_tf("data/output_" + basename + "_0.pb"));
+    cv::Mat ref_ind = cv::dnn::readTensorFromONNX(_tf("data/output_" + basename + "_1.pb"));
+
+    cv::dnn::Net net = readNetFromONNX_ORT(onnxmodel);
+    net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV);
+    net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
+
+    net.setInput(input);
+    std::vector<cv::Mat> outputs;
+    net.forward(outputs, std::vector<std::string>{"values", "indices"});
+    ASSERT_EQ(outputs.size(), 2u);
+
+    normAssert(ref_val, outputs[0], "ORT top_k values", 1e-5, 1e-4);
+    normAssert(ref_ind, outputs[1], "ORT top_k indices", 0.0, 0.0);
+}
+
+#else  // HAVE_ONNXRUNTIME
+
+TEST(Test_ONNX_ORT_Wrapper, DISABLED_NoONNXRuntime) {}
+
+#endif
+
+}} // namespace

samples/dnn/segmentation.cpp

@@ -257,6 +257,7 @@ int main(int argc, char **argv)
         //! [Set input blob]
         net.setInput(blob);
         //! [Set input blob]
+        int64 t0 = getTickCount();
 
         if (modelName == "u2netp")
         {
@@ -291,9 +292,7 @@ int main(int argc, char **argv)
         }
 
         // Put efficiency information.
-        vector<double> layersTimes;
-        double freq = getTickFrequency() / 1000;
-        double t = net.getPerfProfile(layersTimes) / freq;
+        double t = (getTickCount() - t0) * 1000.0 / getTickFrequency();
         string label = format("Inference time: %.2f ms", t);
         Rect r = getTextSize(Size(), label, Point(), fontFace, fontSize, fontWeight);
         r.height += fontSize; // padding

samples/dnn/segmentation.py

@@ -135,6 +135,7 @@ def main(func_args=None):
         blob = cv.dnn.blobFromImage(frame, args.scale, (inpWidth, inpHeight), args.mean, args.rgb, crop=False)
         net.setInput(blob)
 
+        t0 = cv.getTickCount()
         if args.alias == 'u2netp':
             output = net.forward(net.getUnconnectedOutLayersNames())
             pred = output[0][0, 0, :, :]
@@ -167,9 +168,7 @@ def main(func_args=None):
 
             showLegend(labels, colors, legend)
 
-        # Put efficiency information.
-        t, _ = net.getPerfProfile()
-        label = 'Inference time: %.2f ms' % (t * 1000.0 / cv.getTickFrequency())
+        label = 'Inference time: %.2f ms' % ((cv.getTickCount() - t0) * 1000.0 / cv.getTickFrequency())
         labelSize, _ = cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, fontSize, fontThickness)
         cv.rectangle(frame, (0, 0), (labelSize[0]+10, labelSize[1]), (255,255,255), cv.FILLED)
         cv.putText(frame, label, (10, int(25*fontSize)), cv.FONT_HERSHEY_SIMPLEX, fontSize, (0, 0, 0), fontThickness)