diff --git a/CMakeLists.txt b/CMakeLists.txt index 872874b73c..22d8c78f26 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -395,6 +395,8 @@ OCV_OPTION(WITH_ONNXRUNTIME "Include Microsoft ONNX Runtime support" OFF VERIFY HAVE_ONNX) OCV_OPTION(DOWNLOAD_ONNXRUNTIME "Download ONNX Runtime prebuilt binaries" OFF VISIBLE_IF WITH_ONNXRUNTIME) +OCV_OPTION(DOWNLOAD_ONNXRUNTIME_GPU "Download GPU-enabled ONNX Runtime prebuilt binaries when available" OFF + VISIBLE_IF WITH_ONNXRUNTIME) OCV_OPTION(ONNXRUNTIME_PREFER_STATIC "Prefer static ONNX Runtime library when available" ON VISIBLE_IF WITH_ONNXRUNTIME) set(ONNXRUNTIME_VERSION "1.24.2" CACHE STRING "ONNX Runtime version to download (prebuilt binaries)") diff --git a/doc/tutorials/introduction/config_reference/config_reference.markdown b/doc/tutorials/introduction/config_reference/config_reference.markdown index 4b966627f5..62523f4120 100644 --- a/doc/tutorials/introduction/config_reference/config_reference.markdown +++ b/doc/tutorials/introduction/config_reference/config_reference.markdown @@ -526,6 +526,7 @@ OpenCV have own DNN inference module which have own build-in engine, but can als | `WITH_OPENVINO` | _OFF_ | Enable Intel OpenVINO Toolkit support. Should be used for OpenVINO>=2022.1 instead of `WITH_INF_ENGINE` and `WITH_NGRAPH`. | | `WITH_ONNXRUNTIME` | _OFF_ | Enable Microsoft ONNX Runtime backend support for OpenCV DNN. | | `DOWNLOAD_ONNXRUNTIME` | _OFF_ | Download official ONNX Runtime prebuilt binaries when enabled (or when ONNX Runtime is not available in system paths). | +| `DOWNLOAD_ONNXRUNTIME_GPU` | _OFF_ | Download GPU-enabled ONNX Runtime prebuilt binaries when available (Windows x64 and Linux x64 only). Requires `WITH_ONNXRUNTIME=ON`. | | `ONNXRUNTIME_PREFER_STATIC` | _ON_ | Prefer static `libonnxruntime.a` when both static and shared ONNX Runtime libraries are available. | | `ONNXRUNTIME_VERSION` | _1.24.2_ | ONNX Runtime version to download for prebuilt packages. | | `OPENCV_DNN_CUDA` | _OFF_ | Enable CUDA backend. [CUDA](https://en.wikipedia.org/wiki/CUDA), CUBLAS and [CUDNN](https://developer.nvidia.com/cudnn) must be installed. | diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt index 8b69d77584..2365e77173 100644 --- a/modules/dnn/CMakeLists.txt +++ b/modules/dnn/CMakeLists.txt @@ -110,12 +110,19 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS set(include_dirs "") set(libs "") +# ONNX Runtime +ocv_option(WITH_ONNXRUNTIME "Build with ONNX Runtime support" OFF) +ocv_option(DOWNLOAD_ONNXRUNTIME "Download ONNX Runtime prebuilt binaries" OFF IF WITH_ONNXRUNTIME) +ocv_option(DOWNLOAD_ONNXRUNTIME_GPU "Download GPU-enabled ONNX Runtime prebuilt binaries when available" OFF IF WITH_ONNXRUNTIME) + +set(ONNXRUNTIME_VERSION "1.24.2" CACHE STRING "ONNX Runtime version to download (prebuilt binaries)") + if(WITH_ONNXRUNTIME) include("${OpenCV_SOURCE_DIR}/cmake/FindONNX.cmake") set(_ort_download_requested OFF) set(_ort_download_forced OFF) - if(DOWNLOAD_ONNXRUNTIME) + if(DOWNLOAD_ONNXRUNTIME OR DOWNLOAD_ONNXRUNTIME_GPU) set(_ort_download_requested ON) set(_ort_download_forced ON) elseif(NOT HAVE_ONNXRUNTIME) @@ -125,57 +132,143 @@ if(WITH_ONNXRUNTIME) if(_ort_download_requested) set(_ort_filename "") - set(_ort_md5 "") + set(_ort_package_kind "CPU") + if(DOWNLOAD_ONNXRUNTIME_GPU) + set(_ort_package_kind "GPU") + endif() + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _ort_processor) + set(_ort_is_x64 FALSE) + set(_ort_is_arm64 FALSE) + set(_ort_is_x86 FALSE) + if(X86_64 OR _ort_processor MATCHES "^(x86_64|amd64)$") + set(_ort_is_x64 TRUE) + endif() + if(ARM64 OR AARCH64 OR _ort_processor MATCHES "^(aarch64|arm64)$") + set(_ort_is_arm64 TRUE) + endif() + if(X86 OR _ort_processor MATCHES "^(x86|i[3-6]86)$") + set(_ort_is_x86 TRUE) + endif() if(WIN32) - if(X86_64 OR CMAKE_SIZEOF_VOID_P EQUAL 8) - set(_ort_filename "onnxruntime-win-x64-${ONNXRUNTIME_VERSION}.zip") - set(_ort_md5 "c535ec65cf6f850f2a70b4029acdc5d7") - elseif(ARM64 OR AARCH64) + if(_ort_is_arm64) + if(DOWNLOAD_ONNXRUNTIME_GPU) + message(FATAL_ERROR "DNN: ONNX Runtime GPU package is not available for Windows ARM64. " + "Disable DOWNLOAD_ONNXRUNTIME_GPU or use a supported platform (Windows x64 or Linux x64).") + endif() set(_ort_filename "onnxruntime-win-arm64-${ONNXRUNTIME_VERSION}.zip") - set(_ort_md5 "01df64c14cf9285a6b23744f1b2ddf63") + elseif(_ort_is_x64) + if(DOWNLOAD_ONNXRUNTIME_GPU) + set(_ort_filename "onnxruntime-win-x64-gpu-${ONNXRUNTIME_VERSION}.zip") + else() + set(_ort_filename "onnxruntime-win-x64-${ONNXRUNTIME_VERSION}.zip") + endif() + elseif(_ort_is_x86) + message(FATAL_ERROR + "DNN: No official ONNX Runtime prebuilt package detected for 32-bit Windows and " + "ONNXRUNTIME_VERSION='${ONNXRUNTIME_VERSION}'. " + "Disable DOWNLOAD_ONNXRUNTIME and provide ONNXRT_ROOT_DIR manually." + ) endif() elseif(APPLE) - if(ARM64 OR AARCH64 OR CMAKE_OSX_ARCHITECTURES MATCHES "arm64") + if(DOWNLOAD_ONNXRUNTIME_GPU) + message(FATAL_ERROR "DNN: ONNX Runtime GPU package is not available for macOS. " + "Disable DOWNLOAD_ONNXRUNTIME_GPU or use a supported platform (Windows x64 or Linux x64).") + endif() + if(_ort_is_arm64) set(_ort_filename "onnxruntime-osx-arm64-${ONNXRUNTIME_VERSION}.tgz") - set(_ort_md5 "44302ee5651926ba6b0a3e92a3a303db") + elseif(_ort_is_x64) + set(_ort_filename "onnxruntime-osx-x86_64-${ONNXRUNTIME_VERSION}.tgz") + else() + set(_ort_filename "onnxruntime-osx-universal2-${ONNXRUNTIME_VERSION}.tgz") endif() elseif(UNIX) - if(X86_64) - set(_ort_filename "onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz") - set(_ort_md5 "8e444d3ed1ea286013e339132ed4f08e") - elseif(AARCH64 OR ARM64) + if(_ort_is_x64) + if(DOWNLOAD_ONNXRUNTIME_GPU) + set(_ort_filename "onnxruntime-linux-x64-gpu-${ONNXRUNTIME_VERSION}.tgz") + else() + set(_ort_filename "onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz") + endif() + elseif(_ort_is_arm64) + if(DOWNLOAD_ONNXRUNTIME_GPU) + message(FATAL_ERROR "DNN: ONNX Runtime GPU package is not available for Linux AArch64. " + "Disable DOWNLOAD_ONNXRUNTIME_GPU or use a supported platform (Windows x64 or Linux x64).") + endif() set(_ort_filename "onnxruntime-linux-aarch64-${ONNXRUNTIME_VERSION}.tgz") - set(_ort_md5 "857b0fbb35f9d4cc117033242fc2a3da") endif() endif() if(NOT _ort_filename) - if(_ort_download_forced OR NOT HAVE_ONNXRUNTIME) + if(_ort_download_forced) message(FATAL_ERROR - "DNN: there is no official ONNX Runtime prebuilt package for " + "DNN: DOWNLOAD_ONNXRUNTIME=ON, but there is no official ONNX Runtime prebuilt package for " "CMAKE_SYSTEM_NAME='${CMAKE_SYSTEM_NAME}', CMAKE_SYSTEM_PROCESSOR='${CMAKE_SYSTEM_PROCESSOR}'. " - "Provide an installed ORT via ONNXRT_ROOT_DIR, or use a supported platform." + "Disable DOWNLOAD_ONNXRUNTIME and provide an installed ORT via ONNXRT_ROOT_DIR, or use a supported platform." ) endif() else() set(_ort_url "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/${_ort_filename}") string(REGEX REPLACE "\\.(tgz|zip)$" "" _ort_unpack_dirname "${_ort_filename}") - ocv_download( - FILENAME ${_ort_filename} - URL ${_ort_url} - DESTINATION_DIR "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime" - ID "ORT" - HASH ${_ort_md5} - UNPACK - ) + set(_ort_download_dir "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime") + set(_ort_cache_dir "${OPENCV_DOWNLOAD_PATH}/onnxruntime") + set(_ort_cache_archive "${_ort_cache_dir}/${_ort_filename}") + set(_ort_extracted_dir "${_ort_download_dir}/${_ort_unpack_dirname}") - set(ONNXRT_ROOT_DIR "${OpenCV_BINARY_DIR}/3rdparty/onnxruntime/${_ort_unpack_dirname}" + message(STATUS "DNN: ONNX Runtime download mode: ${_ort_package_kind}") + message(STATUS "DNN: ONNX Runtime package: ${_ort_filename}") + + # Download to persistent cache if not already present + if(NOT EXISTS "${_ort_cache_archive}") + file(MAKE_DIRECTORY "${_ort_cache_dir}") + message(STATUS "DNN: Downloading ONNX Runtime package from ${_ort_url}") + file(DOWNLOAD "${_ort_url}" "${_ort_cache_archive}" + SHOW_PROGRESS + STATUS _ort_download_status + LOG _ort_download_log) + list(GET _ort_download_status 0 _ort_download_status_code) + if(NOT _ort_download_status_code EQUAL 0) + file(REMOVE "${_ort_cache_archive}") + if(_ort_download_forced) + message(FATAL_ERROR "DNN: ONNX Runtime download failed. URL='${_ort_url}'. Log: ${_ort_download_log}") + else() + message(STATUS "DNN: ONNX Runtime download failed, skipping. Log: ${_ort_download_log}") + endif() + endif() + else() + message(STATUS "DNN: ONNX Runtime package found in cache: ${_ort_cache_archive}") + endif() + + # Extract to build dir if not already extracted + if(EXISTS "${_ort_cache_archive}" AND NOT EXISTS "${_ort_extracted_dir}") + file(MAKE_DIRECTORY "${_ort_download_dir}") + message(STATUS "DNN: Extracting ONNX Runtime package to ${_ort_download_dir}") + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.18") + file(ARCHIVE_EXTRACT INPUT "${_ort_cache_archive}" + DESTINATION "${_ort_download_dir}") + else() + if(_ort_filename MATCHES "\\.zip$") + set(_ort_tar_flags xvf) + else() + set(_ort_tar_flags xzf) + endif() + execute_process( + COMMAND "${CMAKE_COMMAND}" -E tar ${_ort_tar_flags} "${_ort_cache_archive}" + WORKING_DIRECTORY "${_ort_download_dir}" + RESULT_VARIABLE _ort_extract_status + ) + if(NOT _ort_extract_status EQUAL 0) + message(FATAL_ERROR "DNN: ONNX Runtime extraction failed for '${_ort_cache_archive}'.") + endif() + endif() + endif() + + set(ONNXRT_ROOT_DIR "${_ort_extracted_dir}" CACHE PATH "ONNX Runtime install directory" FORCE) if(NOT APPLE AND NOT WIN32) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,'$ORIGIN/../3rdparty/onnxruntime/${_ort_unpack_dirname}/lib'") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath,'$ORIGIN/../3rdparty/onnxruntime/${_ort_unpack_dirname}/lib'") endif() include("${OpenCV_SOURCE_DIR}/cmake/FindONNX.cmake") @@ -275,8 +368,7 @@ if(WITH_ONNXRUNTIME) else() message(SEND_ERROR "DNN: ONNX Runtime support was requested (WITH_ONNXRUNTIME=ON), but it was not found. " - "Set ONNXRT_ROOT_DIR to an existing installation. " - "OpenCV attempts to download supported prebuilt ONNX Runtime packages automatically when needed." + "Set ONNXRT_ROOT_DIR to an existing installation or enable DOWNLOAD_ONNXRUNTIME=ON." ) endif() diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 31759c7984..06c21e2266 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -765,6 +765,19 @@ CV__DNN_INLINE_NS_BEGIN */ CV_WRAP void setPreferableTarget(int targetId); + /** @brief Finalizes the network configuration and prepares it for inference. + * + * This method must be called after setting backend/target via + * setPreferableBackend() and setPreferableTarget(), and before the first + * forward() call. It creates the underlying execution session (e.g. ONNX + * Runtime session) on the configured backend/target. If not called + * explicitly, the first forward() will call it automatically. + * + * Calling finalizeNet() early lets you pay the one-time setup cost at a + * predictable point and catch configuration errors before inference. + */ + CV_WRAP void finalizeNet(); + /** * @brief Set the tracing mode * @param[in] tracingMode the tracing mode, see DNN_TRACE_* diff --git a/modules/dnn/src/net.cpp b/modules/dnn/src/net.cpp index c7f7a0fb21..6f167df3bb 100644 --- a/modules/dnn/src/net.cpp +++ b/modules/dnn/src/net.cpp @@ -131,6 +131,19 @@ void Net::setPreferableTarget(int targetId) return impl->setPreferableTarget(targetId); } +void Net::finalizeNet() +{ + CV_TRACE_FUNCTION(); + CV_Assert(impl); +#ifdef HAVE_ONNXRUNTIME + if (impl->mainGraph && impl->modelFormat == DNN_MODEL_ONNX && !impl->modelFileName.empty()) + { + impl->finalizeOrt(); + return; + } +#endif +} + void Net::setInputsNames(const std::vector& inputBlobNames) { CV_TRACE_FUNCTION(); diff --git a/modules/dnn/src/net_impl.cpp b/modules/dnn/src/net_impl.cpp index 8a759a6d27..4f37fb2df0 100644 --- a/modules/dnn/src/net_impl.cpp +++ b/modules/dnn/src/net_impl.cpp @@ -1145,8 +1145,6 @@ void Net::Impl::forward(OutputArrayOfArrays outputBlobs, const std::vector& outBlobNames) { CV_Assert(!empty()); - if (outBlobNames.empty()) - CV_Error(Error::StsBadArg, "in Net::forward(), outBlobNames cannot be empty"); FPDenormalsIgnoreHintScope fp_denormals_ignore_scope; if (mainGraph) { @@ -1154,6 +1152,9 @@ void Net::Impl::forward(OutputArrayOfArrays outputBlobs, return; } + if (outBlobNames.empty()) + CV_Error(Error::StsBadArg, "in Net::forward(), outBlobNames cannot be empty"); + std::vector pins; for (int i = 0; i < outBlobNames.size(); i++) { diff --git a/modules/dnn/src/net_impl.hpp b/modules/dnn/src/net_impl.hpp index d4bab924a9..435ad5b7f9 100644 --- a/modules/dnn/src/net_impl.hpp +++ b/modules/dnn/src/net_impl.hpp @@ -251,10 +251,14 @@ struct Net::Impl : public detail::NetImplBase #endif #ifdef HAVE_ONNXRUNTIME + void finalizeOrt(); + void refreshOrtMainGraphOutputs(); + void applyStagedOrtInputs(); + std::vector> ort_staged_inputs; std::shared_ptr ort_env; std::shared_ptr ort_session; - std::shared_ptr ort_session_options; std::shared_ptr ort_names_cache; + bool ortNeedsReinit = true; // session needs (re)creation on next finalizeNet #endif void allocateLayer(int lid, const LayersShapesMap& layersShapes); diff --git a/modules/dnn/src/net_impl2.cpp b/modules/dnn/src/net_impl2.cpp index 0021611c96..434c387476 100644 --- a/modules/dnn/src/net_impl2.cpp +++ b/modules/dnn/src/net_impl2.cpp @@ -52,6 +52,54 @@ struct OrtNamesCache #endif #ifdef HAVE_ONNXRUNTIME +void Net::Impl::applyStagedOrtInputs() +{ + if (!ort_session || ort_staged_inputs.empty()) + return; + + if (!ort_names_cache) + ort_names_cache = std::make_shared(*ort_session); + + OrtNamesCache& names = *ort_names_cache; + const size_t ninputs = names.input_names.size(); + if (ninputs == 0) + CV_Error(Error::StsError, "DNN/ORT: ORT session has no inputs"); + + if (!netInputLayer) + { + netInputLayer = Ptr(new DataLayer()); + netInputLayer->name = "ort_data_layer"; + netInputLayer->type = "Data"; + } + if (netInputLayer->blobs.size() != ninputs) + netInputLayer->blobs.resize(ninputs); + + for (size_t k = 0; k < ort_staged_inputs.size(); ++k) + { + const std::string& inpname = ort_staged_inputs[k].first; + const Mat& inputMat = ort_staged_inputs[k].second; + if (inputMat.empty()) + CV_Error(Error::StsBadArg, "DNN/ORT: Input blob is empty"); + + size_t inputIdx = 0; + if (inpname.empty()) + { + if (ninputs != 1) + CV_Error(Error::StsBadArg, "DNN/ORT: input name must be specified for models with multiple inputs"); + inputIdx = 0; + } + else + { + auto it = names.input_name_to_index.find(inpname); + if (it == names.input_name_to_index.end()) + CV_Error_(Error::StsObjectNotFound, ("DNN/ORT: input '%s' is not found", inpname.c_str())); + inputIdx = (size_t)it->second; + } + inputMat.copyTo(netInputLayer->blobs[inputIdx]); + } + ort_staged_inputs.clear(); +} + static int cvTypeFromONNXElemType(const ONNXTensorElementDataType t) { switch (t) @@ -571,6 +619,8 @@ void Net::Impl::allocateLayerOutputs( void Net::Impl::forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays outputs) { #ifdef HAVE_ONNXRUNTIME + if (mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty()) + finalizeOrt(); if (this->ort_session) { if (!netInputLayer || netInputLayer->blobs.empty()) @@ -628,6 +678,8 @@ void Net::Impl::forwardMainGraph(InputArrayOfArrays inputs, OutputArrayOfArrays void Net::Impl::forwardWithSingleOutput(const std::string& outname, OutputArrayOfArrays outputBlobs) { #ifdef HAVE_ONNXRUNTIME + if (mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty()) + finalizeOrt(); if (this->ort_session) { if (!netInputLayer || netInputLayer->blobs.empty()) @@ -711,6 +763,8 @@ void Net::Impl::forwardWithSingleOutput(const std::string& outname, OutputArrayO void Net::Impl::forwardWithMultipleOutputs(OutputArrayOfArrays outblobs, const std::vector& outnames) { #ifdef HAVE_ONNXRUNTIME + if (mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty()) + finalizeOrt(); if (this->ort_session) { if (!netInputLayer || netInputLayer->blobs.empty()) @@ -893,6 +947,27 @@ void Net::Impl::traceArg(std::ostream& strm_, const char* prefix, size_t i, Arg void Net::Impl::setMainGraphInput(InputArray m, const std::string& inpname) { #ifdef HAVE_ONNXRUNTIME + if (ortNeedsReinit && mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty()) + { + Mat inputMat = m.getMat(); + if (inputMat.empty()) + CV_Error(Error::StsBadArg, "DNN/ORT: Input blob is empty"); + + bool updated = false; + for (size_t i = 0; i < ort_staged_inputs.size(); ++i) + { + if (ort_staged_inputs[i].first == inpname) + { + inputMat.copyTo(ort_staged_inputs[i].second); + updated = true; + break; + } + } + if (!updated) + ort_staged_inputs.push_back(std::make_pair(inpname, inputMat.clone())); + return; + } + if (this->ort_session) { if (!this->ort_names_cache) diff --git a/modules/dnn/src/net_impl_backend.cpp b/modules/dnn/src/net_impl_backend.cpp index ed4a7e73db..1326681409 100644 --- a/modules/dnn/src/net_impl_backend.cpp +++ b/modules/dnn/src/net_impl_backend.cpp @@ -14,10 +14,93 @@ #include "cuda4dnn/init.hpp" #endif +#ifdef HAVE_ONNXRUNTIME +#include +#endif + namespace cv { namespace dnn { CV__DNN_INLINE_NS_BEGIN +#ifdef HAVE_ONNXRUNTIME +void Net::Impl::refreshOrtMainGraphOutputs() +{ + CV_Assert(mainGraph && ort_session); + + Ort::Session& session = *ort_session; + Ort::AllocatorWithDefaultOptions allocator; + const size_t noutputs = session.GetOutputCount(); + CV_Assert(noutputs > 0); + + std::vector outs; + outs.reserve(noutputs); + for (size_t i = 0; i < noutputs; ++i) + { + Ort::AllocatedStringPtr out = session.GetOutputNameAllocated(i, allocator); + std::string outName = out ? std::string(out.get()) : std::string(); + if (outName.empty()) + outName = format("output_%zu", i); + + if (haveArg(outName)) + { + const int existingIdx = (int)argnames[outName]; + if (args[(size_t)existingIdx].kind == DNN_ARG_OUTPUT) + { + outs.push_back(Arg(existingIdx)); + continue; + } + + std::string uniqueOutName = format("%s_%zu", outName.c_str(), i); + while (haveArg(uniqueOutName)) + uniqueOutName += "_"; + outName = uniqueOutName; + } + + outs.push_back(newArg(outName, DNN_ARG_OUTPUT)); + } + mainGraph->setOutputs(outs); +} + +void Net::Impl::finalizeOrt() +{ + if (ort_session && !ortNeedsReinit) + return; + CV_Assert(!modelFileName.empty()); + + int target = IS_DNN_CUDA_TARGET(preferableTarget) ? preferableTarget : DNN_TARGET_CPU; + ort_session.reset(); + ort_names_cache.reset(); + + static auto s_env = std::make_shared(ORT_LOGGING_LEVEL_WARNING, "OpenCV_DNN_ORT"); + ort_env = s_env; + + Ort::SessionOptions opts; + if (IS_DNN_CUDA_TARGET(target)) + { + OrtCUDAProviderOptions cuda{}; + cuda.device_id = 0; + try { opts.AppendExecutionProvider_CUDA(cuda); } + catch (const std::exception& e) + { + CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: CUDA EP unavailable (" << e.what() << "), using CPU"); + target = DNN_TARGET_CPU; + opts = Ort::SessionOptions(); + } + } + +#ifdef _WIN32 + std::wstring wpath(modelFileName.begin(), modelFileName.end()); + ort_session = std::make_shared(*ort_env, wpath.c_str(), opts); +#else + ort_session = std::make_shared(*ort_env, modelFileName.c_str(), opts); +#endif + preferableTarget = target; + ortNeedsReinit = false; + + refreshOrtMainGraphOutputs(); + applyStagedOrtInputs(); +} +#endif Ptr Net::Impl::wrap(Mat& host) { @@ -170,7 +253,7 @@ void Net::Impl::setPreferableBackend(Net& net, int backendId) backendId = (Backend)getParam_DNN_BACKEND_DEFAULT(); if (backendId == DNN_BACKEND_INFERENCE_ENGINE) - backendId = DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; // = getInferenceEngineBackendTypeParam(); + backendId = DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; if (netWasQuantized && backendId != DNN_BACKEND_OPENCV && backendId != DNN_BACKEND_TIMVX && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) @@ -186,44 +269,66 @@ void Net::Impl::setPreferableBackend(Net& net, int backendId) } #endif - if (preferableBackend != backendId) - { - if (mainGraph) - { - CV_LOG_WARNING(NULL, "Back-ends are not supported by the new graph engine for now"); - preferableBackend = backendId; - return; - } + if (preferableBackend == backendId) + return; - clear(); - if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) - { +#ifdef HAVE_ONNXRUNTIME + if (mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty()) + { + preferableBackend = backendId; + ortNeedsReinit = true; // will be applied on finalizeNet() + return; + } +#endif + + if (mainGraph) + { + CV_LOG_WARNING(NULL, "Back-ends are not supported by the new graph engine for now"); + preferableBackend = backendId; + return; + } + + clear(); + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + { #if defined(HAVE_INF_ENGINE) - switchToOpenVINOBackend(net); + switchToOpenVINOBackend(net); #elif defined(ENABLE_PLUGINS) - auto& networkBackend = dnn_backend::createPluginDNNNetworkBackend("openvino"); - networkBackend.switchBackend(net); + auto& networkBackend = dnn_backend::createPluginDNNNetworkBackend("openvino"); + networkBackend.switchBackend(net); #else - CV_Error(Error::StsNotImplemented, "OpenVINO backend is not available in the current OpenCV build"); + CV_Error(Error::StsNotImplemented, "OpenVINO backend is not available in the current OpenCV build"); #endif - } - else if (backendId == DNN_BACKEND_CANN) - { + } + else if (backendId == DNN_BACKEND_CANN) + { #ifdef HAVE_CANN - switchToCannBackend(net); + switchToCannBackend(net); #else - CV_Error(Error::StsNotImplemented, "CANN backend is not availlable in the current OpenCV build"); + CV_Error(Error::StsNotImplemented, "CANN backend is not availlable in the current OpenCV build"); #endif - } - else - { - preferableBackend = backendId; - } + } + else + { + preferableBackend = backendId; } } void Net::Impl::setPreferableTarget(int targetId) { +#ifdef HAVE_ONNXRUNTIME + if (mainGraph && modelFormat == DNN_MODEL_ONNX && !modelFileName.empty()) + { + int resolved = IS_DNN_CUDA_TARGET(targetId) ? targetId : DNN_TARGET_CPU; + if (preferableTarget != resolved) + { + preferableTarget = resolved; + ortNeedsReinit = true; // will be applied on finalizeNet() + } + return; + } +#endif + if (mainGraph) { CV_LOG_WARNING(NULL, "Targets are not supported by the new graph engine for now"); @@ -286,6 +391,5 @@ void Net::Impl::setPreferableTarget(int targetId) } } - CV__DNN_INLINE_NS_END }} // namespace cv::dnn diff --git a/modules/dnn/src/onnx/onnx_importer.cpp b/modules/dnn/src/onnx/onnx_importer.cpp index a9f020e69c..46a21842dc 100644 --- a/modules/dnn/src/onnx/onnx_importer.cpp +++ b/modules/dnn/src/onnx/onnx_importer.cpp @@ -4144,8 +4144,8 @@ Net readNetFromONNX(const String& onnxFile, int engine) Net net = readNetFromONNX2_ORT(onnxFile); if (net.empty()) CV_Error(Error::StsError, "DNN/ONNX/ORT: failed to load model"); - if (!net.getImpl() || !net.getImpl()->ort_session) - CV_Error(Error::StsError, "DNN/ONNX/ORT: ONNX Runtime session was not initialized"); + if (!net.getImpl() || net.getImpl()->modelFileName.empty()) + CV_Error(Error::StsError, "DNN/ONNX/ORT: ONNX Runtime model metadata was not initialized"); return net; #else CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: OpenCV was built without ONNX Runtime (WITH_ONNXRUNTIME=OFF). Falling back to ENGINE_AUTO."); diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index 340d9f4d72..a74251cd5e 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -5,10 +5,6 @@ #include "../precomp.hpp" #include "../net_impl.hpp" -#ifdef HAVE_ONNXRUNTIME -#include -#endif - #include #include #include @@ -2808,56 +2804,18 @@ Net readNetFromONNX2(const String& onnxFile) #ifdef HAVE_ONNXRUNTIME Net readNetFromONNX2_ORT(const String& onnxFile) { - try - { - static auto s_ort_env = std::make_shared(ORT_LOGGING_LEVEL_WARNING, "OpenCV_DNN_ORT"); + Net net; + auto impl = net.getImpl(); + impl->modelFileName = onnxFile; + impl->modelFormat = DNN_MODEL_ONNX; + impl->ortNeedsReinit = true; - Net net; - auto impl = net.getImpl(); + // Create an empty main graph placeholder so that callers can detect + // that a model has been loaded (e.g. !net.empty()). + impl->newGraph("ort_deferred", {}, true); - impl->ort_env = s_ort_env; - impl->ort_session_options = std::make_shared(); -#ifdef _WIN32 - std::wstring w_onnxFile(onnxFile.begin(), onnxFile.end()); - impl->ort_session = std::make_shared(*s_ort_env, w_onnxFile.c_str(), *impl->ort_session_options); -#else - impl->ort_session = std::make_shared(*s_ort_env, onnxFile.c_str(), *impl->ort_session_options); -#endif - impl->modelFileName = onnxFile; - impl->modelFormat = DNN_MODEL_ONNX; - Ptr g = impl->newGraph("ort_session_active", {}, true); - - Ort::Session& session = *impl->ort_session; - Ort::AllocatorWithDefaultOptions allocator; - const size_t noutputs = session.GetOutputCount(); - if (noutputs == 0) - CV_Error(Error::StsError, "DNN/ONNX/ORT: ORT session has no outputs"); - - std::vector outs; - outs.reserve(noutputs); - for (size_t i = 0; i < noutputs; ++i) - { - Ort::AllocatedStringPtr out = session.GetOutputNameAllocated(i, allocator); - std::string n = out ? std::string(out.get()) : std::string(); - if (n.empty()) - n = format("output_%zu", i); - - if (impl->haveArg(n)) - n = format("%s_%zu", n.c_str(), i); - - outs.push_back(impl->newArg(n, DNN_ARG_OUTPUT)); - } - if (g) - g->setOutputs(outs); - - CV_LOG_INFO(NULL, "DNN/ONNX: Successfully initialized ORT Session for " << onnxFile); - return net; - } - catch (const std::exception& e) - { - CV_LOG_WARNING(NULL, "DNN/ONNX/ORT: ORT initialization failed (" << e.what() << ")"); - return Net(); - } + CV_LOG_INFO(NULL, "DNN/ONNX/ORT: model registered for deferred ORT session: " << onnxFile); + return net; } #endif // HAVE_ONNXRUNTIME diff --git a/samples/dnn/alpha_matting.py b/samples/dnn/alpha_matting.py index 0c31f1055e..38db3608ff 100644 --- a/samples/dnn/alpha_matting.py +++ b/samples/dnn/alpha_matting.py @@ -129,11 +129,13 @@ def apply_modnet(args, model, image): image, args.scale, (args.width, args.height), args.mean, swapRB=args.rgb ) model.setInput(inp) + t0 = cv.getTickCount() out = model.forward() + t = (cv.getTickCount() - t0) / cv.getTickFrequency() alpha_mask = postprocess_output(image, out) alpha_3ch = cv.merge([alpha_mask / 255.0, alpha_mask / 255.0, alpha_mask / 255.0]) composite = (image.astype(np.float32) * alpha_3ch).astype(np.uint8) - return alpha_mask, composite + return alpha_mask, composite, t def main(func_args=None): @@ -156,10 +158,8 @@ def main(func_args=None): args.model = findModel(args.model, args.sha1) net = loadModel(args, engine) - alpha_mask, composite = apply_modnet(args, net, image) - - t, _ = net.getPerfProfile() - label = "Inference time: %.2f ms" % (t * 1000.0 / cv.getTickFrequency()) + alpha_mask, composite, t = apply_modnet(args, net, image) + label = "Inference time: %.2f ms" % (t * 1000.0) draw_label(image, label, (0, 255, 0)) draw_label(alpha_mask, label, (255, 255, 255)) diff --git a/samples/dnn/classification.py b/samples/dnn/classification.py index 4ab1126a31..bdf8a73541 100644 --- a/samples/dnn/classification.py +++ b/samples/dnn/classification.py @@ -135,7 +135,9 @@ def main(func_args=None): # Run a model net.setInput(blob) + t0 = cv.getTickCount() out = net.forward() + t = (cv.getTickCount() - t0) / cv.getTickFrequency() (h, w, _) = frame.shape roi_rows = min(300, h) @@ -143,8 +145,7 @@ def main(func_args=None): frame[:roi_rows,:roi_cols,:] >>= 1 # Put efficiency information. - t, _ = net.getPerfProfile() - label = 'Inference time: %.1f ms' % (t * 1000.0 / cv.getTickFrequency()) + label = 'Inference time: %.1f ms' % (t * 1000.0) cv.putText(frame, label, (15, 30), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0)) # Print predicted classes. diff --git a/samples/dnn/edge_detection.py b/samples/dnn/edge_detection.py index c18ce3a6fe..ee912baf25 100644 --- a/samples/dnn/edge_detection.py +++ b/samples/dnn/edge_detection.py @@ -99,10 +99,11 @@ def loadModel(args, engine): return net def apply_dexined(model, image): + t0 = cv.getTickCount() out = model.forward() + t = (cv.getTickCount() - t0) / cv.getTickFrequency() result,_ = post_processing(out, image.shape[:2]) - t, _ = model.getPerfProfile() - label = 'Inference time: %.2f ms' % (t * 1000.0 / cv.getTickFrequency()) + label = 'Inference time: %.2f ms' % (t * 1000.0) cv.putText(image, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255)) cv.putText(result, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255)) cv.imshow("Output", result) diff --git a/samples/dnn/fast_neural_style.py b/samples/dnn/fast_neural_style.py index 43b8b121d6..0a97e7a9a6 100644 --- a/samples/dnn/fast_neural_style.py +++ b/samples/dnn/fast_neural_style.py @@ -34,14 +34,14 @@ while cv.waitKey(1) < 0: swapRB=True, crop=False) net.setInput(inp) + t0 = cv.getTickCount() out = net.forward() + t = (cv.getTickCount() - t0) / cv.getTickFrequency() out = out.reshape(3, out.shape[2], out.shape[3]) out = out.transpose(1, 2, 0) - t, _ = net.getPerfProfile() - freq = cv.getTickFrequency() / 1000 - print(t / freq, 'ms') + print('%.2f ms' % (t * 1000.0)) if args.median_filter: out = cv.medianBlur(out, args.median_filter) diff --git a/samples/dnn/mask_rcnn.py b/samples/dnn/mask_rcnn.py index bbd3eacc15..0f19661b4b 100644 --- a/samples/dnn/mask_rcnn.py +++ b/samples/dnn/mask_rcnn.py @@ -105,7 +105,9 @@ while cv.waitKey(1) < 0: # NOTE: In OpenCV 5.0, requesting 'detection_out_final' will fail if the .pbtxt # does not register it as an output. See file header for details. + t0 = cv.getTickCount() boxes, masks = net.forward(['detection_out_final', 'detection_masks']) + t = (cv.getTickCount() - t0) / cv.getTickFrequency() numClasses = masks.shape[1] numDetections = boxes.shape[2] @@ -148,8 +150,7 @@ while cv.waitKey(1) < 0: drawBox(*box) # Put efficiency information. - t, _ = net.getPerfProfile() - label = 'Inference time: %.2f ms' % (t * 1000.0 / cv.getTickFrequency()) + label = 'Inference time: %.2f ms' % (t * 1000.0) cv.putText(frame, label, (0, 15), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0)) showLegend(classes) diff --git a/samples/dnn/object_detection.cpp b/samples/dnn/object_detection.cpp index d2be2f1eff..f8ae3c06b1 100644 --- a/samples/dnn/object_detection.cpp +++ b/samples/dnn/object_detection.cpp @@ -367,8 +367,11 @@ int main(int argc, char** argv) } preprocess(frame, net, Size(inpWidth, inpHeight)); + TickMeter tickMeter; vector outs; + tickMeter.start(); net.forward(outs, net.getUnconnectedOutLayersNames()); + tickMeter.stop(); classIds.clear(); confidences.clear(); @@ -378,13 +381,10 @@ int main(int argc, char** argv) drawPred(classIds, confidences, boxes, frame, sans, stdSize, stdWeight, stdImgSize, stdThickness); - vector layersTimes; int imgWidth = max(frame.rows, frame.cols); int size = static_cast((stdSize * imgWidth) / (stdImgSize * 1.5)); int weight = static_cast((stdWeight * imgWidth) / (stdImgSize * 1.5)); - double freq = getTickFrequency() / 1000; - double t = net.getPerfProfile(layersTimes) / freq; - string label = format("FPS: %.2f", 1000/t); + string label = format("FPS: %.2f", 1000.0 / tickMeter.getTimeMilli()); putText(frame, label, Point(0, size), Scalar(0, 255, 0), sans, size, weight); imshow(kWinName, frame); } diff --git a/samples/dnn/openpose.py b/samples/dnn/openpose.py index 191d23edd4..0df4e3d629 100644 --- a/samples/dnn/openpose.py +++ b/samples/dnn/openpose.py @@ -82,7 +82,9 @@ while cv.waitKey(1) < 0: inp = cv.dnn.blobFromImage(frame, inScale, (inWidth, inHeight), (0, 0, 0), swapRB=False, crop=False) net.setInput(inp) + t0 = cv.getTickCount() out = net.forward() + t = (cv.getTickCount() - t0) / cv.getTickFrequency() assert(len(BODY_PARTS) <= out.shape[1]) @@ -115,8 +117,6 @@ while cv.waitKey(1) < 0: cv.ellipse(frame, points[idFrom], (3, 3), 0, 0, 360, (0, 0, 255), cv.FILLED) cv.ellipse(frame, points[idTo], (3, 3), 0, 0, 360, (0, 0, 255), cv.FILLED) - t, _ = net.getPerfProfile() - freq = cv.getTickFrequency() / 1000 - cv.putText(frame, '%.2fms' % (t / freq), (10, 20), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0)) + cv.putText(frame, '%.2f ms' % (t * 1000.0), (10, 20), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0)) cv.imshow('OpenPose using OpenCV', frame) diff --git a/samples/dnn/super_resolution.py b/samples/dnn/super_resolution.py index 3386dfeeb5..022753c687 100644 --- a/samples/dnn/super_resolution.py +++ b/samples/dnn/super_resolution.py @@ -132,12 +132,13 @@ def apply_super_resolution(net, image, args): ) net.setInput(blob) + t0 = cv.getTickCount() output = net.forward() + t = (cv.getTickCount() - t0) / cv.getTickFrequency() result = postprocess_output(output, args, original_shape) - t, _ = net.getPerfProfile() - label = "Inference time: %.2f ms" % (t * 1000.0 / cv.getTickFrequency()) + label = "Inference time: %.2f ms" % (t * 1000.0) cv.putText(result, label, (10, 30), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2) return result