From 5261961a6eb9b5572e2e3b5cb0849531e2cfc1e0 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 21 Jul 2023 10:18:06 +0100 Subject: [PATCH] Merge pull request #24024 from TolyaTalamanov:at/add-onnx-openvino-execution-provider G-API: Support OpenVINO Execution Provider for ONNXRT Backend #24024 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../opencv2/gapi/infer/bindings_onnx.hpp | 6 + .../gapi/include/opencv2/gapi/infer/onnx.hpp | 149 +++++++++++++++++- modules/gapi/misc/python/pyopencv_gapi.hpp | 1 + .../gapi/src/backends/onnx/bindings_onnx.cpp | 12 ++ .../gapi/src/backends/onnx/gonnxbackend.cpp | 41 ++++- 5 files changed, 207 insertions(+), 2 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp index af9f3c6f6f..0f764fe2cd 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp @@ -33,6 +33,12 @@ public: GAPI_WRAP PyParams& cfgNormalize(const std::string &layer_name, bool flag); + GAPI_WRAP + PyParams& cfgExecutionProvider(ep::OpenVINO ov_ep); + + GAPI_WRAP + PyParams& cfgDisableMemPattern(); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index dc9a51e541..9c2118c3ad 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -27,6 +27,118 @@ namespace gapi { */ namespace onnx { +/** + * @brief This namespace contains Execution Providers structures for G-API ONNX Runtime backend. + */ +namespace ep { + +/** + * @brief This structure provides functions + * that fill inference options for ONNX OpenVINO Execution Provider. + * Please follow https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options + */ +struct GAPI_EXPORTS_W_SIMPLE OpenVINO { + // NB: Used from python. + /// @private -- Exclude this constructor from OpenCV documentation + GAPI_WRAP + OpenVINO() = default; + + /** @brief Class constructor. + + Constructs OpenVINO parameters based on device information. + + @param device Target device to use. + */ + GAPI_WRAP + OpenVINO(const std::string &device) + : device_id(device) { + } + + /** @brief Specifies OpenVINO Execution Provider device type. + + This function is used to override the accelerator hardware type + and precision at runtime. If this option is not explicitly configured, default + hardware and precision specified during onnxruntime build time is used. + + @param type Device type ("CPU_FP32", "GPU_FP16", etc) + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgDeviceType(const std::string &type) { + device_type = cv::util::make_optional(type); + return *this; + } + + /** @brief Specifies OpenVINO Execution Provider cache dir. + + This function is used to explicitly specify the path to save and load + the blobs enabling model caching feature. + + @param dir Path to the directory what will be used as cache. + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgCacheDir(const std::string &dir) { + cache_dir = dir; + return *this; + } + + /** @brief Specifies OpenVINO Execution Provider number of threads. + + This function is used to override the accelerator default value + of number of threads with this value at runtime. If this option + is not explicitly set, default value of 8 is used during build time. + + @param nthreads Number of threads. + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgNumThreads(size_t nthreads) { + num_of_threads = cv::util::make_optional(nthreads); + return *this; + } + + /** @brief Enables OpenVINO Execution Provider opencl throttling. + + This function is used to enable OpenCL queue throttling for GPU devices + (reduces CPU utilization when using GPU). + + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgEnableOpenCLThrottling() { + enable_opencl_throttling = true; + return *this; + } + + /** @brief Enables OpenVINO Execution Provider dynamic shapes. + + This function is used to enable OpenCL queue throttling for GPU devices + (reduces CPU utilization when using GPU). + This function is used to enable work with dynamic shaped models + whose shape will be set dynamically based on the infer input + image/data shape at run time in CPU. + + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgEnableDynamicShapes() { + enable_dynamic_shapes = true; + return *this; + } + + std::string device_id; + std::string cache_dir; + cv::optional device_type; + cv::optional num_of_threads; + bool enable_opencl_throttling = false; + bool enable_dynamic_shapes = false; +}; + +using EP = cv::util::variant; + +} // namespace ep + GAPI_EXPORTS cv::gapi::GBackend backend(); enum class TraitAs: int { @@ -78,6 +190,9 @@ struct ParamDesc { // when the generic infer parameters are unpacked (see GONNXBackendImpl::unpackKernel) std::unordered_map > generic_mstd; std::unordered_map generic_norm; + + cv::gapi::onnx::ep::EP execution_provider; + bool disable_mem_pattern; }; } // namespace detail @@ -115,6 +230,7 @@ public: desc.num_in = std::tuple_size::value; desc.num_out = std::tuple_size::value; desc.is_generic = false; + desc.disable_mem_pattern = false; }; /** @brief Specifies sequence of network input layers names for inference. @@ -279,6 +395,29 @@ public: return *this; } + /** @brief Specifies execution provider for runtime. + + The function is used to set ONNX Runtime OpenVINO Execution Provider options. + + @param ovep OpenVINO Execution Provider options. + @see cv::gapi::onnx::ep::OpenVINO. + + @return the reference on modified object. + */ + Params& cfgExecutionProvider(ep::OpenVINO&& ovep) { + desc.execution_provider = std::move(ovep); + return *this; + } + + /** @brief Disables the memory pattern optimization. + + @return the reference on modified object. + */ + Params& cfgDisableMemPattern() { + desc.disable_mem_pattern = true; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::onnx::backend(); } std::string tag() const { return Net::tag(); } @@ -306,7 +445,7 @@ public: @param model_path path to model file (.onnx file). */ Params(const std::string& tag, const std::string& model_path) - : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {} }, m_tag(tag) {} + : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, false }, m_tag(tag) {} void cfgMeanStdDev(const std::string &layer, const cv::Scalar &m, @@ -318,6 +457,14 @@ public: desc.generic_norm[layer] = flag; } + void cfgExecutionProvider(ep::OpenVINO&& ov_ep) { + desc.execution_provider = std::move(ov_ep); + } + + void cfgDisableMemPattern() { + desc.disable_mem_pattern = true; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::onnx::backend(); } std::string tag() const { return m_tag; } diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 70698ffd48..bdd0f0232f 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -29,6 +29,7 @@ using map_string_and_string = std::map; using map_string_and_vector_size_t = std::map>; using map_string_and_vector_float = std::map>; using map_int_and_double = std::map; +using ep_OpenVINO = cv::gapi::onnx::ep::OpenVINO; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs diff --git a/modules/gapi/src/backends/onnx/bindings_onnx.cpp b/modules/gapi/src/backends/onnx/bindings_onnx.cpp index c9c5fc58fa..ada90bd130 100644 --- a/modules/gapi/src/backends/onnx/bindings_onnx.cpp +++ b/modules/gapi/src/backends/onnx/bindings_onnx.cpp @@ -21,6 +21,18 @@ cv::gapi::onnx::PyParams& cv::gapi::onnx::PyParams::cfgNormalize(const std::stri return *this; } +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgExecutionProvider(cv::gapi::onnx::ep::OpenVINO ov_ep) { + m_priv->cfgExecutionProvider(std::move(ov_ep)); + return *this; +} + +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgDisableMemPattern() { + m_priv->cfgDisableMemPattern(); + return *this; +} + cv::gapi::GBackend cv::gapi::onnx::PyParams::backend() const { return m_priv->backend(); } diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp index 1194caeeb3..79bdad93d9 100644 --- a/modules/gapi/src/backends/onnx/gonnxbackend.cpp +++ b/modules/gapi/src/backends/onnx/gonnxbackend.cpp @@ -143,6 +143,41 @@ public: void run(); }; +static void appendExecutionProvider(Ort::SessionOptions *session_options, + const cv::gapi::onnx::ep::EP &execution_provider) { + namespace ep = cv::gapi::onnx::ep; + switch (execution_provider.index()) { + case ep::EP::index_of(): { + GAPI_LOG_INFO(NULL, "OpenVINO Execution Provider is selected."); + const auto &ovep = cv::util::get(execution_provider); + OrtOpenVINOProviderOptions options; + options.device_id = ovep.device_id.c_str(); + options.cache_dir = ovep.cache_dir.c_str(); + options.enable_opencl_throttling = ovep.enable_opencl_throttling; + options.enable_dynamic_shapes = ovep.enable_dynamic_shapes; + // NB: If are not specified, will be taken from onnxruntime build. + if (ovep.device_type) { + options.device_type = ovep.device_type->c_str(); + } + if (ovep.num_of_threads) { + options.num_of_threads = *ovep.num_of_threads; + } + try { + session_options->AppendExecutionProvider_OpenVINO(options); + } catch (const std::exception &e) { + std::stringstream ss; + ss << "ONNX Backend: Failed to enable OpenVINO Execution Provider: " + << e.what() << "\nMake sure that onnxruntime has" + " been compiled with OpenVINO support."; + cv::util::throw_error(std::runtime_error(ss.str())); + } + break; + } + default: + break; + } +} + } // namespace onnx } // namespace gimpl } // namespace cv @@ -592,9 +627,13 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp) cv::util::throw_error(std::logic_error("Please specify output layer names for " + params.model_path)); } - // Create and initialize the ONNX session Ort::SessionOptions session_options; + cv::gimpl::onnx::appendExecutionProvider(&session_options, pp.execution_provider); + + if (pp.disable_mem_pattern) { + session_options.DisableMemPattern(); + } this_env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, ""); #ifndef _WIN32 this_session = Ort::Session(this_env, params.model_path.data(), session_options);