diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index eb6316b446..f2e2cfac54 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -20,6 +20,7 @@ #include // GAPI_EXPORTS #include // GKernelPackage #include // Generic +#include namespace cv { namespace gapi { @@ -752,8 +753,16 @@ protected: std::string m_tag; }; +class WorkloadTypeONNX : public WorkloadType {}; +using WorkloadTypeONNXPtr = std::shared_ptr; + } // namespace onnx } // namespace gapi +namespace detail { +template<> struct CompileArgTag { + static const char* tag() { return "gapi.onnx.workload_type"; } +}; +} // namespace detail } // namespace cv #endif // OPENCV_GAPI_INFER_HPP diff --git a/modules/gapi/include/opencv2/gapi/infer/ov.hpp b/modules/gapi/include/opencv2/gapi/infer/ov.hpp index 3673ff53a2..d228879fd1 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ov.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ov.hpp @@ -13,6 +13,7 @@ #include // GAPI_EXPORTS #include // GKernelType[M], GBackend #include // Generic +#include #include @@ -745,6 +746,9 @@ namespace wip { namespace ov { */ struct benchmark_mode { }; +class WorkloadTypeOV : public WorkloadType {}; +using WorkloadTypeOVPtr = std::shared_ptr; + } // namespace ov } // namespace wip @@ -756,6 +760,10 @@ namespace detail { static const char* tag() { return "gapi.wip.ov.benchmark_mode"; } }; + template<> struct CompileArgTag + { + static const char* tag() { return "gapi.wip.ov.workload_type"; } + }; } } // namespace cv diff --git a/modules/gapi/include/opencv2/gapi/infer/workload_type.hpp b/modules/gapi/include/opencv2/gapi/infer/workload_type.hpp new file mode 100644 index 0000000000..76482451f7 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/infer/workload_type.hpp @@ -0,0 +1,58 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2025 Intel Corporation + +#ifndef OPENCV_WORKLOADTYPE_HPP +#define OPENCV_WORKLOADTYPE_HPP + +#include +#include +#include +#include + +using Callback = std::function; + +class WorkloadListener { + Callback callback; +public: + uint64_t id; + WorkloadListener(const Callback &cb, uint64_t listener_id) : callback(cb), id(listener_id) {} + + void operator()(const std::string &type) const { + if (callback) { + callback(type); + } + } + + bool operator==(const WorkloadListener& other) const { + return id == other.id; + } +}; +class WorkloadType { + std::vector listeners; + uint64_t nextId = 1; +public: + uint64_t addListener(const Callback &cb) { + uint64_t id = nextId++; + listeners.emplace_back(cb, id); + return id; + } + + void removeListener(uint64_t id) { + auto it = std::remove_if(listeners.begin(), listeners.end(), + [id](const WorkloadListener& entry) { return entry.id == id; }); + if (it != listeners.end()) { + listeners.erase(it, listeners.end()); + } + } + + void notify(const std::string &type) { + for (const auto &listener : listeners) { + listener(type); + } + } +}; + +#endif // OPENCV_WORKLOADTYPE_HPP diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp index 546e5e9088..cbdd615c20 100644 --- a/modules/gapi/src/backends/onnx/gonnxbackend.cpp +++ b/modules/gapi/src/backends/onnx/gonnxbackend.cpp @@ -126,8 +126,14 @@ class ONNXCompiled { std::vector& outs); std::vector in_names_without_const; + + cv::gapi::onnx::WorkloadTypeONNXPtr m_workload_type; + uint64_t m_workload_listener_id = 0; + void setWorkloadType(const std::string &type); public: explicit ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp); + ~ONNXCompiled(); + void listenToWorkloadType(cv::gapi::onnx::WorkloadTypeONNXPtr workload); // Extract the information about output layer #i cv::GMatDesc outMeta(int i) const; @@ -577,8 +583,10 @@ using GConstGONNXModel = ade::ConstTypedGraph } // anonymous namespace // GCPUExcecutable implementation ////////////////////////////////////////////// -cv::gimpl::onnx::GONNXExecutable::GONNXExecutable(const ade::Graph &g, - const std::vector &nodes) +cv::gimpl::onnx::GONNXExecutable::GONNXExecutable(const ade::Graph &g, + const std::vector &nodes, + const cv::GCompileArgs &compileArgs) + : m_g(g), m_gm(m_g) { // FIXME: Currently this backend is capable to run a single inference node only. // Need to extend our island fusion with merge/not-to-merge decision making parametrization @@ -589,6 +597,11 @@ cv::gimpl::onnx::GONNXExecutable::GONNXExecutable(const ade::Graph &g, case NodeType::OP: if (this_nh == nullptr) { this_nh = nh; + auto workload_arg = cv::gapi::getCompileArg(compileArgs); + if(workload_arg.has_value()) { + const auto &onnx_unit = iem.metadata(nh).get(); + onnx_unit.oc->listenToWorkloadType(workload_arg.value()); + } } else { util::throw_error(std::logic_error("Multi-node inference is not supported!")); @@ -834,6 +847,23 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp) out_data.resize(params.num_out); } +ONNXCompiled::~ONNXCompiled() { + if (m_workload_type) { + m_workload_type->removeListener(m_workload_listener_id); + } +} + +void ONNXCompiled::setWorkloadType(const std::string &type) { + const char* keys[] = {"ep.dynamic.workload_type"}; + const char* values[] = {type.c_str()}; + this_session.SetEpDynamicOptions(keys, values, 1); +} + +void ONNXCompiled::listenToWorkloadType(cv::gapi::onnx::WorkloadTypeONNXPtr workload) { + m_workload_type = workload; + m_workload_listener_id = m_workload_type->addListener(std::bind(&ONNXCompiled::setWorkloadType, this, std::placeholders::_1)); +} + std::vector ONNXCompiled::getTensorInfo(TensorPosition pos) { GAPI_Assert(pos == INPUT || pos == OUTPUT); @@ -1348,9 +1378,9 @@ namespace { } virtual EPtr compile(const ade::Graph &graph, - const cv::GCompileArgs &, + const cv::GCompileArgs &compileArgs, const std::vector &nodes) const override { - return EPtr{new cv::gimpl::onnx::GONNXExecutable(graph, nodes)}; + return EPtr{new cv::gimpl::onnx::GONNXExecutable(graph, nodes, compileArgs)}; } virtual cv::GKernelPackage auxiliaryKernels() const override { diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.hpp b/modules/gapi/src/backends/onnx/gonnxbackend.hpp index 8c67df1e1e..28f5feb3aa 100644 --- a/modules/gapi/src/backends/onnx/gonnxbackend.hpp +++ b/modules/gapi/src/backends/onnx/gonnxbackend.hpp @@ -39,7 +39,8 @@ class GONNXExecutable final: public GIslandExecutable public: GONNXExecutable(const ade::Graph &graph, - const std::vector &nodes); + const std::vector &nodes, + const cv::GCompileArgs &compileArgs); virtual inline bool canReshape() const override { return false; } virtual inline void reshape(ade::Graph&, const GCompileArgs&) override { diff --git a/modules/gapi/src/backends/ov/govbackend.cpp b/modules/gapi/src/backends/ov/govbackend.cpp index 15c216d716..99574195e2 100644 --- a/modules/gapi/src/backends/ov/govbackend.cpp +++ b/modules/gapi/src/backends/ov/govbackend.cpp @@ -1599,7 +1599,16 @@ cv::gimpl::ov::GOVExecutable::GOVExecutable(const ade::Graph &g, const cv::GCompileArgs &compileArgs, const std::vector &nodes) : m_g(g), m_gm(m_g) { + auto workload_arg = cv::gapi::getCompileArg(compileArgs); + if(workload_arg.has_value()) { +#if INF_ENGINE_RELEASE >= 2024030000 + m_workload_type = workload_arg.value(); + m_workload_listener_id = m_workload_type->addListener(std::bind(&GOVExecutable::setWorkloadType, this, std::placeholders::_1)); +#else + util::throw_error(std::logic_error("Workload type not supported in this version of OpenVINO, use >= 2024.3.0")); +#endif + } m_options.inference_only = cv::gapi::getCompileArg(compileArgs).has_value(); // FIXME: Currently this backend is capable to run a single inference node only. @@ -1636,6 +1645,25 @@ cv::gimpl::ov::GOVExecutable::GOVExecutable(const ade::Graph &g, } } +#if INF_ENGINE_RELEASE >= 2024030000 +cv::gimpl::ov::GOVExecutable::~GOVExecutable() { + if (m_workload_type) + m_workload_type->removeListener(m_workload_listener_id); +} + +void cv::gimpl::ov::GOVExecutable::setWorkloadType(const std::string &type) { + if (type == "Default") { + compiled.compiled_model.set_property({{"WORKLOAD_TYPE", ::ov::WorkloadType::DEFAULT}}); + } + else if (type == "Efficient") { + compiled.compiled_model.set_property({{"WORKLOAD_TYPE", ::ov::WorkloadType::EFFICIENT}}); + } + else { + GAPI_LOG_WARNING(NULL, "Unknown value for WORKLOAD_TYPE"); + } +} +#endif + void cv::gimpl::ov::GOVExecutable::run(cv::gimpl::GIslandExecutable::IInput &in, cv::gimpl::GIslandExecutable::IOutput &out) { std::vector input_objs; diff --git a/modules/gapi/src/backends/ov/govbackend.hpp b/modules/gapi/src/backends/ov/govbackend.hpp index ff9793afad..4c7d627a08 100644 --- a/modules/gapi/src/backends/ov/govbackend.hpp +++ b/modules/gapi/src/backends/ov/govbackend.hpp @@ -50,12 +50,18 @@ class GOVExecutable final: public GIslandExecutable // To manage additional execution options Options m_options; - +#if INF_ENGINE_RELEASE >= 2024030000 + cv::gapi::wip::ov::WorkloadTypeOVPtr m_workload_type; + uint64_t m_workload_listener_id = 0; + void setWorkloadType(const std::string &type); +#endif public: GOVExecutable(const ade::Graph &graph, const cv::GCompileArgs &compileArgs, const std::vector &nodes); - +#if INF_ENGINE_RELEASE >= 2024030000 + ~GOVExecutable(); +#endif virtual inline bool canReshape() const override { return false; } virtual inline void reshape(ade::Graph&, const GCompileArgs&) override { GAPI_Error("InternalError"); // Not implemented yet