Merge pull request #27460 from fcmiron:gapi_set_workloadtype_dynamically

G-API: Add support to set workload type dynamically in both OpenVINO and ONNX OVEP #27460

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [ ] The PR is proposed to the proper branch
- [ ] There is a reference to the original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake

modules/gapi/src/backends/onnx/gonnxbackend.cpp

@@ -126,8 +126,14 @@ class ONNXCompiled {
              std::vector<cv::Mat>& outs);
 
     std::vector<std::string> in_names_without_const;
+
+    cv::gapi::onnx::WorkloadTypeONNXPtr m_workload_type;
+    uint64_t m_workload_listener_id = 0;
+    void setWorkloadType(const std::string &type);
 public:
     explicit ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp);
+    ~ONNXCompiled();
+    void listenToWorkloadType(cv::gapi::onnx::WorkloadTypeONNXPtr workload);
 
     // Extract the information about output layer #i
     cv::GMatDesc outMeta(int i) const;
@@ -577,8 +583,10 @@ using GConstGONNXModel = ade::ConstTypedGraph
 } // anonymous namespace
 
 // GCPUExcecutable implementation //////////////////////////////////////////////
-cv::gimpl::onnx::GONNXExecutable::GONNXExecutable(const ade::Graph &g,
-                                                  const std::vector<ade::NodeHandle> &nodes)
+cv::gimpl::onnx::GONNXExecutable::GONNXExecutable(const ade::Graph                   &g,
+                                                  const std::vector<ade::NodeHandle> &nodes,
+                                                  const cv::GCompileArgs             &compileArgs)
+
     : m_g(g), m_gm(m_g) {
     // FIXME: Currently this backend is capable to run a single inference node only.
     // Need to extend our island fusion with merge/not-to-merge decision making parametrization
@@ -589,6 +597,11 @@ cv::gimpl::onnx::GONNXExecutable::GONNXExecutable(const ade::Graph &g,
         case NodeType::OP:
             if (this_nh == nullptr) {
                 this_nh = nh;
+                auto workload_arg = cv::gapi::getCompileArg<cv::gapi::onnx::WorkloadTypeONNXPtr>(compileArgs);
+                if(workload_arg.has_value()) {
+                    const auto &onnx_unit = iem.metadata(nh).get<ONNXUnit>();
+                    onnx_unit.oc->listenToWorkloadType(workload_arg.value());
+                }
             }
             else {
                 util::throw_error(std::logic_error("Multi-node inference is not supported!"));
@@ -834,6 +847,23 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp)
     out_data.resize(params.num_out);
 }
 
+ONNXCompiled::~ONNXCompiled() {
+    if (m_workload_type) {
+        m_workload_type->removeListener(m_workload_listener_id);
+    }
+}
+
+void ONNXCompiled::setWorkloadType(const std::string &type) {
+    const char* keys[] = {"ep.dynamic.workload_type"};
+    const char* values[] = {type.c_str()};
+    this_session.SetEpDynamicOptions(keys, values, 1);
+}
+
+void ONNXCompiled::listenToWorkloadType(cv::gapi::onnx::WorkloadTypeONNXPtr workload) {
+    m_workload_type = workload;
+    m_workload_listener_id = m_workload_type->addListener(std::bind(&ONNXCompiled::setWorkloadType, this, std::placeholders::_1));
+}
+
 std::vector<TensorInfo> ONNXCompiled::getTensorInfo(TensorPosition pos) {
     GAPI_Assert(pos == INPUT || pos == OUTPUT);
 
@@ -1348,9 +1378,9 @@ namespace {
         }
 
         virtual EPtr compile(const ade::Graph &graph,
-                             const cv::GCompileArgs &,
+                             const cv::GCompileArgs &compileArgs,
                              const std::vector<ade::NodeHandle> &nodes) const override {
-            return EPtr{new cv::gimpl::onnx::GONNXExecutable(graph, nodes)};
+            return EPtr{new cv::gimpl::onnx::GONNXExecutable(graph, nodes, compileArgs)};
         }
 
         virtual cv::GKernelPackage auxiliaryKernels() const override {

modules/gapi/src/backends/onnx/gonnxbackend.hpp

@@ -39,7 +39,8 @@ class GONNXExecutable final: public GIslandExecutable
 
 public:
     GONNXExecutable(const ade::Graph                   &graph,
-                    const std::vector<ade::NodeHandle> &nodes);
+                    const std::vector<ade::NodeHandle> &nodes,
+                    const cv::GCompileArgs             &compileArgs);
 
     virtual inline bool canReshape() const override { return false; }
     virtual inline void reshape(ade::Graph&, const GCompileArgs&) override {

modules/gapi/src/backends/ov/govbackend.cpp

@@ -1599,7 +1599,16 @@ cv::gimpl::ov::GOVExecutable::GOVExecutable(const ade::Graph &g,
                                             const cv::GCompileArgs &compileArgs,
                                             const std::vector<ade::NodeHandle> &nodes)
     : m_g(g), m_gm(m_g) {
+    auto workload_arg = cv::gapi::getCompileArg<cv::gapi::wip::ov::WorkloadTypeOVPtr>(compileArgs);
+    if(workload_arg.has_value()) {
+#if INF_ENGINE_RELEASE >= 2024030000
+        m_workload_type = workload_arg.value();
+        m_workload_listener_id = m_workload_type->addListener(std::bind(&GOVExecutable::setWorkloadType, this, std::placeholders::_1));
+#else
+        util::throw_error(std::logic_error("Workload type not supported in this version of OpenVINO, use >= 2024.3.0"));
+#endif
 
+    }
     m_options.inference_only =
         cv::gapi::getCompileArg<cv::gapi::wip::ov::benchmark_mode>(compileArgs).has_value();
     // FIXME: Currently this backend is capable to run a single inference node only.
@@ -1636,6 +1645,25 @@ cv::gimpl::ov::GOVExecutable::GOVExecutable(const ade::Graph &g,
     }
 }
 
+#if INF_ENGINE_RELEASE >= 2024030000
+cv::gimpl::ov::GOVExecutable::~GOVExecutable() {
+    if (m_workload_type)
+        m_workload_type->removeListener(m_workload_listener_id);
+}
+
+void cv::gimpl::ov::GOVExecutable::setWorkloadType(const std::string &type) {
+    if (type == "Default") {
+        compiled.compiled_model.set_property({{"WORKLOAD_TYPE", ::ov::WorkloadType::DEFAULT}});
+    }
+    else if (type == "Efficient") {
+        compiled.compiled_model.set_property({{"WORKLOAD_TYPE", ::ov::WorkloadType::EFFICIENT}});
+    }
+    else {
+        GAPI_LOG_WARNING(NULL, "Unknown value for WORKLOAD_TYPE");
+    }
+}
+#endif
+
 void cv::gimpl::ov::GOVExecutable::run(cv::gimpl::GIslandExecutable::IInput  &in,
                                        cv::gimpl::GIslandExecutable::IOutput &out) {
     std::vector<InObj>  input_objs;

modules/gapi/src/backends/ov/govbackend.hpp

@@ -50,12 +50,18 @@ class GOVExecutable final: public GIslandExecutable
 
     // To manage additional execution options
     Options m_options;
-
+#if INF_ENGINE_RELEASE >= 2024030000
+    cv::gapi::wip::ov::WorkloadTypeOVPtr m_workload_type;
+    uint64_t m_workload_listener_id = 0;
+    void setWorkloadType(const std::string &type);
+#endif
 public:
     GOVExecutable(const ade::Graph                   &graph,
                   const cv::GCompileArgs             &compileArgs,
                   const std::vector<ade::NodeHandle> &nodes);
-
+#if INF_ENGINE_RELEASE >= 2024030000
+    ~GOVExecutable();
+#endif
     virtual inline bool canReshape() const override { return false; }
     virtual inline void reshape(ade::Graph&, const GCompileArgs&) override {
         GAPI_Error("InternalError"); // Not implemented yet