From f174363f600aa578874b3f0cb7b190c5dad8a07d Mon Sep 17 00:00:00 2001 From: Dmitry Matveev Date: Wed, 6 Mar 2024 14:15:49 +0000 Subject: [PATCH] Merge pull request #25055 from dmatveev:dm/value_initialized_gmat G-API: A quick value-initialization support GMat #25055 This PR enables `GMat` objects to be value-initialized in the same way as it was done for `GScalar`s (and, possibly, other types). - Added some helper methods in backends to distinguish if a certain G-type value initialization is supported or not; - Added tests, including negative. Where it is needed: - Further extension of the OVCV backend (#24379 - will be refreshed soon); - Further experiments with DNN module; - Further experiments with "G-API behind UMat" sort of aggregation. In the current form, PR can be reviewed & merged (@TolyaTalamanov please have a look) ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/gapi/include/opencv2/gapi/gmat.hpp | 12 ++ modules/gapi/include/opencv2/gapi/gscalar.hpp | 11 +- modules/gapi/src/api/gbackend.cpp | 3 + modules/gapi/src/api/gbackend_priv.hpp | 8 ++ modules/gapi/src/api/gcomputation.cpp | 9 +- modules/gapi/src/api/gmat.cpp | 4 + modules/gapi/src/api/gproto.cpp | 1 + modules/gapi/src/backends/cpu/gcpubackend.cpp | 11 ++ modules/gapi/src/backends/ocl/goclbackend.cpp | 11 ++ modules/gapi/src/compiler/gobjref.hpp | 1 + modules/gapi/src/compiler/passes/exec.cpp | 24 +++- modules/gapi/src/executor/gexecutor.cpp | 6 + .../gapi/src/executor/gthreadedexecutor.cpp | 6 +- modules/gapi/test/gapi_mat_tests.cpp | 114 ++++++++++++++++++ 14 files changed, 208 insertions(+), 13 deletions(-) create mode 100644 modules/gapi/test/gapi_mat_tests.cpp diff --git a/modules/gapi/include/opencv2/gapi/gmat.hpp b/modules/gapi/include/opencv2/gapi/gmat.hpp index 198cb728d0..6d6f74ff7f 100644 --- a/modules/gapi/include/opencv2/gapi/gmat.hpp +++ b/modules/gapi/include/opencv2/gapi/gmat.hpp @@ -77,6 +77,18 @@ public: */ GAPI_WRAP GMat(); // Empty constructor + /** + * @brief Constructs a value-initialized GMat + * + * GMat may be associated with a buffer at graph construction time. + * It is useful when some operation has a Mat input which doesn't + * change during the program execution, and is set only once. + * In this case, there's no need to declare such GMat as graph input. + * + * @param m a cv::Mat buffer to associate with this GMat object. + */ + GAPI_WRAP explicit GMat(cv::Mat m); // Value-initialization constructor + /// @private GMat(const GNode &n, std::size_t out); // Operation result constructor /// @private diff --git a/modules/gapi/include/opencv2/gapi/gscalar.hpp b/modules/gapi/include/opencv2/gapi/gscalar.hpp index 43d12c782a..de0dfe1383 100644 --- a/modules/gapi/include/opencv2/gapi/gscalar.hpp +++ b/modules/gapi/include/opencv2/gapi/gscalar.hpp @@ -54,12 +54,11 @@ public: /** * @brief Constructs a value-initialized GScalar * - * In contrast with GMat (which can be either an explicit graph input - * or a result of some operation), GScalars may have their values - * be associated at graph construction time. It is useful when - * some operation has a GScalar input which doesn't change during - * the program execution, and is set only once. In this case, - * there is no need to declare such GScalar as a graph input. + * GScalars may have their values be associated at graph + * construction time. It is useful when some operation has a + * GScalar input which doesn't change during the program + * execution, and is set only once. In this case, there is no need + * to declare such GScalar as a graph input. * * @note The value of GScalar may be overwritten by assigning some * other GScalar to the object using `operator=` -- on the diff --git a/modules/gapi/src/api/gbackend.cpp b/modules/gapi/src/api/gbackend.cpp index 46c8dc1640..dc51eef76d 100644 --- a/modules/gapi/src/api/gbackend.cpp +++ b/modules/gapi/src/api/gbackend.cpp @@ -80,6 +80,9 @@ bool cv::gapi::GBackend::Priv::allowsMerge(const cv::gimpl::GIslandModel::Graph return true; } +bool cv::gapi::GBackend::Priv::supportsConst(cv::GShape) const { + return false; +} // GBackend public implementation ////////////////////////////////////////////// cv::gapi::GBackend::GBackend() diff --git a/modules/gapi/src/api/gbackend_priv.hpp b/modules/gapi/src/api/gbackend_priv.hpp index 5609b304aa..80436abde8 100644 --- a/modules/gapi/src/api/gbackend_priv.hpp +++ b/modules/gapi/src/api/gbackend_priv.hpp @@ -84,6 +84,14 @@ public: const ade::NodeHandle &slot_nh, const ade::NodeHandle &b_nh) const; + // Ask backend if it supports CONST_VAL data of the given shape or not. + // If the backend does support this data type, a Data node with such + // value can be fused into the backend's Island body. + // If the backend doesn't support this data type, a Data node won't + // be fused into the Islands's body -- will be marked as an in-graph + // input connection for this Island. + virtual bool supportsConst(cv::GShape shape) const; + virtual ~Priv() = default; }; diff --git a/modules/gapi/src/api/gcomputation.cpp b/modules/gapi/src/api/gcomputation.cpp index c57c66b879..076c1acee3 100644 --- a/modules/gapi/src/api/gcomputation.cpp +++ b/modules/gapi/src/api/gcomputation.cpp @@ -191,8 +191,8 @@ void cv::GComputation::recompile(GMetaArgs&& in_metas, GCompileArgs &&args) if (m_priv->m_lastMetas != in_metas) { if (m_priv->m_lastCompiled && - m_priv->m_lastCompiled.canReshape() && - formats_are_same(m_priv->m_lastMetas, in_metas)) + m_priv->m_lastCompiled.canReshape() && + formats_are_same(m_priv->m_lastMetas, in_metas)) { m_priv->m_lastCompiled.reshape(in_metas, args); } @@ -203,6 +203,11 @@ void cv::GComputation::recompile(GMetaArgs&& in_metas, GCompileArgs &&args) } m_priv->m_lastMetas = in_metas; } + else if (in_metas.size() == 0) { + // Happens when the graph is head-less (e.g. starts with const-vals only) + // always compile ad-hoc + m_priv->m_lastCompiled = compile(GMetaArgs(in_metas), std::move(args)); + } } void cv::GComputation::apply(GRunArgs &&ins, GRunArgsP &&outs, GCompileArgs &&args) diff --git a/modules/gapi/src/api/gmat.cpp b/modules/gapi/src/api/gmat.cpp index 03f2e736be..c6bcb288ba 100644 --- a/modules/gapi/src/api/gmat.cpp +++ b/modules/gapi/src/api/gmat.cpp @@ -26,6 +26,10 @@ cv::GMat::GMat(const GNode &n, std::size_t out) { } +cv::GMat::GMat(cv::Mat m) + : m_priv(new GOrigin(GShape::GMAT, cv::gimpl::ConstVal(m))) { +} + cv::GOrigin& cv::GMat::priv() { return *m_priv; diff --git a/modules/gapi/src/api/gproto.cpp b/modules/gapi/src/api/gproto.cpp index 43bcfb9c14..bb60f4e51f 100644 --- a/modules/gapi/src/api/gproto.cpp +++ b/modules/gapi/src/api/gproto.cpp @@ -80,6 +80,7 @@ cv::GRunArg cv::value_of(const cv::GOrigin &origin) { case GShape::GSCALAR: return GRunArg(util::get(origin.value)); case GShape::GARRAY: return GRunArg(util::get(origin.value)); + case GShape::GMAT: return GRunArg(util::get(origin.value)); default: util::throw_error(std::logic_error("Unsupported shape for constant")); } } diff --git a/modules/gapi/src/backends/cpu/gcpubackend.cpp b/modules/gapi/src/backends/cpu/gcpubackend.cpp index f50f8ecd28..c6a35ead27 100644 --- a/modules/gapi/src/backends/cpu/gcpubackend.cpp +++ b/modules/gapi/src/backends/cpu/gcpubackend.cpp @@ -65,6 +65,17 @@ namespace { return EPtr{new cv::gimpl::GCPUExecutable(graph, compileArgs, nodes)}; } + + virtual bool supportsConst(cv::GShape shape) const override + { + // Supports all types of const values + return shape == cv::GShape::GOPAQUE + || shape == cv::GShape::GSCALAR + || shape == cv::GShape::GARRAY; + // yes, value-initialized GMats are not supported currently + // as in-island data -- compiler will lift these values to the + // GIslandModel's SLOT level (will be handled uniformly) + } }; } diff --git a/modules/gapi/src/backends/ocl/goclbackend.cpp b/modules/gapi/src/backends/ocl/goclbackend.cpp index 9c6d7154e4..eb63cec35b 100644 --- a/modules/gapi/src/backends/ocl/goclbackend.cpp +++ b/modules/gapi/src/backends/ocl/goclbackend.cpp @@ -62,6 +62,17 @@ namespace { return EPtr{new cv::gimpl::GOCLExecutable(graph, nodes)}; } + + virtual bool supportsConst(cv::GShape shape) const override + { + // Supports all types of const values + return shape == cv::GShape::GOPAQUE + || shape == cv::GShape::GSCALAR + || shape == cv::GShape::GARRAY; + // yes, value-initialized GMats are not supported currently + // as in-island data -- compiler will lift these values to the + // GIslandModel's SLOT level (will be handled uniformly) + } }; } diff --git a/modules/gapi/src/compiler/gobjref.hpp b/modules/gapi/src/compiler/gobjref.hpp index bca6fa525e..8f68142cc5 100644 --- a/modules/gapi/src/compiler/gobjref.hpp +++ b/modules/gapi/src/compiler/gobjref.hpp @@ -24,6 +24,7 @@ namespace gimpl < util::monostate , cv::Scalar , cv::detail::VectorRef + , cv::Mat >; struct RcDesc diff --git a/modules/gapi/src/compiler/passes/exec.cpp b/modules/gapi/src/compiler/passes/exec.cpp index 24dcef7bc1..a978e47b42 100644 --- a/modules/gapi/src/compiler/passes/exec.cpp +++ b/modules/gapi/src/compiler/passes/exec.cpp @@ -9,7 +9,7 @@ #include #include // list -#include // setw, etc +#include // setw, etc #include // ofstream #include #include @@ -85,7 +85,7 @@ namespace const auto& backend = *src_g.metadata().get().backends.cbegin(); const auto& proto = src_g.metadata().get(); - GIsland::node_set all, in_ops, out_ops; + GIsland::node_set all, in_ops, out_ops, in_cvals; all.insert(src_g.nodes().begin(), src_g.nodes().end()); @@ -99,7 +99,22 @@ namespace all.erase(nh); out_ops.insert(nh->inNodes().begin(), nh->inNodes().end()); } - + for (const auto& nh : src_g.nodes()) + { + if (src_g.metadata(nh).get().t == NodeType::DATA) + { + const auto &d = src_g.metadata(nh).get(); + if (d.storage == Data::Storage::CONST_VAL + && !backend.priv().supportsConst(d.shape)) { + // don't put this node into the island's graph - so the island + // executable don't need to handle value-initialized G-type manually. + // Still mark its readers as inputs + all.erase(nh); + in_cvals.insert(nh); + in_ops.insert(nh->outNodes().begin(), nh->outNodes().end()); + } + } + } auto isl = std::make_shared(backend, std::move(all), std::move(in_ops), @@ -108,7 +123,8 @@ namespace auto ih = GIslandModel::mkIslandNode(g, std::move(isl)); - for (const auto& nh : proto.in_nhs) + for (const auto& nh : ade::util::chain(ade::util::toRange(proto.in_nhs), + ade::util::toRange(in_cvals))) { auto slot = GIslandModel::mkSlotNode(g, nh); g.link(slot, ih); diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index bf25302b75..f68aa18fbb 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -208,6 +208,12 @@ void cv::gimpl::GExecutor::initResource(const ade::NodeHandle & nh, const ade::N switch (d.shape) { case GShape::GMAT: + if (d.storage == Data::Storage::CONST_VAL) + { + auto rc = RcDesc{d.rc, d.shape, d.ctor}; + magazine::bindInArgExec(m_res, rc, m_gm.metadata(orig_nh).get().arg); + } + else { // Let island allocate it's outputs if it can, // allocate cv::Mat and wrap it with RMat otherwise diff --git a/modules/gapi/src/executor/gthreadedexecutor.cpp b/modules/gapi/src/executor/gthreadedexecutor.cpp index 6999b7bb8d..ecd53eb952 100644 --- a/modules/gapi/src/executor/gthreadedexecutor.cpp +++ b/modules/gapi/src/executor/gthreadedexecutor.cpp @@ -175,7 +175,11 @@ void cv::gimpl::GThreadedExecutor::initResource(const ade::NodeHandle &nh, const // to as it is bound externally (e.g. already in the m_state.mag) switch (d.shape) { - case GShape::GMAT: { + case GShape::GMAT: + if (d.storage == Data::Storage::CONST_VAL) { + auto rc = RcDesc{d.rc, d.shape, d.ctor}; + magazine::bindInArgExec(m_state.mag, rc, m_gm.metadata(orig_nh).get().arg); + } else { // Let island allocate it's outputs if it can, // allocate cv::Mat and wrap it with RMat otherwise GAPI_Assert(!nh->inNodes().empty()); diff --git a/modules/gapi/test/gapi_mat_tests.cpp b/modules/gapi/test/gapi_mat_tests.cpp new file mode 100644 index 0000000000..0f2d1cc7a2 --- /dev/null +++ b/modules/gapi/test/gapi_mat_tests.cpp @@ -0,0 +1,114 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2024 Intel Corporation + + +#include "test_precomp.hpp" + +#include +#include +#include + +namespace opencv_test +{ +namespace +{ +enum class KernelPackage: int +{ + OCV, + OCL, + FLUID, +}; +std::ostream& operator<< (std::ostream &os, const KernelPackage &e) +{ + switch (e) + { +#define _C(X) case KernelPackage::X: os << #X; break + _C(OCV); + _C(OCL); + _C(FLUID); +#undef _C + default: GAPI_Error("Unknown package"); + } + return os; +} +} // namespace + +struct GMatWithValue : public TestWithParam { + cv::GKernelPackage getKernelPackage() { + switch (GetParam()) { + case KernelPackage::OCV: return cv::gapi::core::cpu::kernels(); + case KernelPackage::OCL: return cv::gapi::core::ocl::kernels(); + case KernelPackage::FLUID: return cv::gapi::core::fluid::kernels(); + default: GAPI_Error("Unknown package"); + } + } +}; + +TEST_P(GMatWithValue, SingleIsland) +{ + cv::Size sz(2, 2); + cv::Mat in_mat = cv::Mat::eye(sz, CV_8U); + + cv::GComputationT addEye([&](cv::GMat in) { + return in + cv::GMat(cv::Mat::eye(sz, CV_8U)); + }); + + cv::Mat out_mat; + addEye.apply(in_mat, out_mat, cv::compile_args(cv::gapi::use_only{getKernelPackage()})); + + cv::Mat out_mat_ref = in_mat*2; + EXPECT_EQ(0, cvtest::norm(out_mat, out_mat_ref, NORM_INF)); +} + +TEST_P(GMatWithValue, GraphWithNoInput) +{ + cv::Mat cval = cv::Mat::eye(cv::Size(2, 2), CV_8U); + cv::GMat gval = cv::GMat(cval); + cv::GMat out = cv::gapi::bitwise_not(gval); + + cv::Mat out_mat; + cv::GComputation f(cv::GIn(), cv::GOut(out)); + + // Compiling this isn't supported for now + EXPECT_ANY_THROW(f.compile(cv::descr_of(cval), + cv::compile_args(cv::gapi::use_only{getKernelPackage()}))); +} + +INSTANTIATE_TEST_CASE_P(GAPI_GMat, GMatWithValue, + Values(KernelPackage::OCV, + KernelPackage::OCL, + KernelPackage::FLUID)); + +TEST(GAPI_MatWithValue, MultipleIslands) +{ + // This test employs a non-trivial island fusion process + // as there's multiple backends in the graph + + cv::Size sz(2, 2); + cv::Mat cval2 = cv::Mat::eye(sz, CV_8U) * 2; + cv::Mat cval1 = cv::Mat::eye(sz, CV_8U); + + cv::GMat in; + cv::GMat tmp = in + cv::GMat(cval2); // Will be a Fluid operation + cv::GMat out = tmp - cv::GMat(cval1); // Will be an OCV operation + + cv::GKernelPackage fluid_kernels = cv::gapi::core::fluid::kernels(); + cv::GKernelPackage opencv_kernels = cv::gapi::core::cpu::kernels(); + fluid_kernels.remove(); + opencv_kernels.remove(); + auto kernels = cv::gapi::combine(fluid_kernels, opencv_kernels); + + cv::Mat in_mat = cv::Mat::zeros(sz, CV_8U); + cv::Mat out_mat; + auto cc = cv::GComputation(in, out) + .compile(cv::descr_of(in_mat), + cv::compile_args(cv::gapi::use_only{kernels})); + cc(cv::gin(in_mat), cv::gout(out_mat)); + + EXPECT_EQ(0, cvtest::norm(out_mat, cv::Mat::eye(sz, CV_8U), NORM_INF)); +} + +} // namespace opencv_test