Merge pull request #19009 from TolyaTalamanov:at/media-frame-copy

[G-API] GStreamingBackend

* Snapshot

* Implement StreamingBackend

* Refactoring

* Refactoring 2

* Clean up

* Add missing functionality to support MediaFrame as output

* Partially address review comments

* Fix build

* Implement reshape for gstreamingbackend and add a test on it

* Address more comments

* Add format.hpp to gapi.hpp

* Fix debug build

* Address review comments

Co-authored-by: Smirnov Alexey <alexey.smirnov@intel.com>

modules/gapi/src/api/gbackend.cpp

@@ -211,6 +211,9 @@ void bindOutArg(Mag& mag, const RcDesc &rc, const GRunArgP &arg, HandleRMat hand
         }
         break;
     }
+    case GShape::GFRAME:
+        mag.template slot<cv::MediaFrame>()[rc.id] = *util::get<cv::MediaFrame*>(arg);
+        break;
     case GShape::GARRAY:
         mag.template slot<cv::detail::VectorRef>()[rc.id] = util::get<cv::detail::VectorRef>(arg);
         break;
@@ -319,6 +322,9 @@ cv::GRunArgP getObjPtr(Mag& mag, const RcDesc &rc, bool is_umat)
         // debugging this!!!1
         return GRunArgP(const_cast<const Mag&>(mag)
                         .template slot<cv::detail::OpaqueRef>().at(rc.id));
+    case GShape::GFRAME:
+        return GRunArgP(&mag.template slot<cv::MediaFrame>()[rc.id]);
+
     default:
         util::throw_error(std::logic_error("Unsupported GShape type"));
         break;
@@ -345,6 +351,12 @@ void writeBack(const Mag& mag, const RcDesc &rc, GRunArgP &g_arg)
         break;
     }
 
+    case GShape::GFRAME:
+    {
+        *util::get<cv::MediaFrame*>(g_arg) = mag.template slot<cv::MediaFrame>().at(rc.id);
+        break;
+    }
+
     default:
         util::throw_error(std::logic_error("Unsupported GShape type"));
         break;

modules/gapi/src/api/gcall.cpp

@@ -69,6 +69,11 @@ cv::detail::GOpaqueU cv::GCall::yieldOpaque(int output)
     return cv::detail::GOpaqueU(m_priv->m_node, output);
 }
 
+cv::GFrame cv::GCall::yieldFrame(int output)
+{
+    return cv::GFrame(m_priv->m_node, output);
+}
+
 cv::GCall::Priv& cv::GCall::priv()
 {
     return *m_priv;

modules/gapi/src/api/gframe.cpp

@@ -8,6 +8,7 @@
 #include "precomp.hpp"
 
 #include <opencv2/gapi/gframe.hpp>
+#include <opencv2/gapi/media.hpp>
 
 #include "api/gorigin.hpp"
 
@@ -34,6 +35,10 @@ bool GFrameDesc::operator== (const GFrameDesc &rhs) const {
     return fmt == rhs.fmt && size == rhs.size;
 }
 
+GFrameDesc descr_of(const cv::MediaFrame &frame) {
+    return frame.desc();
+}
+
 std::ostream& operator<<(std::ostream& os, const cv::GFrameDesc &d) {
     os << '[';
     switch (d.fmt) {

modules/gapi/src/api/gproto.cpp

@@ -146,6 +146,7 @@ cv::GMetaArg cv::descr_of(const cv::GRunArgP &argp)
 #endif //  !defined(GAPI_STANDALONE)
     case GRunArgP::index_of<cv::Mat*>():               return GMetaArg(cv::descr_of(*util::get<cv::Mat*>(argp)));
     case GRunArgP::index_of<cv::Scalar*>():            return GMetaArg(descr_of(*util::get<cv::Scalar*>(argp)));
+    case GRunArgP::index_of<cv::MediaFrame*>():        return GMetaArg(descr_of(*util::get<cv::MediaFrame*>(argp)));
     case GRunArgP::index_of<cv::detail::VectorRef>():  return GMetaArg(util::get<cv::detail::VectorRef>(argp).descr_of());
     case GRunArgP::index_of<cv::detail::OpaqueRef>():  return GMetaArg(util::get<cv::detail::OpaqueRef>(argp).descr_of());
     default: util::throw_error(std::logic_error("Unsupported GRunArgP type"));
@@ -163,6 +164,7 @@ bool cv::can_describe(const GMetaArg& meta, const GRunArgP& argp)
     case GRunArgP::index_of<cv::Mat*>():               return util::holds_alternative<GMatDesc>(meta) &&
                                                               util::get<GMatDesc>(meta).canDescribe(*util::get<cv::Mat*>(argp));
     case GRunArgP::index_of<cv::Scalar*>():            return meta == GMetaArg(cv::descr_of(*util::get<cv::Scalar*>(argp)));
+    case GRunArgP::index_of<cv::MediaFrame*>():        return meta == GMetaArg(cv::descr_of(*util::get<cv::MediaFrame*>(argp)));
     case GRunArgP::index_of<cv::detail::VectorRef>():  return meta == GMetaArg(util::get<cv::detail::VectorRef>(argp).descr_of());
     case GRunArgP::index_of<cv::detail::OpaqueRef>():  return meta == GMetaArg(util::get<cv::detail::OpaqueRef>(argp).descr_of());
     default: util::throw_error(std::logic_error("Unsupported GRunArgP type"));
@@ -288,6 +290,8 @@ const void* cv::gimpl::proto::ptr(const GRunArgP &arg)
         return cv::util::get<cv::detail::VectorRef>(arg).ptr();
     case GRunArgP::index_of<cv::detail::OpaqueRef>():
         return cv::util::get<cv::detail::OpaqueRef>(arg).ptr();
+    case GRunArgP::index_of<cv::MediaFrame*>():
+        return static_cast<const void*>(cv::util::get<cv::MediaFrame*>(arg));
     default:
         util::throw_error(std::logic_error("Unknown GRunArgP type!"));
     }

modules/gapi/src/api/kernels_streaming.cpp

@@ -7,6 +7,8 @@
 #include "precomp.hpp"
 
 #include <opencv2/gapi/streaming/desync.hpp>
+#include <opencv2/gapi/streaming/format.hpp>
+
 #include <opencv2/gapi/core.hpp>
 
 cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) {
@@ -72,3 +74,11 @@ cv::GMat cv::gapi::streaming::desync(const cv::GMat &g) {
     // connected to a desynchronized data object, and this sole last_written_value
     // object will feed both branches of the streaming executable.
 }
+
+cv::GFrame cv::gapi::streaming::copy(const cv::GFrame& in) {
+    return cv::gapi::streaming::GCopy::on(in);
+}
+
+cv::GMat cv::gapi::streaming::BGR(const cv::GFrame& in) {
+    return cv::gapi::streaming::GBGR::on(in);
+}

modules/gapi/src/backends/cpu/gcpukernel.cpp

@@ -41,6 +41,11 @@ cv::detail::OpaqueRef& cv::GCPUContext::outOpaqueRef(int output)
     return util::get<cv::detail::OpaqueRef>(m_results.at(output));
 }
 
+cv::MediaFrame& cv::GCPUContext::outFrame(int output)
+{
+    return *util::get<cv::MediaFrame*>(m_results.at(output));
+}
+
 cv::GCPUKernel::GCPUKernel()
 {
 }

modules/gapi/src/backends/streaming/gstreamingbackend.cpp

@@ -0,0 +1,203 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+#include <opencv2/imgproc.hpp>
+#include <opencv2/gapi/util/throw.hpp> // throw_error
+#include <opencv2/gapi/streaming/format.hpp> // kernels
+
+#include "api/gbackend_priv.hpp"
+#include "backends/common/gbackend.hpp"
+
+#include "gstreamingbackend.hpp"
+#include "gstreamingkernel.hpp"
+
+namespace {
+
+struct StreamingCreateFunction
+{
+    static const char *name() { return "StreamingCreateFunction";  }
+    cv::gapi::streaming::CreateActorFunction createActorFunction;
+};
+
+using StreamingGraph = ade::TypedGraph
+    < cv::gimpl::Op
+    , StreamingCreateFunction
+    >;
+
+using ConstStreamingGraph = ade::ConstTypedGraph
+    < cv::gimpl::Op
+    , StreamingCreateFunction
+    >;
+
+
+class GStreamingIntrinExecutable final: public cv::gimpl::GIslandExecutable
+{
+    virtual void run(std::vector<InObj>  &&,
+                     std::vector<OutObj> &&) override {
+        GAPI_Assert(false && "Not implemented");
+    }
+
+    virtual void run(GIslandExecutable::IInput &in,
+                     GIslandExecutable::IOutput &out) override;
+
+    virtual bool allocatesOutputs() const override { return true; }
+    // Return an empty RMat since we will reuse the input.
+    // There is no need to allocate and copy 4k image here.
+    virtual cv::RMat allocate(const cv::GMatDesc&) const override { return {}; }
+
+    virtual bool canReshape() const override { return true; }
+    virtual void reshape(ade::Graph&, const cv::GCompileArgs&) override {
+        // Do nothing here
+    }
+
+public:
+    GStreamingIntrinExecutable(const ade::Graph                   &,
+                               const std::vector<ade::NodeHandle> &);
+
+    const ade::Graph& m_g;
+    cv::gimpl::GModel::ConstGraph m_gm;
+    cv::gapi::streaming::IActor::Ptr m_actor;
+};
+
+void GStreamingIntrinExecutable::run(GIslandExecutable::IInput  &in,
+                                     GIslandExecutable::IOutput &out)
+{
+    m_actor->run(in, out);
+}
+
+class GStreamingBackendImpl final: public cv::gapi::GBackend::Priv
+{
+    virtual void unpackKernel(ade::Graph            &graph,
+                              const ade::NodeHandle &op_node,
+                              const cv::GKernelImpl &impl) override
+    {
+        StreamingGraph gm(graph);
+        const auto &kimpl  = cv::util::any_cast<cv::gapi::streaming::GStreamingKernel>(impl.opaque);
+        gm.metadata(op_node).set(StreamingCreateFunction{kimpl.createActorFunction});
+    }
+
+    virtual EPtr compile(const ade::Graph &graph,
+                         const cv::GCompileArgs &,
+                         const std::vector<ade::NodeHandle> &nodes) const override
+    {
+        return EPtr{new GStreamingIntrinExecutable(graph, nodes)};
+    }
+
+    virtual bool controlsMerge() const override
+    {
+        return true;
+    }
+
+    virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &,
+                             const ade::NodeHandle &,
+                             const ade::NodeHandle &,
+                             const ade::NodeHandle &) const override
+    {
+        return false;
+    }
+};
+
+GStreamingIntrinExecutable::GStreamingIntrinExecutable(const ade::Graph& g,
+                                                       const std::vector<ade::NodeHandle>& nodes)
+    : m_g(g), m_gm(m_g)
+{
+    using namespace cv::gimpl;
+    const auto is_op = [this](const ade::NodeHandle &nh)
+    {
+        return m_gm.metadata(nh).get<NodeType>().t == NodeType::OP;
+    };
+
+    auto it = std::find_if(nodes.begin(), nodes.end(), is_op);
+    GAPI_Assert(it != nodes.end() && "No operators found for this island?!");
+
+    ConstStreamingGraph cag(m_g);
+    m_actor = cag.metadata(*it).get<StreamingCreateFunction>().createActorFunction();
+
+    // Ensure this the only op in the graph
+    if (std::any_of(it+1, nodes.end(), is_op))
+    {
+        cv::util::throw_error
+            (std::logic_error
+             ("Internal error: Streaming subgraph has multiple operations"));
+    }
+}
+
+} // anonymous namespace
+
+cv::gapi::GBackend cv::gapi::streaming::backend()
+{
+    static cv::gapi::GBackend this_backend(std::make_shared<GStreamingBackendImpl>());
+    return this_backend;
+}
+
+cv::gapi::GKernelPackage cv::gapi::streaming::kernels()
+{
+    return cv::gapi::kernels<cv::gimpl::Copy, cv::gimpl::BGR>();
+}
+
+void cv::gimpl::Copy::Actor::run(cv::gimpl::GIslandExecutable::IInput  &in,
+                                 cv::gimpl::GIslandExecutable::IOutput &out)
+{
+    while (true)
+    {
+        const auto in_msg = in.get();
+        if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg))
+        {
+            out.post(cv::gimpl::EndOfStream{});
+            return;
+        }
+
+        const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg);
+        GAPI_Assert(in_args.size() == 1u);
+
+        cv::GRunArgP out_arg = out.get(0);
+        *cv::util::get<cv::MediaFrame*>(out_arg) = cv::util::get<cv::MediaFrame>(in_args[0]);
+        out.post(std::move(out_arg));
+    }
+}
+
+void cv::gimpl::BGR::Actor::run(cv::gimpl::GIslandExecutable::IInput  &in,
+                                cv::gimpl::GIslandExecutable::IOutput &out)
+{
+    while (true)
+    {
+        const auto in_msg = in.get();
+        if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg))
+        {
+            out.post(cv::gimpl::EndOfStream{});
+            return;
+        }
+
+        const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg);
+        GAPI_Assert(in_args.size() == 1u);
+
+        cv::GRunArgP out_arg = out.get(0);
+        auto frame = cv::util::get<cv::MediaFrame>(in_args[0]);
+        const auto& desc = frame.desc();
+
+        auto& rmat = *cv::util::get<cv::RMat*>(out_arg);
+        switch (desc.fmt)
+        {
+            case cv::MediaFormat::BGR:
+                rmat = cv::make_rmat<cv::gimpl::RMatMediaBGRAdapter>(frame);
+                break;
+            case cv::MediaFormat::NV12:
+            {
+                cv::Mat bgr;
+                auto view = frame.access(cv::MediaFrame::Access::R);
+                cv::Mat y_plane (desc.size,     CV_8UC1, view.ptr[0]);
+                cv::Mat uv_plane(desc.size / 2, CV_8UC2, view.ptr[1]);
+                cv::cvtColorTwoPlane(y_plane, uv_plane, bgr, cv::COLOR_YUV2BGR_NV12);
+                rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(bgr);
+                break;
+            }
+            default:
+                cv::util::throw_error(
+                        std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::BGR"));
+        }
+        out.post(std::move(out_arg));
+    }
+}

modules/gapi/src/backends/streaming/gstreamingbackend.hpp

@@ -0,0 +1,89 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+#ifndef OPENCV_GAPI_GSTREAMINGBACKEND_HPP
+#define OPENCV_GAPI_GSTREAMINGBACKEND_HPP
+
+#include <opencv2/gapi/gkernel.hpp>
+#include <opencv2/gapi/streaming/format.hpp>
+#include "gstreamingkernel.hpp"
+
+namespace cv {
+namespace gimpl {
+
+struct RMatMediaBGRAdapter final: public cv::RMat::Adapter
+{
+    RMatMediaBGRAdapter(cv::MediaFrame frame) : m_frame(frame) { };
+
+    virtual cv::RMat::View access(cv::RMat::Access a) override
+    {
+        auto view = m_frame.access(a == cv::RMat::Access::W ? cv::MediaFrame::Access::W
+                                                            : cv::MediaFrame::Access::R);
+        auto ptr = reinterpret_cast<uchar*>(view.ptr[0]);
+        auto stride = view.stride[0];
+
+        std::shared_ptr<cv::MediaFrame::View> view_ptr =
+            std::make_shared<cv::MediaFrame::View>(std::move(view));
+        auto callback = [view_ptr]() mutable { view_ptr.reset(); };
+
+        return cv::RMat::View(desc(), ptr, stride, callback);
+    }
+
+    virtual cv::GMatDesc desc() const override
+    {
+        const auto& desc = m_frame.desc();
+        GAPI_Assert(desc.fmt == cv::MediaFormat::BGR);
+        return cv::GMatDesc{CV_8U, 3, desc.size};
+    }
+
+    cv::MediaFrame m_frame;
+};
+
+struct Copy: public cv::detail::KernelTag
+{
+    using API = cv::gapi::streaming::GCopy;
+
+    static gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
+
+    class Actor final: public cv::gapi::streaming::IActor
+    {
+        public:
+            explicit Actor() {}
+            virtual void run(cv::gimpl::GIslandExecutable::IInput  &in,
+                             cv::gimpl::GIslandExecutable::IOutput &out) override;
+    };
+
+    static cv::gapi::streaming::IActor::Ptr create()
+    {
+        return cv::gapi::streaming::IActor::Ptr(new Actor());
+    }
+
+    static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
+};
+
+struct BGR: public cv::detail::KernelTag
+{
+    using API = cv::gapi::streaming::GBGR;
+    static gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
+
+    class Actor final: public cv::gapi::streaming::IActor {
+        public:
+            explicit Actor() {}
+            virtual void run(cv::gimpl::GIslandExecutable::IInput &in,
+                             cv::gimpl::GIslandExecutable::IOutput&out) override;
+    };
+
+    static cv::gapi::streaming::IActor::Ptr create()
+    {
+        return cv::gapi::streaming::IActor::Ptr(new Actor());
+    }
+    static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
+};
+
+} // namespace gimpl
+} // namespace cv
+
+#endif // OPENCV_GAPI_GSTREAMINGBACKEND_HPP

modules/gapi/src/backends/streaming/gstreamingkernel.hpp

@@ -0,0 +1,37 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2020 Intel Corporation
+
+
+#ifndef OPENCV_GAPI_GSTREAMINGKERNEL_HPP
+#define OPENCV_GAPI_GSTREAMINGKERNEL_HPP
+
+#include "compiler/gislandmodel.hpp"
+
+namespace cv {
+namespace gapi {
+namespace streaming {
+
+class IActor {
+public:
+    using Ptr = std::shared_ptr<IActor>;
+
+    virtual void run(cv::gimpl::GIslandExecutable::IInput  &in,
+                     cv::gimpl::GIslandExecutable::IOutput &out) = 0;
+
+    virtual ~IActor() = default;
+};
+
+using CreateActorFunction = std::function<IActor::Ptr()>;
+struct GStreamingKernel
+{
+    CreateActorFunction createActorFunction;
+};
+
+} // namespace streaming
+} // namespace gapi
+} // namespace cv
+
+#endif // OPENCV_GAPI_GSTREAMINGKERNEL_HPP

modules/gapi/src/compiler/gcompiler.cpp

@@ -43,6 +43,7 @@
 #include <opencv2/gapi/cpu/imgproc.hpp> // ...Imgproc
 #include <opencv2/gapi/cpu/video.hpp>   // ...and Video kernel implementations
 #include <opencv2/gapi/render/render.hpp>   // render::ocv::backend()
+#include <opencv2/gapi/streaming/format.hpp> // streaming::kernels()
 #endif // !defined(GAPI_STANDALONE)
 // </FIXME:>
 
@@ -72,7 +73,8 @@ namespace
             combine(cv::gapi::core::cpu::kernels(),
                     cv::gapi::imgproc::cpu::kernels(),
                     cv::gapi::video::cpu::kernels(),
-                    cv::gapi::render::ocv::kernels());
+                    cv::gapi::render::ocv::kernels(),
+                    cv::gapi::streaming::kernels());
 #else
             cv::gapi::GKernelPackage();
 #endif // !defined(GAPI_STANDALONE)

modules/gapi/src/compiler/gstreaming_priv.hpp

@@ -20,7 +20,7 @@ namespace gimpl
 
 // FIXME: GAPI_EXPORTS is here only due to tests and Windows linker issues
 // FIXME: It seems it clearly duplicates the GStreamingCompiled and
-// GStreamingExecutable APIs so is highly redundant now.
+// GStreamingIntrinExecutable APIs so is highly redundant now.
 // Same applies to GCompiled/GCompiled::Priv/GExecutor.
 class GAPI_EXPORTS GStreamingCompiled::Priv
 {

modules/gapi/src/compiler/passes/intrin.cpp

@@ -201,7 +201,7 @@ void traceDown(cv::gimpl::GModel::Graph &g,
 
 // Streaming case: ensure the graph has proper isolation of the
 // desynchronized parts, set proper Edge metadata hints for
-// GStreamingExecutable
+// GStreamingIntrinExecutable
 void apply(cv::gimpl::GModel::Graph &g) {
     using namespace cv::gimpl;
 

modules/gapi/src/compiler/passes/streaming.cpp

@@ -32,7 +32,7 @@ namespace cv { namespace gimpl { namespace passes {
  * connected to a new "Sink" node which becomes its _consumer_.
  *
  * These extra nodes are required to streamline the queues
- * initialization by the GStreamingExecutable and its derivatives.
+ * initialization by the GStreamingIntrinExecutable and its derivatives.
  */
 void addStreaming(ade::passes::PassContext &ctx)
 {

modules/gapi/src/executor/gstreamingexecutor.cpp

@@ -144,6 +144,9 @@ void sync_data(cv::GRunArgs &results, cv::GRunArgsP &outputs)
         case T::index_of<cv::detail::OpaqueRef>():
             cv::util::get<cv::detail::OpaqueRef>(out_obj).mov(cv::util::get<cv::detail::OpaqueRef>(res_obj));
             break;
+        case T::index_of<cv::MediaFrame*>():
+            *cv::util::get<cv::MediaFrame*>(out_obj) = std::move(cv::util::get<cv::MediaFrame>(res_obj));
+            break;
         default:
             GAPI_Assert(false && "This value type is not supported!"); // ...maybe because of STANDALONE mode.
             break;
@@ -636,6 +639,13 @@ class StreamingOutput final: public cv::gimpl::GIslandExecutable::IOutput
                 ret_val = cv::GRunArgP(rr);
             }
             break;
+        case cv::GShape::GFRAME:
+            {
+                cv::MediaFrame frame;
+                out_arg = cv::GRunArg(std::move(frame));
+                ret_val = cv::GRunArgP(&cv::util::get<cv::MediaFrame>(out_arg));
+            }
+            break;
         default:
             cv::util::throw_error(std::logic_error("Unsupported GShape"));
         }