Merge pull request #19516 from dbudniko:dbudniko/gapi_render_frame_cpu

Render Frame NV12 on CPU

* render frame on CPU

* doxygen fix

* address review from Alexey

* comment fixes

* Address comments from Ruslan

* remove NV12 specific

* mat clone and more renaming

* misprint fix

modules/gapi/src/api/render.cpp

@@ -33,6 +33,20 @@ void cv::gapi::wip::draw::render(cv::Mat& y_plane,
                cv::gout(y_plane, uv_plane), std::move(args));
 }
 
+void cv::gapi::wip::draw::render(cv::MediaFrame& frame,
+                                 const Prims& prims,
+                                 cv::GCompileArgs&& args)
+{
+    cv::GFrame in, out;
+    cv::GArray<cv::gapi::wip::draw::Prim> arr;
+    out = cv::gapi::wip::draw::renderFrame(in, arr);
+
+    cv::GComputation comp(cv::GIn(in, arr), cv::GOut(out));
+    comp.apply(cv::gin(frame, prims),
+               cv::gout(frame), std::move(args));
+}
+
+
 void cv::gapi::wip::draw::cvtYUVToNV12(const cv::Mat& yuv,
                                        cv::Mat& y,
                                        cv::Mat& uv)
@@ -69,3 +83,9 @@ cv::gapi::wip::draw::renderNV12(const cv::GMat& y,
 {
     return cv::gapi::wip::draw::GRenderNV12::on(y, uv, prims);
 }
+
+cv::GFrame cv::gapi::wip::draw::renderFrame(const cv::GFrame& frame,
+                                                const cv::GArray<cv::gapi::wip::draw::Prim>& prims)
+{
+    return cv::gapi::wip::draw::GRenderFrame::on(frame, prims);
+}

modules/gapi/src/backends/render/grenderocv.cpp

@@ -114,8 +114,83 @@ GAPI_OCV_KERNEL_ST(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12, RenderOC
     }
 };
 
+GAPI_OCV_KERNEL_ST(RenderFrameOCVImpl, cv::gapi::wip::draw::GRenderFrame, RenderOCVState)
+{
+    static void run(const cv::MediaFrame & in,
+                    const cv::gapi::wip::draw::Prims & prims,
+                    cv::MediaFrame & out,
+                    RenderOCVState & state)
+    {
+        GAPI_Assert(in.desc().fmt == cv::MediaFormat::NV12);
+
+        // FIXME: consider a better approach (aka native inplace operation)
+        // Non-intuitive logic with shared_ptr Priv class
+        out = in;
+
+        auto desc = out.desc();
+        auto w_out = out.access(cv::MediaFrame::Access::W);
+
+        auto out_y = cv::Mat(desc.size, CV_8UC1, w_out.ptr[0], w_out.stride[0]);
+        auto out_uv = cv::Mat(desc.size / 2, CV_8UC2, w_out.ptr[1], w_out.stride[1]);
+
+        auto r_in = in.access(cv::MediaFrame::Access::R);
+
+        auto in_y = cv::Mat(desc.size, CV_8UC1, r_in.ptr[0], r_in.stride[0]);
+        auto in_uv = cv::Mat(desc.size / 2, CV_8UC2, r_in.ptr[1], r_in.stride[1]);
+
+        /* FIXME How to render correctly on NV12 format ?
+         *
+         * Rendering on NV12 via OpenCV looks like this:
+         *
+         * y --------> 1)(NV12 -> YUV) -> yuv -> 2)draw -> yuv -> 3)split -------> out_y
+         *                  ^                                         |
+         *                  |                                         |
+         * uv --------------                                          `----------> out_uv
+         *
+         *
+         * 1) Collect yuv mat from two planes, uv plain in two times less than y plane
+         *    so, upsample uv in two times, with bilinear interpolation
+         *
+         * 2) Render primitives on YUV
+         *
+         * 3) Convert yuv to NV12 (using bilinear interpolation)
+         *
+         */
+
+         // NV12 -> YUV
+        cv::Mat upsample_uv, yuv;
+        cv::resize(in_uv, upsample_uv, in_uv.size() * 2, cv::INTER_LINEAR);
+        cv::merge(std::vector<cv::Mat>{in_y, upsample_uv}, yuv);
+
+        cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, state.ftpr);
+
+        // YUV -> NV12
+        cv::Mat out_u, out_v, uv_plane;
+        std::vector<cv::Mat> chs = { out_y, out_u, out_v };
+        cv::split(yuv, chs);
+        cv::merge(std::vector<cv::Mat>{chs[1], chs[2]}, uv_plane);
+        cv::resize(uv_plane, out_uv, uv_plane.size() / 2, cv::INTER_LINEAR);
+    }
+
+    static void setup(const cv::GFrameDesc&   /* in_nv12  */,
+        const cv::GArrayDesc& /* prims */,
+        std::shared_ptr<RenderOCVState>&state,
+        const cv::GCompileArgs & args)
+    {
+        using namespace cv::gapi::wip::draw;
+        auto has_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
+        state = std::make_shared<RenderOCVState>();
+
+        if (has_freetype_font)
+        {
+            state->ftpr = std::make_shared<FTTextRender>(has_freetype_font->path);
+        }
+    }
+};
+
+
 cv::gapi::GKernelPackage cv::gapi::render::ocv::kernels()
 {
-    const static auto pkg = cv::gapi::kernels<RenderBGROCVImpl, RenderNV12OCVImpl>();
+    const static auto pkg = cv::gapi::kernels<RenderBGROCVImpl, RenderNV12OCVImpl, RenderFrameOCVImpl>();
     return pkg;
 }