Merge pull request #19002 from TolyaTalamanov:at/infer_gframe

[G-API] Support GFrame for infer

* GInfer(GFrame), currently broken

* Fixed (API only)

* Support GFrame in GIEBackend

* Fix comments to review

* Fix comments to review

* Fix doxygen

* Fix building with different IE versions

* Fix warning on MacOS

Co-authored-by: Dmitry Matveev <dmitry.matveev@intel.com>
Co-authored-by: Smirnov Alexey <alexey.smirnov@intel.com>

modules/gapi/test/infer/gapi_infer_ie_test.cpp

@@ -23,6 +23,45 @@ namespace opencv_test
 {
 namespace {
 
+class TestMediaBGR final: public cv::MediaFrame::IAdapter {
+    cv::Mat m_mat;
+    using Cb = cv::MediaFrame::View::Callback;
+    Cb m_cb;
+
+public:
+    explicit TestMediaBGR(cv::Mat m, Cb cb = [](){})
+        : m_mat(m), m_cb(cb) {
+    }
+    cv::GFrameDesc meta() const override {
+        return cv::GFrameDesc{cv::MediaFormat::BGR, cv::Size(m_mat.cols, m_mat.rows)};
+    }
+    cv::MediaFrame::View access(cv::MediaFrame::Access) override {
+        cv::MediaFrame::View::Ptrs pp = { m_mat.ptr(), nullptr, nullptr, nullptr };
+        cv::MediaFrame::View::Strides ss = { m_mat.step, 0u, 0u, 0u };
+        return cv::MediaFrame::View(std::move(pp), std::move(ss), Cb{m_cb});
+    }
+};
+
+class TestMediaNV12 final: public cv::MediaFrame::IAdapter {
+    cv::Mat m_y;
+    cv::Mat m_uv;
+public:
+    TestMediaNV12(cv::Mat y, cv::Mat uv) : m_y(y), m_uv(uv) {
+    }
+    cv::GFrameDesc meta() const override {
+        return cv::GFrameDesc{cv::MediaFormat::NV12, cv::Size(m_y.cols, m_y.rows)};
+    }
+    cv::MediaFrame::View access(cv::MediaFrame::Access) override {
+        cv::MediaFrame::View::Ptrs pp = {
+            m_y.ptr(), m_uv.ptr(), nullptr, nullptr
+        };
+        cv::MediaFrame::View::Strides ss = {
+            m_y.step, m_uv.step, 0u, 0u
+        };
+        return cv::MediaFrame::View(std::move(pp), std::move(ss));
+    }
+};
+
 // FIXME: taken from DNN module
 static void initDLDTDataPath()
 {
@@ -64,11 +103,15 @@ void normAssert(cv::InputArray ref, cv::InputArray test,
 
 namespace IE = InferenceEngine;
 
-void setNetParameters(IE::CNNNetwork& net) {
-    auto &ii = net.getInputsInfo().at("data");
+void setNetParameters(IE::CNNNetwork& net, bool is_nv12 = false) {
+    auto ii = net.getInputsInfo().at("data");
     ii->setPrecision(IE::Precision::U8);
     ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
+    if (is_nv12) {
+        ii->getPreProcess().setColorFormat(IE::ColorFormat::NV12);
+    }
 }
+
 } // anonymous namespace
 
 // TODO: Probably DNN/IE part can be further parametrized with a template
@@ -246,6 +289,81 @@ struct ROIList: public ::testing::Test {
     }
 }; // ROIList
 
+struct ROIListNV12: public ::testing::Test {
+    cv::gapi::ie::detail::ParamDesc params;
+
+    cv::Mat m_in_uv;
+    cv::Mat m_in_y;
+    std::vector<cv::Rect> m_roi_list;
+
+    std::vector<cv::Mat> m_out_ie_ages;
+    std::vector<cv::Mat> m_out_ie_genders;
+
+    std::vector<cv::Mat> m_out_gapi_ages;
+    std::vector<cv::Mat> m_out_gapi_genders;
+
+    using AGInfo = std::tuple<cv::GMat, cv::GMat>;
+    G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
+
+    void SetUp() {
+        initDLDTDataPath();
+        params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
+        params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
+        params.device_id = "CPU";
+
+        cv::Size sz{320, 240};
+        m_in_y = cv::Mat{sz, CV_8UC1};
+        cv::randu(m_in_y, 0, 255);
+        m_in_uv = cv::Mat{sz / 2, CV_8UC2};
+        cv::randu(m_in_uv, 0, 255);
+
+        // both ROIs point to the same face, with a slightly changed geometry
+        m_roi_list = {
+            cv::Rect(cv::Point{64, 60}, cv::Size{ 96,  96}),
+            cv::Rect(cv::Point{50, 32}, cv::Size{128, 160}),
+        };
+
+        // Load & run IE network
+        {
+            auto plugin        = cv::gimpl::ie::wrap::getPlugin(params);
+            auto net           = cv::gimpl::ie::wrap::readNetwork(params);
+            setNetParameters(net, true);
+            auto this_network  = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+            auto infer_request = this_network.CreateInferRequest();
+            auto frame_blob = cv::gapi::ie::util::to_ie(m_in_y, m_in_uv);
+
+            for (auto &&rc : m_roi_list) {
+                const auto ie_rc = IE::ROI {
+                    0u
+                        , static_cast<std::size_t>(rc.x)
+                        , static_cast<std::size_t>(rc.y)
+                        , static_cast<std::size_t>(rc.width)
+                        , static_cast<std::size_t>(rc.height)
+                };
+                infer_request.SetBlob("data", IE::make_shared_blob(frame_blob, ie_rc));
+                infer_request.Infer();
+
+                using namespace cv::gapi::ie::util;
+                m_out_ie_ages.push_back(to_ocv(infer_request.GetBlob("age_conv3")).clone());
+                m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone());
+            }
+        } // namespace IE = ..
+    } // ROIList()
+
+    void validate() {
+        // Validate with IE itself (avoid DNN module dependency here)
+        ASSERT_EQ(2u, m_out_ie_ages.size());
+        ASSERT_EQ(2u, m_out_ie_genders.size());
+        ASSERT_EQ(2u, m_out_gapi_ages.size());
+        ASSERT_EQ(2u, m_out_gapi_genders.size());
+
+        normAssert(m_out_ie_ages   [0], m_out_gapi_ages   [0], "0: Test age output");
+        normAssert(m_out_ie_genders[0], m_out_gapi_genders[0], "0: Test gender output");
+        normAssert(m_out_ie_ages   [1], m_out_gapi_ages   [1], "1: Test age output");
+        normAssert(m_out_ie_genders[1], m_out_gapi_genders[1], "1: Test gender output");
+    }
+};
+
 TEST_F(ROIList, TestInfer)
 {
     cv::GArray<cv::Rect> rr;
@@ -505,6 +623,320 @@ TEST(TestAgeGenderIE, CPUConfig)
                     cv::compile_args(cv::gapi::networks(pp))));
 }
 
+TEST_F(ROIList, MediaInputBGR)
+{
+    initDLDTDataPath();
+
+    cv::GFrame in;
+    cv::GArray<cv::Rect> rr;
+    cv::GArray<cv::GMat> age, gender;
+    std::tie(age, gender) = cv::gapi::infer<AgeGender>(rr, in);
+    cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
+
+    auto frame = MediaFrame::Create<TestMediaBGR>(m_in_mat);
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" });
+    comp.apply(cv::gin(frame, m_roi_list),
+               cv::gout(m_out_gapi_ages, m_out_gapi_genders),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+    validate();
+}
+
+TEST_F(ROIListNV12, MediaInputNV12)
+{
+    initDLDTDataPath();
+
+    cv::GFrame in;
+    cv::GArray<cv::Rect> rr;
+    cv::GArray<cv::GMat> age, gender;
+    std::tie(age, gender) = cv::gapi::infer<AgeGender>(rr, in);
+    cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
+
+    auto frame = MediaFrame::Create<TestMediaNV12>(m_in_y, m_in_uv);
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" });
+    comp.apply(cv::gin(frame, m_roi_list),
+               cv::gout(m_out_gapi_ages, m_out_gapi_genders),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+    validate();
+}
+
+TEST(TestAgeGenderIE, MediaInputNV12)
+{
+    initDLDTDataPath();
+
+    cv::gapi::ie::detail::ParamDesc params;
+    params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
+    params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
+    params.device_id = "CPU";
+
+    cv::Size sz{320, 240};
+    cv::Mat in_y_mat(sz, CV_8UC1);
+    cv::randu(in_y_mat, 0, 255);
+    cv::Mat in_uv_mat(sz / 2, CV_8UC2);
+    cv::randu(in_uv_mat, 0, 255);
+
+    cv::Mat gapi_age, gapi_gender;
+
+    // Load & run IE network
+    IE::Blob::Ptr ie_age, ie_gender;
+    {
+        auto plugin        = cv::gimpl::ie::wrap::getPlugin(params);
+        auto net           = cv::gimpl::ie::wrap::readNetwork(params);
+        setNetParameters(net, true);
+        auto this_network  = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+        auto infer_request = this_network.CreateInferRequest();
+        infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in_y_mat, in_uv_mat));
+        infer_request.Infer();
+        ie_age    = infer_request.GetBlob("age_conv3");
+        ie_gender = infer_request.GetBlob("prob");
+    }
+
+    // Configure & run G-API
+    using AGInfo = std::tuple<cv::GMat, cv::GMat>;
+    G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
+
+    cv::GFrame in;
+    cv::GMat age, gender;
+    std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
+    cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
+
+    auto frame = MediaFrame::Create<TestMediaNV12>(in_y_mat, in_uv_mat);
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" });
+    comp.apply(cv::gin(frame), cv::gout(gapi_age, gapi_gender),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+
+    // Validate with IE itself (avoid DNN module dependency here)
+    normAssert(cv::gapi::ie::util::to_ocv(ie_age),    gapi_age,    "Test age output"   );
+    normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output");
+}
+
+TEST(TestAgeGenderIE, MediaInputBGR)
+{
+    initDLDTDataPath();
+
+    cv::gapi::ie::detail::ParamDesc params;
+    params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
+    params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
+    params.device_id = "CPU";
+
+    cv::Size sz{320, 240};
+    cv::Mat in_mat(sz, CV_8UC3);
+    cv::randu(in_mat, 0, 255);
+
+    cv::Mat gapi_age, gapi_gender;
+
+    // Load & run IE network
+    IE::Blob::Ptr ie_age, ie_gender;
+    {
+        auto plugin        = cv::gimpl::ie::wrap::getPlugin(params);
+        auto net           = cv::gimpl::ie::wrap::readNetwork(params);
+        setNetParameters(net);
+        auto this_network  = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+        auto infer_request = this_network.CreateInferRequest();
+        infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in_mat));
+        infer_request.Infer();
+        ie_age    = infer_request.GetBlob("age_conv3");
+        ie_gender = infer_request.GetBlob("prob");
+    }
+
+    // Configure & run G-API
+    using AGInfo = std::tuple<cv::GMat, cv::GMat>;
+    G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
+
+    cv::GFrame in;
+    cv::GMat age, gender;
+    std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
+    cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
+
+    auto frame = MediaFrame::Create<TestMediaBGR>(in_mat);
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" });
+    comp.apply(cv::gin(frame), cv::gout(gapi_age, gapi_gender),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+
+    // Validate with IE itself (avoid DNN module dependency here)
+    normAssert(cv::gapi::ie::util::to_ocv(ie_age),    gapi_age,    "Test age output"   );
+    normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output");
+}
+
+TEST(InferROI, MediaInputBGR)
+{
+    initDLDTDataPath();
+
+    cv::gapi::ie::detail::ParamDesc params;
+    params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
+    params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
+    params.device_id = "CPU";
+
+    cv::Size sz{320, 240};
+    cv::Mat in_mat(sz, CV_8UC3);
+    cv::randu(in_mat, 0, 255);
+
+    cv::Mat gapi_age, gapi_gender;
+    cv::Rect rect(cv::Point{64, 60}, cv::Size{96, 96});
+
+    // Load & run IE network
+    IE::Blob::Ptr ie_age, ie_gender;
+    {
+        auto plugin        = cv::gimpl::ie::wrap::getPlugin(params);
+        auto net           = cv::gimpl::ie::wrap::readNetwork(params);
+        setNetParameters(net);
+        auto this_network  = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+        auto infer_request = this_network.CreateInferRequest();
+        const auto ie_rc = IE::ROI {
+            0u
+            , static_cast<std::size_t>(rect.x)
+            , static_cast<std::size_t>(rect.y)
+            , static_cast<std::size_t>(rect.width)
+            , static_cast<std::size_t>(rect.height)
+        };
+        IE::Blob::Ptr roi_blob = IE::make_shared_blob(cv::gapi::ie::util::to_ie(in_mat), ie_rc);
+        infer_request.SetBlob("data", roi_blob);
+        infer_request.Infer();
+        ie_age    = infer_request.GetBlob("age_conv3");
+        ie_gender = infer_request.GetBlob("prob");
+    }
+
+    // Configure & run G-API
+    using AGInfo = std::tuple<cv::GMat, cv::GMat>;
+    G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
+
+    cv::GFrame in;
+    cv::GOpaque<cv::Rect> roi;
+    cv::GMat age, gender;
+    std::tie(age, gender) = cv::gapi::infer<AgeGender>(roi, in);
+    cv::GComputation comp(cv::GIn(in, roi), cv::GOut(age, gender));
+
+    auto frame = MediaFrame::Create<TestMediaBGR>(in_mat);
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" });
+    comp.apply(cv::gin(frame, rect), cv::gout(gapi_age, gapi_gender),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+
+    // Validate with IE itself (avoid DNN module dependency here)
+    normAssert(cv::gapi::ie::util::to_ocv(ie_age),    gapi_age,    "Test age output"   );
+    normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output");
+}
+
+TEST(InferROI, MediaInputNV12)
+{
+    initDLDTDataPath();
+
+    cv::gapi::ie::detail::ParamDesc params;
+    params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
+    params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
+    params.device_id = "CPU";
+
+    cv::Size sz{320, 240};
+    auto in_y_mat = cv::Mat{sz, CV_8UC1};
+    cv::randu(in_y_mat, 0, 255);
+    auto in_uv_mat = cv::Mat{sz / 2, CV_8UC2};
+    cv::randu(in_uv_mat, 0, 255);
+
+    cv::Mat gapi_age, gapi_gender;
+    cv::Rect rect(cv::Point{64, 60}, cv::Size{96, 96});
+
+    // Load & run IE network
+    IE::Blob::Ptr ie_age, ie_gender;
+    {
+        auto plugin        = cv::gimpl::ie::wrap::getPlugin(params);
+        auto net           = cv::gimpl::ie::wrap::readNetwork(params);
+        setNetParameters(net, true);
+        auto this_network  = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+        auto infer_request = this_network.CreateInferRequest();
+        const auto ie_rc = IE::ROI {
+            0u
+            , static_cast<std::size_t>(rect.x)
+            , static_cast<std::size_t>(rect.y)
+            , static_cast<std::size_t>(rect.width)
+            , static_cast<std::size_t>(rect.height)
+        };
+        IE::Blob::Ptr roi_blob = IE::make_shared_blob(cv::gapi::ie::util::to_ie(in_y_mat, in_uv_mat), ie_rc);
+        infer_request.SetBlob("data", roi_blob);
+        infer_request.Infer();
+        ie_age    = infer_request.GetBlob("age_conv3");
+        ie_gender = infer_request.GetBlob("prob");
+    }
+
+    // Configure & run G-API
+    using AGInfo = std::tuple<cv::GMat, cv::GMat>;
+    G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
+
+    cv::GFrame in;
+    cv::GOpaque<cv::Rect> roi;
+    cv::GMat age, gender;
+    std::tie(age, gender) = cv::gapi::infer<AgeGender>(roi, in);
+    cv::GComputation comp(cv::GIn(in, roi), cv::GOut(age, gender));
+
+    auto frame = MediaFrame::Create<TestMediaNV12>(in_y_mat, in_uv_mat);
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" });
+    comp.apply(cv::gin(frame, rect), cv::gout(gapi_age, gapi_gender),
+               cv::compile_args(cv::gapi::networks(pp)));
+
+
+    // Validate with IE itself (avoid DNN module dependency here)
+    normAssert(cv::gapi::ie::util::to_ocv(ie_age),    gapi_age,    "Test age output"   );
+    normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output");
+}
+
+TEST_F(ROIList, Infer2MediaInputBGR)
+{
+    cv::GArray<cv::Rect> rr;
+    cv::GFrame in;
+    cv::GArray<cv::GMat> age, gender;
+    std::tie(age, gender) = cv::gapi::infer2<AgeGender>(in, rr);
+    cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
+
+    auto frame = MediaFrame::Create<TestMediaBGR>(m_in_mat);
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" });
+    comp.apply(cv::gin(frame, m_roi_list),
+               cv::gout(m_out_gapi_ages, m_out_gapi_genders),
+               cv::compile_args(cv::gapi::networks(pp)));
+    validate();
+}
+
+TEST_F(ROIListNV12, Infer2MediaInputNV12)
+{
+    cv::GArray<cv::Rect> rr;
+    cv::GFrame in;
+    cv::GArray<cv::GMat> age, gender;
+    std::tie(age, gender) = cv::gapi::infer2<AgeGender>(in, rr);
+    cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
+
+    auto frame = MediaFrame::Create<TestMediaNV12>(m_in_y, m_in_uv);
+
+    auto pp = cv::gapi::ie::Params<AgeGender> {
+        params.model_path, params.weights_path, params.device_id
+    }.cfgOutputLayers({ "age_conv3", "prob" });
+    comp.apply(cv::gin(frame, m_roi_list),
+               cv::gout(m_out_gapi_ages, m_out_gapi_genders),
+               cv::compile_args(cv::gapi::networks(pp)));
+    validate();
+}
+
 } // namespace opencv_test
 
 #endif //  HAVE_INF_ENGINE