potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 15:53:03 +04:00
Code Packages Releases Wiki Activity

Merge pull request #19002 from TolyaTalamanov:at/infer_gframe

Browse Source 
[G-API] Support GFrame for infer

* GInfer(GFrame), currently broken

* Fixed (API only)

* Support GFrame in GIEBackend

* Fix comments to review

* Fix comments to review

* Fix doxygen

* Fix building with different IE versions

* Fix warning on MacOS

Co-authored-by: Dmitry Matveev <dmitry.matveev@intel.com>
Co-authored-by: Smirnov Alexey <alexey.smirnov@intel.com>
This commit is contained in:
Anatoliy Talamanov
2020-12-09 17:00:56 +03:00
committed by GitHub
parent b5a9ef6b7b
commit a55150b1bc
4 changed files with 670 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/gapi/src/backends/ie/giebackend.cpp
+135 -34
View File
@@ -36,6 +36,7 @@
#include <opencv2/gapi/gtype_traits.hpp>
#include <opencv2/gapi/infer.hpp>
#include <opencv2/gapi/own/convert.hpp>
#include <opencv2/gapi/gframe.hpp>
#include "compiler/gobjref.hpp"
#include "compiler/gmodel.hpp"
@@ -45,6 +46,10 @@
#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK!
#if INF_ENGINE_RELEASE < 2021010000
#include "ie_compound_blob.h"
#endif
namespace IE = InferenceEngine;
namespace {
@@ -151,6 +156,25 @@ inline IE::Blob::Ptr wrapIE(const cv::Mat &mat, cv::gapi::ie::TraitAs hint) {
return IE::Blob::Ptr{};
}
inline IE::Blob::Ptr wrapIE(const cv::MediaFrame::View& view,
const cv::GFrameDesc& desc) {
switch (desc.fmt) {
case cv::MediaFormat::BGR: {
auto bgr = cv::Mat(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
return wrapIE(bgr, cv::gapi::ie::TraitAs::IMAGE);
}
case cv::MediaFormat::NV12: {
auto y_plane = cv::Mat(desc.size, CV_8UC1, view.ptr[0], view.stride[0]);
auto uv_plane = cv::Mat(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]);
return cv::gapi::ie::util::to_ie(y_plane, uv_plane);
}
default:
GAPI_Assert(false && "Unsupported media format for IE backend");
}
GAPI_Assert(false);
}
template<class MatType>
inline void copyFromIE(const IE::Blob::Ptr &blob, MatType &mat) {
switch (blob->getTensorDesc().getPrecision()) {
@@ -256,6 +280,7 @@ struct IECallContext
{
// Input parameters passed to an inference operation.
std::vector<cv::GArg> args;
cv::GShapes in_shapes;
//FIXME: avoid conversion of arguments from internal representation to OpenCV one on each call
//to OCV kernel. (This can be achieved by a two single time conversions in GCPUExecutable::run,
@@ -267,6 +292,10 @@ struct IECallContext
template<typename T>
const T& inArg(std::size_t input) { return args.at(input).get<T>(); }
const cv::MediaFrame& inFrame(std::size_t input) {
return inArg<cv::MediaFrame>(input);
}
// Syntax sugar
const cv::Mat& inMat(std::size_t input) {
return inArg<cv::Mat>(input);
@@ -319,6 +348,24 @@ using GConstGIEModel = ade::ConstTypedGraph
, IEUnit
, IECallable
>;
using Views = std::vector<std::unique_ptr<cv::MediaFrame::View>>;
inline IE::Blob::Ptr extractBlob(IECallContext& ctx, std::size_t i, Views& views) {
switch (ctx.in_shapes[i]) {
case cv::GShape::GFRAME: {
const auto& frame = ctx.inFrame(i);
views.emplace_back(new cv::MediaFrame::View(frame.access(cv::MediaFrame::Access::R)));
return wrapIE(*views.back(), frame.desc());
}
case cv::GShape::GMAT: {
return wrapIE(ctx.inMat(i), cv::gapi::ie::TraitAs::IMAGE);
}
default:
GAPI_Assert("Unsupported input shape for IE backend");
}
GAPI_Assert(false);
}
} // anonymous namespace
// GCPUExcecutable implementation //////////////////////////////////////////////
@@ -384,6 +431,8 @@ cv::GArg cv::gimpl::ie::GIEExecutable::packArg(const cv::GArg &arg) {
// (and constructed by either bindIn/Out or resetInternal)
case GShape::GOPAQUE: return GArg(m_res.slot<cv::detail::OpaqueRef>().at(ref.id));
case GShape::GFRAME: return GArg(m_res.slot<cv::MediaFrame>().at(ref.id));
default:
util::throw_error(std::logic_error("Unsupported GShape type"));
break;
@@ -413,6 +462,12 @@ void cv::gimpl::ie::GIEExecutable::run(std::vector<InObj> &&input_objs,
std::back_inserter(context.args),
std::bind(&GIEExecutable::packArg, this, _1));
// NB: Need to store inputs shape to recognize GFrame/GMat
ade::util::transform(op.args,
std::back_inserter(context.in_shapes),
[](const cv::GArg& arg) {
return arg.get<cv::gimpl::RcDesc>().shape;
});
// - Output parameters.
for (const auto &out_it : ade::util::indexed(op.outs)) {
// FIXME: Can the same GArg type resolution mechanism be reused here?
@@ -438,6 +493,34 @@ namespace cv {
namespace gimpl {
namespace ie {
static void configureInputInfo(const IE::InputInfo::Ptr& ii, const cv::GMetaArg mm) {
switch (mm.index()) {
case cv::GMetaArg::index_of<cv::GMatDesc>():
{
ii->setPrecision(toIE(util::get<cv::GMatDesc>(mm).depth));
break;
}
case cv::GMetaArg::index_of<cv::GFrameDesc>():
{
const auto &meta = util::get<cv::GFrameDesc>(mm);
switch (meta.fmt) {
case cv::MediaFormat::NV12:
ii->getPreProcess().setColorFormat(IE::ColorFormat::NV12);
break;
case cv::MediaFormat::BGR:
// NB: Do nothing
break;
default:
GAPI_Assert(false && "Unsupported media format for IE backend");
}
ii->setPrecision(toIE(CV_8U));
break;
}
default:
util::throw_error(std::runtime_error("Unsupported input meta for IE backend"));
}
}
struct Infer: public cv::detail::KernelTag {
using API = cv::GInferBase;
static cv::gapi::GBackend backend() { return cv::gapi::ie::backend(); }
@@ -468,11 +551,7 @@ struct Infer: public cv::detail::KernelTag {
auto &&ii = uu.inputs.at(std::get<0>(it));
const auto & mm = std::get<1>(it);
GAPI_Assert(util::holds_alternative<cv::GMatDesc>(mm)
&& "Non-GMat inputs are not supported");
const auto &meta = util::get<cv::GMatDesc>(mm);
ii->setPrecision(toIE(meta.depth));
configureInputInfo(ii, mm);
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
}
@@ -495,15 +574,12 @@ struct Infer: public cv::detail::KernelTag {
static void run(IECompiled &iec, const IEUnit &uu, IECallContext &ctx) {
// non-generic version for now:
// - assumes all inputs/outputs are always Mats
Views views;
for (auto i : ade::util::iota(uu.params.num_in)) {
// TODO: Ideally we shouldn't do SetBlob() but GetBlob() instead,
// and redirect our data producers to this memory
// (A memory dialog comes to the picture again)
const cv::Mat this_mat = ctx.inMat(i);
// FIXME: By default here we trait our inputs as images.
// May be we need to make some more intelligence here about it
IE::Blob::Ptr this_blob = wrapIE(this_mat, cv::gapi::ie::TraitAs::IMAGE);
IE::Blob::Ptr this_blob = extractBlob(ctx, i, views);
iec.this_request.SetBlob(uu.params.input_names[i], this_blob);
}
iec.this_request.Infer();
@@ -540,10 +616,10 @@ struct InferROI: public cv::detail::KernelTag {
GAPI_Assert(1u == uu.params.input_names.size());
GAPI_Assert(2u == in_metas.size());
// 0th is ROI, 1st is in0put image
auto &&ii = uu.inputs.at(uu.params.input_names.at(0));
const auto &meta = util::get<cv::GMatDesc>(in_metas.at(1));
ii->setPrecision(toIE(meta.depth));
// 0th is ROI, 1st is input image
auto &&ii = uu.inputs.at(uu.params.input_names.at(0));
auto &&mm = in_metas.at(1u);
configureInputInfo(ii, mm);
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
// FIXME: It would be nice here to have an exact number of network's
@@ -566,10 +642,12 @@ struct InferROI: public cv::detail::KernelTag {
// non-generic version for now, per the InferROI's definition
GAPI_Assert(uu.params.num_in == 1);
const auto& this_roi = ctx.inArg<cv::detail::OpaqueRef>(0).rref<cv::Rect>();
const auto this_mat = ctx.inMat(1);
IE::Blob::Ptr this_blob = wrapIE(this_mat, cv::gapi::ie::TraitAs::IMAGE);
IE::Blob::Ptr roi_blob = IE::make_shared_blob(this_blob, toIE(this_roi));
iec.this_request.SetBlob(*uu.params.input_names.begin(), roi_blob);
Views views;
IE::Blob::Ptr this_blob = extractBlob(ctx, 1, views);
iec.this_request.SetBlob(*uu.params.input_names.begin(),
IE::make_shared_blob(this_blob, toIE(this_roi)));
iec.this_request.Infer();
for (auto i : ade::util::iota(uu.params.num_out)) {
cv::Mat& out_mat = ctx.outMatR(i);
@@ -606,12 +684,7 @@ struct InferList: public cv::detail::KernelTag {
for (auto &&input_name : uu.params.input_names) {
auto &&ii = uu.inputs.at(input_name);
const auto & mm = in_metas[idx++];
GAPI_Assert(util::holds_alternative<cv::GMatDesc>(mm)
&& "Non-GMat inputs are not supported");
const auto &meta = util::get<cv::GMatDesc>(mm);
ii->setPrecision(toIE(meta.depth));
configureInputInfo(ii, mm);
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
}
@@ -630,9 +703,9 @@ struct InferList: public cv::detail::KernelTag {
GAPI_Assert(uu.params.num_in == 1); // roi list is not counted in net's inputs
const auto& in_roi_vec = ctx.inArg<cv::detail::VectorRef>(0u).rref<cv::Rect>();
const cv::Mat this_mat = ctx.inMat(1u);
// Since we do a ROI list inference, always assume our input buffer is image
IE::Blob::Ptr this_blob = wrapIE(this_mat, cv::gapi::ie::TraitAs::IMAGE);
Views views;
IE::Blob::Ptr this_blob = extractBlob(ctx, 1, views);
// FIXME: This could be done ONCE at graph compile stage!
std::vector< std::vector<int> > cached_dims(uu.params.num_out);
@@ -696,11 +769,30 @@ struct InferList2: public cv::detail::KernelTag {
// "blob"-based ones)
// FIXME: this is filtering not done, actually! GArrayDesc has
// no hint for its underlying type!
const auto &mm_0 = in_metas[0u];
const auto &meta_0 = util::get<cv::GMatDesc>(mm_0);
GAPI_Assert( !meta_0.isND()
const auto &mm_0 = in_metas[0u];
switch (in_metas[0u].index()) {
case cv::GMetaArg::index_of<cv::GMatDesc>(): {
const auto &meta_0 = util::get<cv::GMatDesc>(mm_0);
GAPI_Assert( !meta_0.isND()
&& !meta_0.planar
&& "Only images are supported as the 0th argument");
break;
}
case cv::GMetaArg::index_of<cv::GFrameDesc>(): {
// FIXME: Is there any validation for GFrame ?
break;
}
default:
util::throw_error(std::runtime_error("Unsupported input meta for IE backend"));
}
if (util::holds_alternative<cv::GMatDesc>(mm_0)) {
const auto &meta_0 = util::get<cv::GMatDesc>(mm_0);
GAPI_Assert( !meta_0.isND()
&& !meta_0.planar
&& "Only images are supported as the 0th argument");
}
std::size_t idx = 1u;
for (auto &&input_name : uu.params.input_names) {
auto &ii = uu.inputs.at(input_name);
@@ -710,7 +802,7 @@ struct InferList2: public cv::detail::KernelTag {
if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) {
// This is a cv::Rect -- configure the IE preprocessing
ii->setPrecision(toIE(meta_0.depth));
configureInputInfo(ii, mm_0);
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
} else {
// This is a cv::GMat (equals to: cv::Mat)
@@ -733,9 +825,8 @@ struct InferList2: public cv::detail::KernelTag {
GAPI_Assert(ctx.args.size() > 1u
&& "This operation must have at least two arguments");
// Since we do a ROI list inference, always assume our input buffer is image
const cv::Mat mat_0 = ctx.inMat(0u);
IE::Blob::Ptr blob_0 = wrapIE(mat_0, cv::gapi::ie::TraitAs::IMAGE);
Views views;
IE::Blob::Ptr blob_0 = extractBlob(ctx, 0, views);
// Take the next argument, which must be vector (of any kind).
// Use it only to obtain the ROI list size (sizes of all other
@@ -869,6 +960,16 @@ IE::Blob::Ptr cv::gapi::ie::util::to_ie(cv::Mat &blob) {
return wrapIE(blob, cv::gapi::ie::TraitAs::IMAGE);
}
IE::Blob::Ptr cv::gapi::ie::util::to_ie(cv::Mat &y_plane, cv::Mat &uv_plane) {
auto y_blob = wrapIE(y_plane, cv::gapi::ie::TraitAs::IMAGE);
auto uv_blob = wrapIE(uv_plane, cv::gapi::ie::TraitAs::IMAGE);
#if INF_ENGINE_RELEASE >= 2021010000
return IE::make_shared_blob<IE::NV12Blob>(y_blob, uv_blob);
#else
return IE::make_shared_blob<InferenceEngine::NV12Blob>(y_blob, uv_blob);
#endif
}
#else // HAVE_INF_ENGINE
cv::gapi::GBackend cv::gapi::ie::backend() {
modules/gapi/src/backends/ie/util.hpp
+1
View File
@@ -28,6 +28,7 @@ namespace util {
GAPI_EXPORTS std::vector<int> to_ocv(const InferenceEngine::SizeVector &dims);
GAPI_EXPORTS cv::Mat to_ocv(InferenceEngine::Blob::Ptr blob);
GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &blob);
GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &y_plane, cv::Mat &uv_plane);
}}}}