potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 07:43:03 +04:00
Code Packages Releases Wiki Activity

Merge branch 4.x

Browse Source
This commit is contained in:
Alexander Alekhin
2021-04-09 10:30:38 +00:00
parent 263de1f86e bfb10d74eb
commit fc628014bb
1114 changed files with 64039 additions and 14611 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/gapi/src/backends/common/gbackend.hpp
+54
View File
@@ -58,6 +58,54 @@ namespace gimpl {
struct Data;
struct RcDesc;
struct GAPI_EXPORTS RMatMediaFrameAdapter final: public cv::RMat::Adapter
{
using MapDescF = std::function<cv::GMatDesc(const GFrameDesc&)>;
using MapDataF = std::function<cv::Mat(const GFrameDesc&, const cv::MediaFrame::View&)>;
RMatMediaFrameAdapter(const cv::MediaFrame& frame,
const MapDescF& frameDescToMatDesc,
const MapDataF& frameViewToMat) :
m_frame(frame),
m_frameDesc(frame.desc()),
m_frameDescToMatDesc(frameDescToMatDesc),
m_frameViewToMat(frameViewToMat)
{ }
virtual cv::RMat::View access(cv::RMat::Access a) override
{
auto rmatToFrameAccess = [](cv::RMat::Access rmatAccess) {
switch(rmatAccess) {
case cv::RMat::Access::R:
return cv::MediaFrame::Access::R;
case cv::RMat::Access::W:
return cv::MediaFrame::Access::W;
default:
cv::util::throw_error(std::logic_error("cv::RMat::Access::R or "
"cv::RMat::Access::W can only be mapped to cv::MediaFrame::Access!"));
}
};
auto fv = m_frame.access(rmatToFrameAccess(a));
auto fvHolder = std::make_shared<cv::MediaFrame::View>(std::move(fv));
auto callback = [fvHolder]() mutable { fvHolder.reset(); };
return asView(m_frameViewToMat(m_frame.desc(), *fvHolder), callback);
}
virtual cv::GMatDesc desc() const override
{
return m_frameDescToMatDesc(m_frameDesc);
}
cv::MediaFrame m_frame;
cv::GFrameDesc m_frameDesc;
MapDescF m_frameDescToMatDesc;
MapDataF m_frameViewToMat;
};
namespace magazine {
template<typename... Ts> struct Class
{
@@ -161,6 +209,12 @@ inline cv::util::optional<T> getCompileArg(const cv::GCompileArgs &args)
void GAPI_EXPORTS createMat(const cv::GMatDesc& desc, cv::Mat& mat);
inline void convertInt64ToInt32(const int64_t* src, int* dst, size_t size)
{
std::transform(src, src + size, dst,
[](int64_t el) { return static_cast<int>(el); });
}
}} // cv::gimpl
#endif // OPENCV_GAPI_GBACKEND_HPP
modules/gapi/src/backends/common/gcompoundkernel.cpp
+1 -1
View File
@@ -16,7 +16,7 @@
cv::detail::GCompoundContext::GCompoundContext(const cv::GArgs& in_args)
{
m_args.resize(in_args.size());
for (const auto& it : ade::util::indexed(in_args))
for (const auto it : ade::util::indexed(in_args))
{
const auto& i = ade::util::index(it);
const auto& in_arg = ade::util::value(it);
modules/gapi/src/backends/common/serialization.cpp
+13 -5
View File
@@ -2,7 +2,7 @@
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
// Copyright (C) 2020-2021 Intel Corporation
#include <set> // set
#include <map> // map
@@ -71,7 +71,7 @@ void linkNodes(ade::Graph& g) {
for (const auto& nh : g.nodes()) {
if (gm.metadata(nh).get<cv::gimpl::NodeType>().t == cv::gimpl::NodeType::OP) {
const auto& op = gm.metadata(nh).get<gimpl::Op>();
for (const auto& in : ade::util::indexed(op.args)) {
for (const auto in : ade::util::indexed(op.args)) {
const auto& arg = ade::util::value(in);
if (arg.kind == cv::detail::ArgKind::GOBJREF) {
const auto idx = ade::util::index(in);
@@ -82,9 +82,9 @@ void linkNodes(ade::Graph& g) {
}
}
for (const auto& out : ade::util::indexed(op.outs)) {
const auto idx = ade::util::index(out);
const auto rc = ade::util::value(out);
for (const auto out : ade::util::indexed(op.outs)) {
const auto idx = ade::util::index(out);
const auto& rc = ade::util::value(out);
const auto& out_nh = dataNodes.at(rc);
const auto& out_eh = g.link(nh, out_nh);
gm.metadata(out_eh).set(cv::gimpl::Output{idx});
@@ -906,6 +906,9 @@ GAPI_EXPORTS void serialize(IOStream& os, const cv::GMetaArgs &ma) {
GAPI_EXPORTS void serialize(IOStream& os, const cv::GRunArgs &ra) {
os << ra;
}
GAPI_EXPORTS void serialize(IOStream& os, const std::vector<std::string> &vs) {
os << vs;
}
GAPI_EXPORTS GMetaArgs meta_args_deserialize(IIStream& is) {
GMetaArgs s;
is >> s;
@@ -916,6 +919,11 @@ GAPI_EXPORTS GRunArgs run_args_deserialize(IIStream& is) {
is >> s;
return s;
}
GAPI_EXPORTS std::vector<std::string> vector_of_strings_deserialize(IIStream& is) {
std::vector<std::string> s;
is >> s;
return s;
}
} // namespace s11n
} // namespace gapi
modules/gapi/src/backends/common/serialization.hpp
+3 -1
View File
@@ -5,7 +5,7 @@
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
// Copyright (C) 2020-2021 Intel Corporation
#include <iostream>
#include <fstream>
@@ -217,8 +217,10 @@ GAPI_EXPORTS std::unique_ptr<IIStream> getInStream(const std::vector<char> &p);
GAPI_EXPORTS void serialize(IOStream& os, const cv::GCompileArgs &ca);
GAPI_EXPORTS void serialize(IOStream& os, const cv::GMetaArgs &ma);
GAPI_EXPORTS void serialize(IOStream& os, const cv::GRunArgs &ra);
GAPI_EXPORTS void serialize(IOStream& os, const std::vector<std::string> &vs);
GAPI_EXPORTS GMetaArgs meta_args_deserialize(IIStream& is);
GAPI_EXPORTS GRunArgs run_args_deserialize(IIStream& is);
GAPI_EXPORTS std::vector<std::string> vector_of_strings_deserialize(IIStream& is);
} // namespace s11n
} // namespace gapi
modules/gapi/src/backends/cpu/gcpubackend.cpp
+6 -6
View File
@@ -237,11 +237,11 @@ void cv::gimpl::GCPUExecutable::run(std::vector<InObj> &&input_objs,
// - Output parameters.
// FIXME: pre-allocate internal Mats, etc, according to the known meta
for (const auto &out_it : ade::util::indexed(op.outs))
for (const auto out_it : ade::util::indexed(op.outs))
{
// FIXME: Can the same GArg type resolution mechanism be reused here?
const auto out_port = ade::util::index(out_it);
const auto out_desc = ade::util::value(out_it);
const auto out_port = ade::util::index(out_it);
const auto& out_desc = ade::util::value(out_it);
context.m_results[out_port] = magazine::getObjPtr(m_res, out_desc);
}
@@ -259,10 +259,10 @@ void cv::gimpl::GCPUExecutable::run(std::vector<InObj> &&input_objs,
//FIXME: unify with cv::detail::ensure_out_mats_not_reallocated
//FIXME: when it's done, remove can_describe(const GMetaArg&, const GRunArgP&)
//and descr_of(const cv::GRunArgP &argp)
for (const auto &out_it : ade::util::indexed(op_info.expected_out_metas))
for (const auto out_it : ade::util::indexed(op_info.expected_out_metas))
{
const auto out_index = ade::util::index(out_it);
const auto expected_meta = ade::util::value(out_it);
const auto out_index = ade::util::index(out_it);
const auto& expected_meta = ade::util::value(out_it);
if (!can_describe(expected_meta, context.m_results[out_index]))
{
modules/gapi/src/backends/cpu/gcpucore.cpp
+71 -10
View File
@@ -510,14 +510,6 @@ GAPI_OCV_KERNEL(GCPUCrop, cv::gapi::core::GCrop)
}
};
GAPI_OCV_KERNEL(GCPUCopy, cv::gapi::core::GCopy)
{
static void run(const cv::Mat& in, cv::Mat& out)
{
in.copyTo(out);
}
};
GAPI_OCV_KERNEL(GCPUConcatHor, cv::gapi::core::GConcatHor)
{
static void run(const cv::Mat& in1, const cv::Mat& in2, cv::Mat& out)
@@ -585,6 +577,63 @@ GAPI_OCV_KERNEL(GCPUWarpAffine, cv::gapi::core::GWarpAffine)
}
};
GAPI_OCV_KERNEL(GCPUKMeansND, cv::gapi::core::GKMeansND)
{
static void run(const cv::Mat& data, const int K, const cv::Mat& inBestLabels,
const cv::TermCriteria& criteria, const int attempts,
const cv::KmeansFlags flags,
double& compactness, cv::Mat& outBestLabels, cv::Mat& centers)
{
if (flags & cv::KMEANS_USE_INITIAL_LABELS)
{
inBestLabels.copyTo(outBestLabels);
}
compactness = cv::kmeans(data, K, outBestLabels, criteria, attempts, flags, centers);
}
};
GAPI_OCV_KERNEL(GCPUKMeansNDNoInit, cv::gapi::core::GKMeansNDNoInit)
{
static void run(const cv::Mat& data, const int K, const cv::TermCriteria& criteria,
const int attempts, const cv::KmeansFlags flags,
double& compactness, cv::Mat& outBestLabels, cv::Mat& centers)
{
compactness = cv::kmeans(data, K, outBestLabels, criteria, attempts, flags, centers);
}
};
GAPI_OCV_KERNEL(GCPUKMeans2D, cv::gapi::core::GKMeans2D)
{
static void run(const std::vector<cv::Point2f>& data, const int K,
const std::vector<int>& inBestLabels, const cv::TermCriteria& criteria,
const int attempts, const cv::KmeansFlags flags,
double& compactness, std::vector<int>& outBestLabels,
std::vector<cv::Point2f>& centers)
{
if (flags & cv::KMEANS_USE_INITIAL_LABELS)
{
outBestLabels = inBestLabels;
}
compactness = cv::kmeans(data, K, outBestLabels, criteria, attempts, flags, centers);
}
};
GAPI_OCV_KERNEL(GCPUKMeans3D, cv::gapi::core::GKMeans3D)
{
static void run(const std::vector<cv::Point3f>& data, const int K,
const std::vector<int>& inBestLabels, const cv::TermCriteria& criteria,
const int attempts, const cv::KmeansFlags flags,
double& compactness, std::vector<int>& outBestLabels,
std::vector<cv::Point3f>& centers)
{
if (flags & cv::KMEANS_USE_INITIAL_LABELS)
{
outBestLabels = inBestLabels;
}
compactness = cv::kmeans(data, K, outBestLabels, criteria, attempts, flags, centers);
}
};
GAPI_OCV_KERNEL(GCPUParseSSDBL, cv::gapi::nn::parsers::GParseSSDBL)
{
static void run(const cv::Mat& in_ssd_result,
@@ -643,6 +692,14 @@ GAPI_OCV_KERNEL(GCPUSizeR, cv::gapi::streaming::GSizeR)
}
};
GAPI_OCV_KERNEL(GCPUSizeMF, cv::gapi::streaming::GSizeMF)
{
static void run(const cv::MediaFrame& in, cv::Size& out)
{
out = in.desc().size;
}
};
cv::gapi::GKernelPackage cv::gapi::core::cpu::kernels()
{
static auto pkg = cv::gapi::kernels
@@ -705,7 +762,6 @@ cv::gapi::GKernelPackage cv::gapi::core::cpu::kernels()
, GCPURemap
, GCPUFlip
, GCPUCrop
, GCPUCopy
, GCPUConcatHor
, GCPUConcatVert
, GCPULUT
@@ -714,11 +770,16 @@ cv::gapi::GKernelPackage cv::gapi::core::cpu::kernels()
, GCPUNormalize
, GCPUWarpPerspective
, GCPUWarpAffine
, GCPUKMeansND
, GCPUKMeansNDNoInit
, GCPUKMeans2D
, GCPUKMeans3D
, GCPUParseSSDBL
, GOCVParseSSD
, GCPUParseYolo
, GCPUSize
, GCPUSizeR
>();
, GCPUSizeMF
>();
return pkg;
}
modules/gapi/src/backends/cpu/gcpukernel.cpp
+5
View File
@@ -41,6 +41,11 @@ cv::detail::OpaqueRef& cv::GCPUContext::outOpaqueRef(int output)
return util::get<cv::detail::OpaqueRef>(m_results.at(output));
}
cv::MediaFrame& cv::GCPUContext::outFrame(int output)
{
return *util::get<cv::MediaFrame*>(m_results.at(output));
}
cv::GCPUKernel::GCPUKernel()
{
}
modules/gapi/src/backends/cpu/gcpustereo.cpp
+85
View File
@@ -0,0 +1,85 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2021 Intel Corporation
#include <opencv2/gapi/stereo.hpp>
#include <opencv2/gapi/cpu/stereo.hpp>
#include <opencv2/gapi/cpu/gcpukernel.hpp>
#ifdef HAVE_OPENCV_STEREO
#include <opencv2/stereo.hpp>
#endif // HAVE_OPENCV_STEREO
#ifdef HAVE_OPENCV_STEREO
/** @brief Structure for the Stereo operation setup parameters.*/
struct GAPI_EXPORTS StereoSetup {
double baseline;
double focus;
cv::Ptr<cv::StereoBM> stereoBM;
};
namespace {
cv::Mat calcDepth(const cv::Mat &left, const cv::Mat &right,
const StereoSetup &ss) {
constexpr int DISPARITY_SHIFT_16S = 4;
cv::Mat disp;
ss.stereoBM->compute(left, right, disp);
disp.convertTo(disp, CV_32FC1, 1./(1 << DISPARITY_SHIFT_16S), 0);
return (ss.focus * ss.baseline) / disp;
}
} // anonymous namespace
GAPI_OCV_KERNEL_ST(GCPUStereo, cv::gapi::calib3d::GStereo, StereoSetup)
{
static void setup(const cv::GMatDesc&, const cv::GMatDesc&,
const cv::gapi::StereoOutputFormat,
std::shared_ptr<StereoSetup> &stereoSetup,
const cv::GCompileArgs &compileArgs) {
auto stereoInit = cv::gapi::getCompileArg<cv::gapi::calib3d::cpu::StereoInitParam>(compileArgs)
.value_or(cv::gapi::calib3d::cpu::StereoInitParam{});
StereoSetup ss{stereoInit.baseline,
stereoInit.focus,
cv::StereoBM::create(stereoInit.numDisparities,
stereoInit.blockSize)};
stereoSetup = std::make_shared<StereoSetup>(ss);
}
static void run(const cv::Mat& left,
const cv::Mat& right,
const cv::gapi::StereoOutputFormat oF,
cv::Mat& out_mat,
const StereoSetup &stereoSetup) {
switch(oF){
case cv::gapi::StereoOutputFormat::DEPTH_FLOAT16:
calcDepth(left, right, stereoSetup).convertTo(out_mat, CV_16FC1);
break;
case cv::gapi::StereoOutputFormat::DEPTH_FLOAT32:
calcDepth(left, right, stereoSetup).copyTo(out_mat);
break;
case cv::gapi::StereoOutputFormat::DISPARITY_FIXED16_12_4:
stereoSetup.stereoBM->compute(left, right, out_mat);
break;
case cv::gapi::StereoOutputFormat::DISPARITY_FIXED16_11_5:
GAPI_Assert(false && "This case may be supported in future.");
default:
GAPI_Assert(false && "Unknown output format!");
}
}
};
cv::gapi::GKernelPackage cv::gapi::calib3d::cpu::kernels() {
static auto pkg = cv::gapi::kernels<GCPUStereo>();
return pkg;
}
#else
cv::gapi::GKernelPackage cv::gapi::calib3d::cpu::kernels()
{
return GKernelPackage();
}
#endif // HAVE_OPENCV_STEREO
modules/gapi/src/backends/cpu/gcpuvideo.cpp
+97
View File
@@ -80,12 +80,109 @@ GAPI_OCV_KERNEL(GCPUCalcOptFlowLKForPyr, cv::gapi::video::GCalcOptFlowLKForPyr)
}
};
GAPI_OCV_KERNEL_ST(GCPUBackgroundSubtractor,
cv::gapi::video::GBackgroundSubtractor,
cv::BackgroundSubtractor)
{
static void setup(const cv::GMatDesc&, const cv::gapi::video::BackgroundSubtractorParams& bsParams,
std::shared_ptr<cv::BackgroundSubtractor>& state,
const cv::GCompileArgs&)
{
if (bsParams.operation == cv::gapi::video::TYPE_BS_MOG2)
state = cv::createBackgroundSubtractorMOG2(bsParams.history,
bsParams.threshold,
bsParams.detectShadows);
else if (bsParams.operation == cv::gapi::video::TYPE_BS_KNN)
state = cv::createBackgroundSubtractorKNN(bsParams.history,
bsParams.threshold,
bsParams.detectShadows);
GAPI_Assert(state);
}
static void run(const cv::Mat& in, const cv::gapi::video::BackgroundSubtractorParams& bsParams,
cv::Mat &out, cv::BackgroundSubtractor& state)
{
state.apply(in, out, bsParams.learningRate);
}
};
GAPI_OCV_KERNEL_ST(GCPUKalmanFilter, cv::gapi::video::GKalmanFilter, cv::KalmanFilter)
{
static void setup(const cv::GMatDesc&, const cv::GOpaqueDesc&,
const cv::GMatDesc&, const cv::gapi::KalmanParams& kfParams,
std::shared_ptr<cv::KalmanFilter> &state, const cv::GCompileArgs&)
{
state = std::make_shared<cv::KalmanFilter>(kfParams.transitionMatrix.rows, kfParams.measurementMatrix.rows,
kfParams.controlMatrix.cols, kfParams.transitionMatrix.type());
// initial state
kfParams.state.copyTo(state->statePost);
kfParams.errorCov.copyTo(state->errorCovPost);
// dynamic system initialization
kfParams.controlMatrix.copyTo(state->controlMatrix);
kfParams.measurementMatrix.copyTo(state->measurementMatrix);
kfParams.transitionMatrix.copyTo(state->transitionMatrix);
kfParams.processNoiseCov.copyTo(state->processNoiseCov);
kfParams.measurementNoiseCov.copyTo(state->measurementNoiseCov);
}
static void run(const cv::Mat& measurements, bool haveMeasurement,
const cv::Mat& control, const cv::gapi::KalmanParams&,
cv::Mat &out, cv::KalmanFilter& state)
{
cv::Mat pre = state.predict(control);
if (haveMeasurement)
state.correct(measurements).copyTo(out);
else
pre.copyTo(out);
}
};
GAPI_OCV_KERNEL_ST(GCPUKalmanFilterNoControl, cv::gapi::video::GKalmanFilterNoControl, cv::KalmanFilter)
{
static void setup(const cv::GMatDesc&, const cv::GOpaqueDesc&,
const cv::gapi::KalmanParams& kfParams,
std::shared_ptr<cv::KalmanFilter> &state,
const cv::GCompileArgs&)
{
state = std::make_shared<cv::KalmanFilter>(kfParams.transitionMatrix.rows, kfParams.measurementMatrix.rows,
0, kfParams.transitionMatrix.type());
// initial state
kfParams.state.copyTo(state->statePost);
kfParams.errorCov.copyTo(state->errorCovPost);
// dynamic system initialization
kfParams.measurementMatrix.copyTo(state->measurementMatrix);
kfParams.transitionMatrix.copyTo(state->transitionMatrix);
kfParams.processNoiseCov.copyTo(state->processNoiseCov);
kfParams.measurementNoiseCov.copyTo(state->measurementNoiseCov);
}
static void run(const cv::Mat& measurements, bool haveMeasurement,
const cv::gapi::KalmanParams&, cv::Mat &out,
cv::KalmanFilter& state)
{
cv::Mat pre = state.predict();
if (haveMeasurement)
state.correct(measurements).copyTo(out);
else
pre.copyTo(out);
}
};
cv::gapi::GKernelPackage cv::gapi::video::cpu::kernels()
{
static auto pkg = cv::gapi::kernels
< GCPUBuildOptFlowPyramid
, GCPUCalcOptFlowLK
, GCPUCalcOptFlowLKForPyr
, GCPUBackgroundSubtractor
, GCPUKalmanFilter
, GCPUKalmanFilterNoControl
>();
return pkg;
}
modules/gapi/src/backends/fluid/gfluidbackend.cpp
+7 -7
View File
@@ -232,7 +232,7 @@ void cv::gimpl::FluidAgent::reset()
{
m_producedLines = 0;
for (const auto& it : ade::util::indexed(in_views))
for (const auto it : ade::util::indexed(in_views))
{
auto& v = ade::util::value(it);
if (v)
@@ -505,7 +505,7 @@ void cv::gimpl::FluidAgent::doWork()
k.m_f(in_args, out_buffers);
for (const auto& it : ade::util::indexed(in_views))
for (const auto it : ade::util::indexed(in_views))
{
auto& in_view = ade::util::value(it);
@@ -517,7 +517,7 @@ void cv::gimpl::FluidAgent::doWork()
};
}
for (auto out_buf : out_buffers)
for (auto* out_buf : out_buffers)
{
out_buf->priv().writeDone();
// FIXME WARNING: Scratch buffers rotated here too!
@@ -571,10 +571,10 @@ void cv::gimpl::GFluidExecutable::initBufferRois(std::vector<int>& readStarts,
}
// First, initialize rois for output nodes, add them to traversal stack
for (const auto& it : ade::util::indexed(proto.out_nhs))
for (const auto it : ade::util::indexed(proto.out_nhs))
{
const auto idx = ade::util::index(it);
const auto nh = ade::util::value(it);
const auto& nh = ade::util::value(it);
const auto &d = m_gm.metadata(nh).get<Data>();
@@ -927,7 +927,7 @@ std::size_t cv::gimpl::GFluidExecutable::total_buffers_size() const
{
GConstFluidModel fg(m_g);
std::size_t total_size = 0;
for (const auto &i : ade::util::indexed(m_buffers))
for (const auto i : ade::util::indexed(m_buffers))
{
// Check that all internal and scratch buffers are allocated
const auto idx = ade::util::index(i);
@@ -1310,7 +1310,7 @@ void cv::gimpl::GFluidExecutable::run(std::vector<InObj> &input_objs,
agent->reset();
// Pass input cv::Scalar's to agent argument
const auto& op = m_gm.metadata(agent->op_handle).get<Op>();
for (const auto& it : ade::util::indexed(op.args))
for (const auto it : ade::util::indexed(op.args))
{
const auto& arg = ade::util::value(it);
packArg(agent->in_args[ade::util::index(it)], arg);
modules/gapi/src/backends/fluid/gfluidcore.cpp
+421 -53
View File
@@ -97,6 +97,132 @@ static inline DST divr(SRC1 x, SRC2 y, float scale=1)
// Fluid kernels: addWeighted
//
//---------------------------
#if CV_SIMD
CV_ALWAYS_INLINE v_float32 v_load_f32(const ushort* in)
{
return v_cvt_f32(v_reinterpret_as_s32(vx_load_expand(in)));
}
CV_ALWAYS_INLINE v_float32 v_load_f32(const short* in)
{
return v_cvt_f32(vx_load_expand(in));
}
CV_ALWAYS_INLINE v_float32 v_load_f32(const uchar* in)
{
return v_cvt_f32(v_reinterpret_as_s32(vx_load_expand_q(in)));
}
#endif
#if CV_SSE2
CV_ALWAYS_INLINE void addw_short_store(short* out, const v_int32& c1, const v_int32& c2)
{
vx_store(out, v_pack(c1, c2));
}
CV_ALWAYS_INLINE void addw_short_store(ushort* out, const v_int32& c1, const v_int32& c2)
{
vx_store(out, v_pack_u(c1, c2));
}
template<typename SRC, typename DST>
CV_ALWAYS_INLINE int addw_simd(const SRC in1[], const SRC in2[], DST out[],
const float _alpha, const float _beta,
const float _gamma, int length)
{
static_assert(((std::is_same<SRC, ushort>::value) && (std::is_same<DST, ushort>::value)) ||
((std::is_same<SRC, short>::value) && (std::is_same<DST, short>::value)),
"This templated overload is only for short and ushort type combinations.");
constexpr int nlanes = (std::is_same<DST, ushort>::value) ? static_cast<int>(v_uint16::nlanes) :
static_cast<int>(v_int16::nlanes);
if (length < nlanes)
return 0;
v_float32 alpha = vx_setall_f32(_alpha);
v_float32 beta = vx_setall_f32(_beta);
v_float32 gamma = vx_setall_f32(_gamma);
int x = 0;
for (;;)
{
for (; x <= length - nlanes; x += nlanes)
{
v_float32 a1 = v_load_f32(&in1[x]);
v_float32 a2 = v_load_f32(&in1[x + nlanes / 2]);
v_float32 b1 = v_load_f32(&in2[x]);
v_float32 b2 = v_load_f32(&in2[x + nlanes / 2]);
addw_short_store(&out[x], v_round(v_fma(a1, alpha, v_fma(b1, beta, gamma))),
v_round(v_fma(a2, alpha, v_fma(b2, beta, gamma))));
}
if (x < length)
{
x = length - nlanes;
continue; // process one more time (unaligned tail)
}
break;
}
return x;
}
template<typename SRC>
CV_ALWAYS_INLINE int addw_simd(const SRC in1[], const SRC in2[], uchar out[],
const float _alpha, const float _beta,
const float _gamma, int length)
{
constexpr int nlanes = v_uint8::nlanes;
if (length < nlanes)
return 0;
v_float32 alpha = vx_setall_f32(_alpha);
v_float32 beta = vx_setall_f32(_beta);
v_float32 gamma = vx_setall_f32(_gamma);
int x = 0;
for (;;)
{
for (; x <= length - nlanes; x += nlanes)
{
v_float32 a1 = v_load_f32(&in1[x]);
v_float32 a2 = v_load_f32(&in1[x + nlanes / 4]);
v_float32 a3 = v_load_f32(&in1[x + nlanes / 2]);
v_float32 a4 = v_load_f32(&in1[x + 3 * nlanes / 4]);
v_float32 b1 = v_load_f32(&in2[x]);
v_float32 b2 = v_load_f32(&in2[x + nlanes / 4]);
v_float32 b3 = v_load_f32(&in2[x + nlanes / 2]);
v_float32 b4 = v_load_f32(&in2[x + 3 * nlanes / 4]);
v_int32 sum1 = v_round(v_fma(a1, alpha, v_fma(b1, beta, gamma))),
sum2 = v_round(v_fma(a2, alpha, v_fma(b2, beta, gamma))),
sum3 = v_round(v_fma(a3, alpha, v_fma(b3, beta, gamma))),
sum4 = v_round(v_fma(a4, alpha, v_fma(b4, beta, gamma)));
vx_store(&out[x], v_pack_u(v_pack(sum1, sum2), v_pack(sum3, sum4)));
}
if (x < length)
{
x = length - nlanes;
continue; // process one more time (unaligned tail)
}
break;
}
return x;
}
template<typename SRC>
CV_ALWAYS_INLINE int addw_simd(const SRC*, const SRC*, float*,
const float, const float,
const float, int)
{
//Cases when dst type is float are successfully vectorized with compiler.
return 0;
}
#endif // CV_SSE2
template<typename DST, typename SRC1, typename SRC2>
static void run_addweighted(Buffer &dst, const View &src1, const View &src2,
@@ -117,8 +243,13 @@ static void run_addweighted(Buffer &dst, const View &src1, const View &src2,
auto _beta = static_cast<float>( beta );
auto _gamma = static_cast<float>( gamma );
for (int l=0; l < length; l++)
out[l] = addWeighted<DST>(in1[l], in2[l], _alpha, _beta, _gamma);
int x = 0;
#if CV_SSE2
x = addw_simd(in1, in2, out, _alpha, _beta, _gamma, length);
#endif
for (; x < length; ++x)
out[x] = addWeighted<DST>(in1[x], in2[x], _alpha, _beta, _gamma);
}
GAPI_FLUID_KERNEL(GFluidAddW, cv::gapi::core::GAddW, false)
@@ -843,6 +974,262 @@ static void run_arithm_s(DST out[], const SRC in[], int width, int chan,
CV_Error(cv::Error::StsBadArg, "unsupported number of channels");
}
#if CV_SIMD
CV_ALWAYS_INLINE void absdiffc_short_store_c1c2c4(short* out_ptr, const v_int32& c1, const v_int32& c2)
{
vx_store(out_ptr, v_pack(c1, c2));
}
CV_ALWAYS_INLINE void absdiffc_short_store_c1c2c4(ushort* out_ptr, const v_int32& c1, const v_int32& c2)
{
vx_store(out_ptr, v_pack_u(c1, c2));
}
template<typename T>
CV_ALWAYS_INLINE int absdiffc_simd_c1c2c4(const T in[], T out[],
const v_float32& s, const int length)
{
static_assert((std::is_same<T, ushort>::value) || (std::is_same<T, short>::value),
"This templated overload is only for short or ushort type combinations.");
constexpr int nlanes = (std::is_same<T, ushort>::value) ? static_cast<int>(v_uint16::nlanes) :
static_cast<int>(v_int16::nlanes);
if (length < nlanes)
return 0;
int x = 0;
for (;;)
{
for (; x <= length - nlanes; x += nlanes)
{
v_float32 a1 = v_load_f32(in + x);
v_float32 a2 = v_load_f32(in + x + nlanes / 2);
absdiffc_short_store_c1c2c4(&out[x], v_round(v_absdiff(a1, s)),
v_round(v_absdiff(a2, s)));
}
if (x < length && (in != out))
{
x = length - nlanes;
continue; // process unaligned tail
}
break;
}
return x;
}
template<>
CV_ALWAYS_INLINE int absdiffc_simd_c1c2c4<uchar>(const uchar in[], uchar out[],
const v_float32& s, const int length)
{
constexpr int nlanes = static_cast<int>(v_uint8::nlanes);
if (length < nlanes)
return 0;
int x = 0;
for (;;)
{
for (; x <= length - nlanes; x += nlanes)
{
v_float32 a1 = v_load_f32(in + x);
v_float32 a2 = v_load_f32(in + x + nlanes / 4);
v_float32 a3 = v_load_f32(in + x + nlanes / 2);
v_float32 a4 = v_load_f32(in + x + 3 * nlanes / 4);
vx_store(&out[x], v_pack_u(v_pack(v_round(v_absdiff(a1, s)),
v_round(v_absdiff(a2, s))),
v_pack(v_round(v_absdiff(a3, s)),
v_round(v_absdiff(a4, s)))));
}
if (x < length && (in != out))
{
x = length - nlanes;
continue; // process unaligned tail
}
break;
}
return x;
}
CV_ALWAYS_INLINE void absdiffc_short_store_c3(short* out_ptr, const v_int32& c1,
const v_int32& c2, const v_int32& c3,
const v_int32& c4, const v_int32& c5,
const v_int32& c6)
{
constexpr int nlanes = static_cast<int>(v_int16::nlanes);
vx_store(out_ptr, v_pack(c1, c2));
vx_store(out_ptr + nlanes, v_pack(c3, c4));
vx_store(out_ptr + 2*nlanes, v_pack(c5, c6));
}
CV_ALWAYS_INLINE void absdiffc_short_store_c3(ushort* out_ptr, const v_int32& c1,
const v_int32& c2, const v_int32& c3,
const v_int32& c4, const v_int32& c5,
const v_int32& c6)
{
constexpr int nlanes = static_cast<int>(v_uint16::nlanes);
vx_store(out_ptr, v_pack_u(c1, c2));
vx_store(out_ptr + nlanes, v_pack_u(c3, c4));
vx_store(out_ptr + 2*nlanes, v_pack_u(c5, c6));
}
template<typename T>
CV_ALWAYS_INLINE int absdiffc_simd_c3_impl(const T in[], T out[],
const v_float32& s1, const v_float32& s2,
const v_float32& s3, const int length)
{
static_assert((std::is_same<T, ushort>::value) || (std::is_same<T, short>::value),
"This templated overload is only for short or ushort type combinations.");
constexpr int nlanes = (std::is_same<T, ushort>::value) ? static_cast<int>(v_uint16::nlanes):
static_cast<int>(v_int16::nlanes);
if (length < 3 * nlanes)
return 0;
int x = 0;
for (;;)
{
for (; x <= length - 3 * nlanes; x += 3 * nlanes)
{
v_float32 a1 = v_load_f32(in + x);
v_float32 a2 = v_load_f32(in + x + nlanes / 2);
v_float32 a3 = v_load_f32(in + x + nlanes);
v_float32 a4 = v_load_f32(in + x + 3 * nlanes / 2);
v_float32 a5 = v_load_f32(in + x + 2 * nlanes);
v_float32 a6 = v_load_f32(in + x + 5 * nlanes / 2);
absdiffc_short_store_c3(&out[x], v_round(v_absdiff(a1, s1)),
v_round(v_absdiff(a2, s2)),
v_round(v_absdiff(a3, s3)),
v_round(v_absdiff(a4, s1)),
v_round(v_absdiff(a5, s2)),
v_round(v_absdiff(a6, s3)));
}
if (x < length && (in != out))
{
x = length - 3 * nlanes;
continue; // process unaligned tail
}
break;
}
return x;
}
template<>
CV_ALWAYS_INLINE int absdiffc_simd_c3_impl<uchar>(const uchar in[], uchar out[],
const v_float32& s1, const v_float32& s2,
const v_float32& s3, const int length)
{
constexpr int nlanes = static_cast<int>(v_uint8::nlanes);
if (length < 3 * nlanes)
return 0;
int x = 0;
for (;;)
{
for (; x <= length - 3 * nlanes; x += 3 * nlanes)
{
vx_store(&out[x],
v_pack_u(v_pack(v_round(v_absdiff(v_load_f32(in + x), s1)),
v_round(v_absdiff(v_load_f32(in + x + nlanes/4), s2))),
v_pack(v_round(v_absdiff(v_load_f32(in + x + nlanes/2), s3)),
v_round(v_absdiff(v_load_f32(in + x + 3*nlanes/4), s1)))));
vx_store(&out[x + nlanes],
v_pack_u(v_pack(v_round(v_absdiff(v_load_f32(in + x + nlanes), s2)),
v_round(v_absdiff(v_load_f32(in + x + 5*nlanes/4), s3))),
v_pack(v_round(v_absdiff(v_load_f32(in + x + 3*nlanes/2), s1)),
v_round(v_absdiff(v_load_f32(in + x + 7*nlanes/4), s2)))));
vx_store(&out[x + 2 * nlanes],
v_pack_u(v_pack(v_round(v_absdiff(v_load_f32(in + x + 2*nlanes), s3)),
v_round(v_absdiff(v_load_f32(in + x + 9*nlanes/4), s1))),
v_pack(v_round(v_absdiff(v_load_f32(in + x + 5*nlanes/2), s2)),
v_round(v_absdiff(v_load_f32(in + x + 11*nlanes/4), s3)))));
}
if (x < length && (in != out))
{
x = length - 3 * nlanes;
continue; // process unaligned tail
}
break;
}
return x;
}
template<typename T>
CV_ALWAYS_INLINE int absdiffc_simd_channels(const T in[], const float scalar[], T out[],
const int width, int chan)
{
int length = width * chan;
v_float32 s = vx_load(scalar);
return absdiffc_simd_c1c2c4(in, out, s, length);
}
template<typename T>
CV_ALWAYS_INLINE int absdiffc_simd_c3(const T in[], const float scalar[], T out[], int width)
{
constexpr int chan = 3;
int length = width * chan;
v_float32 s1 = vx_load(scalar);
#if CV_SIMD_WIDTH == 32
v_float32 s2 = vx_load(scalar + 2);
v_float32 s3 = vx_load(scalar + 1);
#else
v_float32 s2 = vx_load(scalar + 1);
v_float32 s3 = vx_load(scalar + 2);
#endif
return absdiffc_simd_c3_impl(in, out, s1, s2, s3, length);
}
template<typename T>
CV_ALWAYS_INLINE int absdiffc_simd(const T in[], const float scalar[], T out[], int width, int chan)
{
switch (chan)
{
case 1:
case 2:
case 4:
return absdiffc_simd_channels(in, scalar, out, width, chan);
case 3:
return absdiffc_simd_c3(in, scalar, out, width);
default:
break;
}
return 0;
}
#endif // CV_SIMD
template<typename DST, typename SRC>
static void run_absdiffc(Buffer &dst, const View &src, const float scalar[])
{
const auto *in = src.InLine<SRC>(0);
auto *out = dst.OutLine<DST>();
int width = dst.length();
int chan = dst.meta().chan;
int w = 0;
#if CV_SIMD
w = absdiffc_simd(in, scalar, out, width, chan);
#endif
for (; w < width*chan; ++w)
out[w] = absdiff<DST>(in[w], scalar[w%chan]);
}
template<typename DST, typename SRC>
static void run_arithm_s(Buffer &dst, const View &src, const float scalar[4], Arithm arithm,
float scale=1)
@@ -861,11 +1248,6 @@ static void run_arithm_s(Buffer &dst, const View &src, const float scalar[4], Ar
switch (arithm)
{
case ARITHM_ABSDIFF:
for (int w=0; w < width; w++)
for (int c=0; c < chan; c++)
out[chan*w + c] = absdiff<DST>(in[chan*w + c], scalar[c]);
break;
case ARITHM_ADD:
if (usemyscal)
{
@@ -960,26 +1342,47 @@ static void run_arithm_rs(Buffer &dst, const View &src, const float scalar[4], A
}
}
GAPI_FLUID_KERNEL(GFluidAbsDiffC, cv::gapi::core::GAbsDiffC, false)
GAPI_FLUID_KERNEL(GFluidAbsDiffC, cv::gapi::core::GAbsDiffC, true)
{
static const int Window = 1;
static void run(const View &src, const cv::Scalar &_scalar, Buffer &dst)
static void run(const View &src, const cv::Scalar& _scalar, Buffer &dst, Buffer& scratch)
{
const float scalar[4] = {
static_cast<float>(_scalar[0]),
static_cast<float>(_scalar[1]),
static_cast<float>(_scalar[2]),
static_cast<float>(_scalar[3])
};
if (dst.y() == 0)
{
const int chan = src.meta().chan;
float* sc = scratch.OutLine<float>();
for (int i = 0; i < scratch.length(); ++i)
sc[i] = static_cast<float>(_scalar[i % chan]);
}
const float* scalar = scratch.OutLine<float>();
// DST SRC OP __VA_ARGS__
UNARY_(uchar , uchar , run_arithm_s, dst, src, scalar, ARITHM_ABSDIFF);
UNARY_(ushort, ushort, run_arithm_s, dst, src, scalar, ARITHM_ABSDIFF);
UNARY_( short, short, run_arithm_s, dst, src, scalar, ARITHM_ABSDIFF);
UNARY_(uchar, uchar, run_absdiffc, dst, src, scalar);
UNARY_(ushort, ushort, run_absdiffc, dst, src, scalar);
UNARY_(short, short, run_absdiffc, dst, src, scalar);
CV_Error(cv::Error::StsBadArg, "unsupported combination of types");
}
static void initScratch(const GMatDesc&, const GScalarDesc&, Buffer& scratch)
{
#if CV_SIMD
constexpr int buflen = static_cast<int>(v_float32::nlanes) + 2; // buffer size
#else
constexpr int buflen = 4;
#endif
cv::Size bufsize(buflen, 1);
GMatDesc bufdesc = { CV_32F, 1, bufsize };
Buffer buffer(bufdesc);
scratch = std::move(buffer);
}
static void resetScratch(Buffer& /* scratch */)
{
}
};
GAPI_FLUID_KERNEL(GFluidAddC, cv::gapi::core::GAddC, false)
@@ -2675,40 +3078,6 @@ GAPI_FLUID_KERNEL(GFluidSqrt, cv::gapi::core::GSqrt, false)
}
};
GAPI_FLUID_KERNEL(GFluidCopy, cv::gapi::core::GCopy, false)
{
static const int Window = 1;
static void run(const View &src, Buffer &dst)
{
const auto *in = src.InLine<uchar>(0);
auto *out = dst.OutLine<uchar>();
GAPI_DbgAssert(dst.length() == src.length());
GAPI_DbgAssert(dst.meta().chan == src.meta().chan);
GAPI_DbgAssert(dst.meta().depth == src.meta().depth);
int width = src.length();
int elem_size = CV_ELEM_SIZE(CV_MAKETYPE(src.meta().depth, src.meta().chan));
int w = 0; // cycle counter
#if CV_SIMD128
for (; w <= width*elem_size-16; w+=16)
{
v_uint8x16 a;
a = v_load(&in[w]);
v_store(&out[w], a);
}
#endif
for (; w < width*elem_size; w++)
{
out[w] = in[w];
}
}
};
} // namespace fliud
} // namespace gapi
} // namespace cv
@@ -2768,7 +3137,6 @@ cv::gapi::GKernelPackage cv::gapi::core::fluid::kernels()
,GFluidInRange
,GFluidResize
,GFluidSqrt
,GFluidCopy
#if 0
,GFluidMean -- not fluid
,GFluidSum -- not fluid
modules/gapi/src/backends/ie/giebackend.cpp
+720 -277
View File
File diff suppressed because it is too large Load Diff
modules/gapi/src/backends/ie/giebackend.hpp
+20 -14
View File
@@ -13,6 +13,7 @@
#ifdef HAVE_INF_ENGINE
#include <ade/util/algorithm.hpp> // type_list_index
#include <condition_variable>
#include <inference_engine.hpp>
@@ -23,20 +24,22 @@
#include "backends/common/gbackend.hpp"
#include "compiler/gislandmodel.hpp"
#include "backends/ie/giebackend/giewrapper.hpp" // wrap::Plugin
namespace cv {
namespace gimpl {
namespace ie {
struct IECompiled {
#if INF_ENGINE_RELEASE < 2019020000 // < 2019.R2
InferenceEngine::InferencePlugin this_plugin;
#else
InferenceEngine::Core this_core;
#endif
InferenceEngine::ExecutableNetwork this_network;
InferenceEngine::InferRequest this_request;
std::vector<InferenceEngine::InferRequest> createInferRequests();
cv::gapi::ie::detail::ParamDesc params;
cv::gimpl::ie::wrap::Plugin this_plugin;
InferenceEngine::ExecutableNetwork this_network;
};
class RequestPool;
class GIEExecutable final: public GIslandExecutable
{
const ade::Graph &m_g;
@@ -50,11 +53,8 @@ class GIEExecutable final: public GIslandExecutable
// List of all resources in graph (both internal and external)
std::vector<ade::NodeHandle> m_dataNodes;
// Actual data of all resources in graph (both internal and external)
Mag m_res;
// Execution helpers
GArg packArg(const GArg &arg);
// To manage multiple async requests
std::unique_ptr<RequestPool> m_reqPool;
public:
GIEExecutable(const ade::Graph &graph,
@@ -65,8 +65,14 @@ public:
GAPI_Assert(false); // Not implemented yet
}
virtual void run(std::vector<InObj> &&input_objs,
std::vector<OutObj> &&output_objs) override;
virtual void run(std::vector<InObj> &&,
std::vector<OutObj> &&) override {
GAPI_Assert(false && "Not implemented");
}
virtual void run(GIslandExecutable::IInput &in,
GIslandExecutable::IOutput &out) override;
};
}}}
modules/gapi/src/backends/ie/util.hpp
+1
View File
@@ -28,6 +28,7 @@ namespace util {
GAPI_EXPORTS std::vector<int> to_ocv(const InferenceEngine::SizeVector &dims);
GAPI_EXPORTS cv::Mat to_ocv(InferenceEngine::Blob::Ptr blob);
GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &blob);
GAPI_EXPORTS InferenceEngine::Blob::Ptr to_ie(cv::Mat &y_plane, cv::Mat &uv_plane);
}}}}
modules/gapi/src/backends/ocl/goclbackend.cpp
+8 -8
View File
@@ -231,21 +231,21 @@ void cv::gimpl::GOCLExecutable::run(std::vector<InObj> &&input_objs,
// - Output parameters.
// FIXME: pre-allocate internal Mats, etc, according to the known meta
for (const auto &out_it : ade::util::indexed(op.outs))
for (const auto out_it : ade::util::indexed(op.outs))
{
// FIXME: Can the same GArg type resolution mechanism be reused here?
const auto out_port = ade::util::index(out_it);
const auto out_desc = ade::util::value(out_it);
const auto out_port = ade::util::index(out_it);
const auto& out_desc = ade::util::value(out_it);
context.m_results[out_port] = magazine::getObjPtr(m_res, out_desc, true);
}
// Now trigger the executable unit
k.apply(context);
for (const auto &out_it : ade::util::indexed(op_info.expected_out_metas))
for (const auto out_it : ade::util::indexed(op_info.expected_out_metas))
{
const auto out_index = ade::util::index(out_it);
const auto expected_meta = ade::util::value(out_it);
const auto out_index = ade::util::index(out_it);
const auto& expected_meta = ade::util::value(out_it);
if (!can_describe(expected_meta, context.m_results[out_index]))
{
@@ -262,8 +262,8 @@ void cv::gimpl::GOCLExecutable::run(std::vector<InObj> &&input_objs,
for (auto &it : output_objs)
{
auto& rc = it.first;
auto& g_arg = it.second;
const auto& rc = it.first;
auto& g_arg = it.second;
magazine::writeBack(m_res, rc, g_arg);
if (rc.shape == GShape::GMAT)
{
modules/gapi/src/backends/ocl/goclcore.cpp
-9
View File
@@ -490,14 +490,6 @@ GAPI_OCL_KERNEL(GOCLCrop, cv::gapi::core::GCrop)
}
};
GAPI_OCL_KERNEL(GOCLCopy, cv::gapi::core::GCopy)
{
static void run(const cv::UMat& in, cv::UMat& out)
{
in.copyTo(out);
}
};
GAPI_OCL_KERNEL(GOCLConcatHor, cv::gapi::core::GConcatHor)
{
static void run(const cv::UMat& in1, const cv::UMat& in2, cv::UMat& out)
@@ -590,7 +582,6 @@ cv::gapi::GKernelPackage cv::gapi::core::ocl::kernels()
, GOCLRemap
, GOCLFlip
, GOCLCrop
, GOCLCopy
, GOCLConcatHor
, GOCLConcatVert
, GOCLLUT
modules/gapi/src/backends/onnx/gonnxbackend.cpp
+279 -72
View File
@@ -13,8 +13,15 @@
#include <ade/util/zip_range.hpp>
#include <opencv2/gapi/infer.hpp>
#include <opencv2/gapi/own/convert.hpp>
#include <opencv2/gapi/gframe.hpp>
#include <codecvt> // wstring_convert
#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK!
#include "logger.hpp"
namespace {
struct ONNXCallContext;
}
namespace cv {
namespace gimpl {
@@ -25,12 +32,35 @@ enum TensorPosition : int {
OUTPUT
};
static std::string pdims(const std::vector<int64_t> &dims) {
std::stringstream ss;
auto it = dims.begin();
ss << *it++;
for (; it != dims.end(); ++it) {
ss << '/' << *it;
}
return ss.str();
}
struct TensorInfo {
TensorInfo() = default;
explicit TensorInfo(const Ort::TensorTypeAndShapeInfo& info)
: dims(info.GetShape())
, type(info.GetElementType())
, is_dynamic(std::find(dims.begin(), dims.end(), -1) != dims.end()) {
, is_dynamic(ade::util::find(dims, -1) != dims.end()) {
// Double-check if the tensor is really dynamic
// Allow N to be -1
if (is_dynamic
&& dims[0] == -1
&& dims.size() > 1
&& std::find(dims.begin() + 1, dims.end(), -1) == dims.end()) {
GAPI_LOG_WARNING(NULL, "Promoting N=-1 to N=1 for tensor " << pdims(dims));
dims[0] = 1;
is_dynamic = false;
}
if (!is_dynamic) {
size = std::accumulate(dims.begin(),
dims.end(),
@@ -64,6 +94,8 @@ struct TensorInfo {
cv::util::optional<MeanStdev> mstd;
};
using Views = std::vector<std::unique_ptr<cv::MediaFrame::View>>;
class ONNXCompiled {
// ONNX Resources
// NOTE: Env must live with the session, otherwise segfaults.
@@ -74,6 +106,7 @@ class ONNXCompiled {
std::vector<TensorInfo> in_tensor_info;
std::vector<TensorInfo> out_tensor_info;
bool is_dynamic = false;
bool is_postproc = false;
// G-API <Net> description
gapi::onnx::detail::ParamDesc params;
@@ -88,6 +121,7 @@ class ONNXCompiled {
void Run(const std::vector<cv::Mat>& ins,
const std::vector<cv::Mat>& outs);
std::vector<std::string> in_names_without_const;
public:
explicit ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp);
@@ -98,9 +132,12 @@ public:
std::size_t numInputs() const { return params.num_in; }
std::size_t numOutputs() const { return params.num_out; }
void setInput(int i, const cv::Mat &m);
void setOutput(int i, cv::Mat &m);
void setOutput(int idx, cv::Mat &m);
cv::Mat allocOutput(int i) const;
// Gets exMat from input
void extractMat(ONNXCallContext &ctx, const size_t in_idx, Views &views);
// Extracted cv::Mat from input cv::Mat/cv::MediaFrame
cv::Mat exMat;
// Run with the assigned inputs/outputs
void run();
};
@@ -121,7 +158,7 @@ inline std::vector<const char*> getCharNames(const std::vector<std::string>& nam
inline int getIdxByName(const std::vector<cv::gimpl::onnx::TensorInfo>& info, const std::string& name) {
// FIXME: Cache the ordering
const auto it = std::find_if(info.begin(), info.end(), [&](const cv::gimpl::onnx::TensorInfo &i) {
const auto it = ade::util::find_if(info, [&](const cv::gimpl::onnx::TensorInfo &i) {
return i.name == name;
});
GAPI_Assert(it != info.end());
@@ -132,7 +169,9 @@ inline int toCV(ONNXTensorElementDataType prec) {
switch (prec) {
case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8: return CV_8U;
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT: return CV_32F;
default: GAPI_Assert(false && "Unsupported data type");
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32: return CV_32S;
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: return CV_32S;
default: GAPI_Assert(false && "ONNX. Unsupported data type");
}
return -1;
}
@@ -146,11 +185,30 @@ inline std::vector<int> toCV(const std::vector<int64_t> &vsz) {
return result;
}
inline cv::Mat toCV(Ort::Value &v) {
auto info = v.GetTensorTypeAndShapeInfo();
return cv::Mat(toCV(info.GetShape()),
toCV(info.GetElementType()),
reinterpret_cast<void*>(v.GetTensorMutableData<uint8_t*>()));
inline void copyFromONNX(Ort::Value &v, cv::Mat& mat) {
const auto info = v.GetTensorTypeAndShapeInfo();
const auto prec = info.GetElementType();
const auto shape = toCV(info.GetShape());
mat.create(shape, toCV(prec));
switch (prec) {
#define HANDLE(E,T) \
case E: std::copy_n(v.GetTensorMutableData<T>(), \
mat.total(), \
reinterpret_cast<T*>(mat.data)); \
break;
HANDLE(ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, uint8_t);
HANDLE(ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, float);
HANDLE(ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, int);
#undef HANDLE
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64: {
GAPI_LOG_WARNING(NULL, "INT64 isn't supported for cv::Mat. Conversion to INT32 is used.");
cv::gimpl::convertInt64ToInt32(v.GetTensorMutableData<int64_t>(),
reinterpret_cast<int*>(mat.data),
mat.total());
break;
}
default: GAPI_Assert(false && "ONNX. Unsupported data type");
}
}
inline std::vector<int64_t> toORT(const cv::MatSize &sz) {
@@ -161,12 +219,13 @@ inline void preprocess(const cv::Mat& src,
const cv::gimpl::onnx::TensorInfo& ti,
cv::Mat& dst) {
GAPI_Assert(src.depth() == CV_32F || src.depth() == CV_8U);
// CNN input type
const auto type = toCV(ti.type);
if (src.depth() == CV_32F) {
// Just pass the tensor as-is.
// No layout or dimension transformations done here!
// TODO: This needs to be aligned across all NN backends.
GAPI_Assert(toCV(ti.type) == CV_32F && "Only 32F model input is supported for 32F data");
GAPI_Assert(type == CV_32F && "Only 32F model input is supported for 32F input data");
const auto tensor_dims = toORT(src.size);
if (tensor_dims.size() == ti.dims.size()) {
for (size_t i = 0; i < ti.dims.size(); ++i) {
@@ -180,18 +239,21 @@ inline void preprocess(const cv::Mat& src,
dst = src;
} else {
// 8U input: full preprocessing path
GAPI_Assert(src.depth() == CV_8U && "Only 8U data type is supported for preproc");
GAPI_Assert(ti.dims.size() == 4u && "Only NCHW/NHWC layouts are supported for preproc");
GAPI_Assert(src.depth() == CV_8U && "Only 8U data type is supported for preproc");
GAPI_Assert((ti.dims.size() == 4u || ti.dims.size() == 3u)
&& "Only NCHW/NHWC/CHW/HWC layouts are supported for preproc");
const auto ddepth = toCV(ti.type);
GAPI_Assert((ddepth == CV_8U || ddepth == CV_32F)
&& "Only 8U and 32F model input is supported for 8U data");
const bool with_batch = ti.dims.size() == 4u ? true : false;
const int shift = with_batch ? 0 : 1;
GAPI_Assert((type == CV_8U || type == CV_32F)
&& "Only 8U and 32F model input is supported for 8U input data");
// Assess the expected input layout
const bool is_hwc = [&](int ch) {
if (ti.is_grayscale) return false; // 1,1,h,w
else if (ti.dims[3] == ch) return true; // _,_,_,c
else if (ti.dims[1] == ch) return false; // _,c,_,_
if (ti.is_grayscale) return false; // 1,1,h,w
else if (ti.dims[3 - shift] == ch) return true; // ?,_,_,c
else if (ti.dims[1 - shift] == ch) return false; // ?,c,_,_
else cv::util::throw_error(std::logic_error("Couldn't identify input tensor layout"));
} (src.channels());
@@ -212,15 +274,15 @@ inline void preprocess(const cv::Mat& src,
new_w = src.cols;
} else {
// take h & w from the ONNX tensor info
new_h = ti.dims[is_hwc ? 1 : 2];
new_w = ti.dims[is_hwc ? 2 : 3];
new_h = ti.dims[(is_hwc ? 1 : 2) - shift];
new_w = ti.dims[(is_hwc ? 2 : 3) - shift];
}
GAPI_Assert(new_h != -1 && new_w != -1);
cv::Mat rsz, pp;
cv::resize(csc, rsz, cv::Size(new_w, new_h));
if (src.depth() == CV_8U && ddepth == CV_32F) {
rsz.convertTo(pp, ddepth, ti.normalize ? 1.f / 255 : 1.f);
if (src.depth() == CV_8U && type == CV_32F) {
rsz.convertTo(pp, type, ti.normalize ? 1.f / 255 : 1.f);
if (ti.mstd.has_value()) {
pp -= ti.mstd->mean;
pp /= ti.mstd->stdev;
@@ -231,7 +293,7 @@ inline void preprocess(const cv::Mat& src,
if (!is_hwc && new_c > 1) {
// Convert to CHW
dst.create(cv::Size(new_w, new_h * new_c), ddepth);
dst.create(cv::Size(new_w, new_h * new_c), type);
std::vector<cv::Mat> planes(new_c);
for (int ch = 0; ch < new_c; ++ch) {
planes[ch] = dst.rowRange(ch * new_h, (ch + 1) * new_h);
@@ -246,8 +308,12 @@ inline void preprocess(const cv::Mat& src,
if (ti.is_dynamic) {
// Reshape to input dimensions
const std::vector<int> out_dims = is_hwc
? std::vector<int>{1, new_h, new_w, new_c}
: std::vector<int>{1, new_c, new_h, new_w};
? with_batch
? std::vector<int>{1, new_h, new_w, new_c}
: std::vector<int>{new_h, new_w, new_c}
: with_batch
? std::vector<int>{1, new_c, new_h, new_w}
: std::vector<int>{new_c, new_h, new_w};
dst = dst.reshape(1, out_dims);
} else {
// Reshape to ONNX dimensions (no -1s there!)
@@ -256,6 +322,26 @@ inline void preprocess(const cv::Mat& src,
}
}
void preprocess(const cv::MediaFrame::View& view,
const cv::GFrameDesc& desc,
cv::Mat& dst) {
// This overload constructs cv::Mat from cv::MediaFrame
switch (desc.fmt) {
case cv::MediaFormat::BGR: {
dst = cv::Mat(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
break;
}
case cv::MediaFormat::NV12: {
const auto y_plane = cv::Mat(desc.size, CV_8UC1, view.ptr[0], view.stride[0]);
const auto uv_plane = cv::Mat(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]);
cvtColorTwoPlane(y_plane, uv_plane, dst, cv::COLOR_YUV2BGR_NV12);
break;
}
default:
GAPI_Assert(false && "Unsupported media format for ONNX backend");
}
}
template <typename T>
inline Ort::Value createTensor(const Ort::MemoryInfo& memory_info,
const cv::gimpl::onnx::TensorInfo& tensor_params,
@@ -278,8 +364,10 @@ inline Ort::Value createTensor(const Ort::MemoryInfo& memory_info,
return createTensor<uint8_t>(memory_info, tensor_params, data);
case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:
return createTensor<float>(memory_info, tensor_params, data);
case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32:
return createTensor<int32_t>(memory_info, tensor_params, data);
default:
GAPI_Assert(false && "Unsupported data type");
GAPI_Assert(false && "ONNX. Unsupported data type");
}
return Ort::Value{nullptr};
}
@@ -297,7 +385,7 @@ struct ONNXUnit {
struct ONNXCallContext {
// Input parameters passed to an inference operation.
std::vector<cv::GArg> args;
cv::GShapes in_shapes;
//FIXME: avoid conversion of arguments from internal representation to OpenCV one on each call
//to OCV kernel. (This can be achieved by a two single time conversions in GCPUExecutable::run,
//once on enter for input and output arguments, and once before return for output arguments only
@@ -312,6 +400,11 @@ struct ONNXCallContext {
const cv::Mat& inMat(std::size_t input) {
return inArg<cv::Mat>(input);
}
const cv::MediaFrame& inFrame(std::size_t input) {
return inArg<cv::MediaFrame>(input);
}
cv::Mat& outMatR(std::size_t output) {
return *cv::util::get<cv::Mat*>(results.at(output));
}
@@ -403,7 +496,8 @@ cv::GArg cv::gimpl::onnx::GONNXExecutable::packArg(const cv::GArg &arg) {
GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT
&& arg.kind != cv::detail::ArgKind::GSCALAR
&& arg.kind != cv::detail::ArgKind::GARRAY
&& arg.kind != cv::detail::ArgKind::GOPAQUE);
&& arg.kind != cv::detail::ArgKind::GOPAQUE
&& arg.kind != cv::detail::ArgKind::GFRAME);
if (arg.kind != cv::detail::ArgKind::GOBJREF) {
util::throw_error(std::logic_error("Inference supports G-types ONLY!"));
@@ -425,6 +519,8 @@ cv::GArg cv::gimpl::onnx::GONNXExecutable::packArg(const cv::GArg &arg) {
// (and constructed by either bindIn/Out or resetInternal)
case GShape::GOPAQUE: return GArg(m_res.slot<cv::detail::OpaqueRef>().at(ref.id));
case GShape::GFRAME: return GArg(m_res.slot<cv::MediaFrame>().at(ref.id));
default:
util::throw_error(std::logic_error("Unsupported GShape type"));
break;
@@ -451,8 +547,16 @@ void cv::gimpl::onnx::GONNXExecutable::run(std::vector<InObj> &&input_objs,
context.args.reserve(op.args.size());
using namespace std::placeholders;
ade::util::transform(op.args,
std::back_inserter(context.args),
std::bind(&GONNXExecutable::packArg, this, _1));
std::back_inserter(context.args),
std::bind(&GONNXExecutable::packArg, this, _1));
// NB: Need to store inputs shape to recognize GFrame/GMat
context.in_shapes.reserve(op.args.size());
ade::util::transform(op.args,
std::back_inserter(context.in_shapes),
[](const cv::GArg& arg) {
return arg.get<cv::gimpl::RcDesc>().shape;
});
// - Output parameters.
for (const auto &out_it : ade::util::indexed(op.outs)) {
@@ -477,7 +581,6 @@ namespace onnx {
ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp)
: params(pp) {
// Validate input parameters before allocating any resources
if (params.num_in > 1u && params.num_in != params.input_names.size()) {
cv::util::throw_error(std::logic_error("Please specify input layer names for "
@@ -491,7 +594,13 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp)
// Create and initialize the ONNX session
Ort::SessionOptions session_options;
this_env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, "");
#ifndef _WIN32
this_session = Ort::Session(this_env, params.model_path.data(), session_options);
#else
std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> converter;
std::wstring w_model_path = converter.from_bytes(params.model_path.data());
this_session = Ort::Session(this_env, w_model_path.data(), session_options);
#endif
this_memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
in_tensor_info = getTensorInfo(INPUT);
@@ -507,6 +616,7 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp)
"Please provide a custom post-processing function "
"(.cfgPostProc) in network parameters"));
}
is_postproc = (params.custom_post_proc != nullptr);
// Update parameters based on session information
if (params.num_in == 1u && params.input_names.empty()) {
@@ -517,8 +627,6 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp)
}
// Validate what is supported currently
GAPI_Assert(params.const_inputs.empty()
&& "Const inputs are not currently supported");
GAPI_Assert(std::all_of(in_tensor_info.begin(),
in_tensor_info.end(),
[](const cv::gimpl::onnx::TensorInfo &p) {
@@ -547,6 +655,17 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp)
}
}
if (!params.const_inputs.empty()) {
// Form input names order without const input names
in_names_without_const.clear();
std::copy_if(params.input_names.begin(), params.input_names.end(),
std::back_inserter(in_names_without_const),
[&](const std::string& name) {
const auto it = params.const_inputs.find(name);
return it == params.const_inputs.end();
});
}
// Pre-allocate vectors (not buffers) for runtime info
in_data.resize(params.num_in);
out_data.resize(params.num_out);
@@ -580,9 +699,9 @@ std::vector<TensorInfo> ONNXCompiled::getTensorInfo(TensorPosition pos) {
}
cv::GMatDesc ONNXCompiled::outMeta(int idx) const {
if (is_dynamic) {
if (is_dynamic || is_postproc) {
GAPI_Assert(!params.out_metas.empty()
&& "Metadata must be specified if NN has dynamic inputs!");
&& "Metadata must be specified if NN has dynamic inputs or post-processing function is used!");
return params.out_metas.at(idx);
}
const auto ort_idx = getIdxByName(out_tensor_info, params.output_names[idx]);
@@ -590,13 +709,32 @@ cv::GMatDesc ONNXCompiled::outMeta(int idx) const {
toCV(out_tensor_info[ort_idx].dims));
}
void ONNXCompiled::setInput(int i, const cv::Mat &m) {
const auto in_idx = i;
void ONNXCompiled::setInput(int in_idx, const cv::Mat &m) {
GAPI_Assert(!m.empty() && "Input data can't be empty!");
const auto in_name = params.input_names[in_idx];
const auto ort_idx = getIdxByName(in_tensor_info, in_name);
preprocess(m, in_tensor_info[ort_idx], in_data[in_idx]);
}
void ONNXCompiled::extractMat(ONNXCallContext &ctx, const size_t in_idx, Views& views) {
switch (ctx.in_shapes[in_idx]) {
case cv::GShape::GFRAME: {
const cv::MediaFrame& frame = ctx.inFrame(in_idx);
views.emplace_back(new cv::MediaFrame::View(frame.access(cv::MediaFrame::Access::R)));
GAPI_Assert(views.size() <= numInputs());
preprocess(*views.back(), frame.desc(), exMat);
break;
}
case cv::GShape::GMAT: {
exMat = ctx.inMat(in_idx);
break;
}
default: {
GAPI_Assert("Unsupported input shape for ONNX backend");
}
}
}
void ONNXCompiled::setOutput(int i, cv::Mat &m) {
// FIXME: No need in double-indexing?
out_data[i] = m;
@@ -613,9 +751,12 @@ void ONNXCompiled::Run(const std::vector<cv::Mat>& ins,
const std::vector<cv::Mat>& outs) {
std::vector<Ort::Value> in_tensors, out_tensors;
auto in_run_names = getCharNames(params.input_names);
for (const auto it : ade::util::indexed(params.input_names)) {
// Layer names order for run
auto input_names = (in_names_without_const.empty() && params.const_inputs.empty())
? params.input_names
: in_names_without_const;
// Creates tensors for unique names that don't contain constant input
for (const auto it : ade::util::indexed(input_names)) {
auto i = ade::util::index(it);
auto in_name = ade::util::value(it);
const auto idx = getIdxByName(in_tensor_info, in_name);
@@ -624,7 +765,19 @@ void ONNXCompiled::Run(const std::vector<cv::Mat>& ins,
ins[i]));
}
if (!is_dynamic) {
for (auto &&c_in_pair : params.const_inputs) {
const auto idx = getIdxByName(in_tensor_info, c_in_pair.first);
in_tensors.emplace_back(createTensor(this_memory_info,
in_tensor_info[idx],
c_in_pair.second.first));
// Puts const input names in sequence for Run
// ONNXRuntime can match input tensors to CNN inputs by names
input_names.emplace_back(c_in_pair.first);
}
GAPI_Assert(input_names.size() == this_session.GetInputCount());
auto in_run_names = getCharNames(input_names);
if (!is_dynamic && !is_postproc) {
// Easy path - just run the session which is bound to G-API's
// internal data
for (auto i : ade::util::iota(params.output_names.size())) {
@@ -636,7 +789,7 @@ void ONNXCompiled::Run(const std::vector<cv::Mat>& ins,
this_session.Run(Ort::RunOptions{nullptr},
in_run_names.data(),
&in_tensors.front(),
params.input_names.size(),
input_names.size(),
out_run_names.data(),
&out_tensors.front(),
params.output_names.size());
@@ -644,14 +797,17 @@ void ONNXCompiled::Run(const std::vector<cv::Mat>& ins,
// Hard path - run session & user-defined post-processing
// NOTE: use another list of output names here
std::vector<const char*> out_names;
for (auto &&ti : out_tensor_info) {
out_names.push_back(ti.name.c_str());
}
out_names.reserve(outs.size());
params.names_to_remap.empty()
? ade::util::transform(out_tensor_info, std::back_inserter(out_names),
[] (const TensorInfo& ti) { return ti.name.c_str(); })
: ade::util::transform(params.names_to_remap, std::back_inserter(out_names),
[] (const std::string& ntr) { return ntr.c_str(); });
auto outputs = this_session.Run(Ort::RunOptions{nullptr},
in_run_names.data(),
&in_tensors.front(),
params.input_names.size(),
input_names.size(),
out_names.data(),
out_names.size());
std::unordered_map<std::string, cv::Mat> onnx_outputs;
@@ -659,18 +815,32 @@ void ONNXCompiled::Run(const std::vector<cv::Mat>& ins,
GAPI_Assert(outputs.size() == out_names.size());
// Fill in ONNX tensors
for (auto &&iter : ade::util::zip(ade::util::toRange(out_tensor_info),
for (auto &&iter : ade::util::zip(ade::util::toRange(out_names),
ade::util::toRange(outputs))) {
const auto &out_name = std::get<0>(iter).name;
const auto &out_name = std::get<0>(iter);
auto &out_tensor = std::get<1>(iter);
onnx_outputs[out_name] = toCV(out_tensor);
copyFromONNX(out_tensor, onnx_outputs[out_name]);
}
std::vector<uint8_t *> tracked_mat_ptrs;
// Fill in G-API outputs
for (auto &&it: ade::util::indexed(params.output_names)) {
gapi_outputs[ade::util::value(it)] = outs[ade::util::index(it)];
tracked_mat_ptrs.push_back(outs[ade::util::index(it)].data);
}
params.custom_post_proc(onnx_outputs, gapi_outputs);
// Checking for possible data reallocation after remapping
GAPI_Assert(tracked_mat_ptrs.size() == params.output_names.size());
for (auto &&iter : ade::util::zip(ade::util::toRange(tracked_mat_ptrs),
ade::util::toRange(params.output_names))) {
const auto &original_data = std::get<0>(iter);
const auto &received_data = gapi_outputs.at(std::get<1>(iter)).data;
if (original_data != received_data) {
cv::util::throw_error
(std::logic_error
("OpenCV kernel output parameter was reallocated after remapping of ONNX output. \n"
"Incorrect logic in remapping function?"));
}
}
}
}
@@ -678,6 +848,23 @@ void ONNXCompiled::run() {
Run(in_data, out_data);
}
static void checkInputMeta(const cv::GMetaArg mm) {
switch (mm.index()) {
case cv::GMetaArg::index_of<cv::GMatDesc>(): break;
case cv::GMetaArg::index_of<cv::GFrameDesc>(): {
const auto &meta = util::get<cv::GFrameDesc>(mm);
switch (meta.fmt) {
case cv::MediaFormat::NV12: break;
case cv::MediaFormat::BGR: break;
default:
GAPI_Assert(false && "Unsupported media format for ONNX backend");
} break;
} break;
default:
util::throw_error(std::runtime_error("Unsupported input meta for ONNX backend"));
}
}
struct Infer: public cv::detail::KernelTag {
using API = cv::GInferBase;
static cv::gapi::GBackend backend() { return cv::gapi::onnx::backend(); }
@@ -695,8 +882,7 @@ struct Infer: public cv::detail::KernelTag {
GAPI_Assert(uu.oc->numInputs() == in_metas.size()
&& "Known input layers count doesn't match input meta count");
for (auto &&mm : in_metas) {
GAPI_Assert(util::holds_alternative<cv::GMatDesc>(mm)
&& "Non-GMat inputs are not supported");
checkInputMeta(mm);
}
for (auto &&idx : ade::util::iota(uu.oc->numOutputs())) {
result.emplace_back(uu.oc->outMeta(idx));
@@ -705,8 +891,10 @@ struct Infer: public cv::detail::KernelTag {
}
static void run(const ONNXUnit &uu, ONNXCallContext &ctx) {
Views views;
for (auto &&idx : ade::util::iota(uu.oc->numInputs())) {
uu.oc->setInput(idx, ctx.inMat(idx));
uu.oc->extractMat(ctx, idx, views);
uu.oc->setInput(idx, uu.oc->exMat);
}
for (auto &&idx : ade::util::iota(uu.oc->numOutputs())) {
uu.oc->setOutput(idx, ctx.outMatR(idx));
@@ -730,7 +918,7 @@ struct InferROI: public cv::detail::KernelTag {
const auto &uu = gm.metadata(nh).get<ONNXUnit>();
GAPI_Assert(1u == uu.oc->numInputs());
GAPI_Assert(2u == in_metas.size());
checkInputMeta(in_metas.at(1));
for (auto &&idx : ade::util::iota(uu.oc->numOutputs())) {
result.emplace_back(uu.oc->outMeta(idx));
}
@@ -738,12 +926,12 @@ struct InferROI: public cv::detail::KernelTag {
}
static void run(const ONNXUnit &uu, ONNXCallContext &ctx) {
Views views;
// non-generic version for now, per the InferROI's definition
GAPI_Assert(uu.oc->numInputs() == 1u);
const auto& this_roi = ctx.inArg<cv::detail::OpaqueRef>(0).rref<cv::Rect>();
const auto this_mat = ctx.inMat(1);
uu.oc->setInput(0, this_mat(this_roi));
uu.oc->extractMat(ctx, 1, views);
uu.oc->setInput(0, uu.oc->exMat(this_roi));
for (auto &&idx : ade::util::iota(uu.oc->numOutputs())) {
uu.oc->setOutput(idx, ctx.outMatR(idx));
}
@@ -769,10 +957,8 @@ struct InferList: public cv::detail::KernelTag {
&& "Known input layers count doesn't match input meta count");
for (auto i : ade::util::iota(uu.oc->numInputs())) {
const auto & mm = in_metas[i + 1];
GAPI_Assert(util::holds_alternative<cv::GMatDesc>(mm)
&& "Non-GMat inputs are not supported");
const auto &mm = in_metas[i + 1];
checkInputMeta(mm);
}
// roi-list version is much easier at the moment.
@@ -784,19 +970,20 @@ struct InferList: public cv::detail::KernelTag {
}
static void run(const ONNXUnit &uu, ONNXCallContext &ctx) {
Views views;
// non-generic version for now:
// - assumes input 0 is always ROI list
// - assumes all inputs/outputs are always Mats
GAPI_Assert(uu.oc->numInputs() == 1); // roi list is not counted in net's inputs
const auto& in_roi_vec = ctx.inArg<cv::detail::VectorRef>(0u).rref<cv::Rect>();
const cv::Mat this_mat = ctx.inMat(1u);
for (auto i : ade::util::iota(uu.oc->numOutputs())) {
ctx.outVecR<cv::Mat>(i).clear();
}
uu.oc->extractMat(ctx, 1, views);
for (const auto &rc : in_roi_vec) {
uu.oc->setInput(0, this_mat(rc));
uu.oc->setInput(0, uu.oc->exMat(rc));
std::vector<cv::Mat> out_mats(uu.oc->numOutputs());
for (auto i : ade::util::iota(uu.oc->numOutputs())) {
out_mats[i] = uu.oc->allocOutput(i);
@@ -837,10 +1024,30 @@ struct InferList2: public cv::detail::KernelTag {
// FIXME: this is filtering not done, actually! GArrayDesc has
// no hint for type!
const auto &mm_0 = in_metas[0u];
const auto &meta_0 = util::get<cv::GMatDesc>(mm_0);
GAPI_Assert( !meta_0.isND()
&& !meta_0.planar
&& "Only images are supported as the 0th argument");
switch (in_metas[0u].index()) {
case cv::GMetaArg::index_of<cv::GMatDesc>(): {
const auto &meta_0 = util::get<cv::GMatDesc>(mm_0);
GAPI_Assert( !meta_0.isND()
&& !meta_0.planar
&& "Only images are supported as the 0th argument");
break;
}
case cv::GMetaArg::index_of<cv::GFrameDesc>(): {
const auto &meta_0 = util::get<cv::GFrameDesc>(mm_0);
GAPI_Assert( (meta_0.fmt == cv::MediaFormat::BGR)
|| (meta_0.fmt == cv::MediaFormat::NV12));
GAPI_Assert((meta_0.size.height !=0) && (meta_0.size.width !=0));
break;
}
default:
util::throw_error(std::runtime_error("Unsupported input meta for ONNX backend"));
}
if (util::holds_alternative<cv::GMatDesc>(mm_0)) {
const auto &meta_0 = util::get<cv::GMatDesc>(mm_0);
GAPI_Assert( !meta_0.isND()
&& !meta_0.planar
&& "Only images are supported as the 0th argument");
}
for (auto i : ade::util::iota(uu.oc->numInputs())) {
const auto &mm = in_metas[i + 1];
GAPI_Assert(util::holds_alternative<cv::GArrayDesc>(mm)
@@ -856,11 +1063,11 @@ struct InferList2: public cv::detail::KernelTag {
}
static void run(const ONNXUnit &uu, ONNXCallContext &ctx) {
Views views;
GAPI_Assert(ctx.args.size() > 1u
&& "This operation must have at least two arguments");
uu.oc->extractMat(ctx, 0, views);
// Since we do a ROI list inference, always assume our input buffer is image
const cv::Mat mat_0 = ctx.inMat(0u);
// Take the next argument, which must be vector (of any kind).
// Use this only to obtain the ROI list size (sizes of all
// other vectors must be equal to this one)
@@ -885,7 +1092,7 @@ struct InferList2: public cv::detail::KernelTag {
if (this_vec.holds<cv::Rect>()) {
// ROI case - create an ROI blob
const auto &vec = this_vec.rref<cv::Rect>();
uu.oc->setInput(in_idx, mat_0(vec[list_idx]));
uu.oc->setInput(in_idx, uu.oc->exMat(vec[list_idx]));
} else if (this_vec.holds<cv::Mat>()) {
// Mat case - create a regular blob
// FIXME: NOW Assume Mats are always BLOBS (not
modules/gapi/src/backends/plaidml/gplaidmlbackend.cpp
+10 -9
View File
@@ -198,9 +198,6 @@ void cv::gimpl::GPlaidMLExecutable::run(std::vector<InObj> &&input_objs,
exec_->run();
for (auto& it : output_objs) bindOutArg(it.first, it.second);
// FIXME:
// PlaidML backend haven't been updated with RMat support
}
void cv::gimpl::GPlaidMLExecutable::bindInArg(const RcDesc &rc, const GRunArg &arg)
@@ -215,10 +212,12 @@ void cv::gimpl::GPlaidMLExecutable::bindInArg(const RcDesc &rc, const GRunArg &
switch (arg.index())
{
case GRunArg::index_of<cv::Mat>() :
case GRunArg::index_of<cv::RMat>():
{
auto& arg_mat = util::get<cv::Mat>(arg);
binder_->input(it->second).copy_from(arg_mat.data);
auto& rmat = cv::util::get<cv::RMat>(arg);
auto view = rmat.access(cv::RMat::Access::R);
auto mat = cv::gimpl::asMat(view);
binder_->input(it->second).copy_from(mat.data);
}
break;
default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?"));
@@ -243,10 +242,12 @@ void cv::gimpl::GPlaidMLExecutable::bindOutArg(const RcDesc &rc, const GRunArgP
switch (arg.index())
{
case GRunArgP::index_of<cv::Mat*>() :
case GRunArgP::index_of<cv::RMat*>() :
{
auto& arg_mat = *util::get<cv::Mat*>(arg);
binder_->output(it->second).copy_into(arg_mat.data);
auto& rmat = *cv::util::get<cv::RMat*>(arg);
auto view = rmat.access(cv::RMat::Access::W);
auto mat = cv::gimpl::asMat(view);
binder_->output(it->second).copy_into(mat.data);
}
break;
default: util::throw_error(std::logic_error("content type of the runtime argument does not match to resource description ?"));
modules/gapi/src/backends/python/gpythonbackend.cpp
+261
View File
@@ -0,0 +1,261 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2021 Intel Corporation
#include <ade/util/zip_range.hpp> // zip_range, indexed
#include <opencv2/gapi/util/throw.hpp> // throw_error
#include <opencv2/gapi/python/python.hpp>
#include "api/gbackend_priv.hpp"
#include "backends/common/gbackend.hpp"
cv::gapi::python::GPythonKernel::GPythonKernel(cv::gapi::python::Impl run)
: m_run(run)
{
}
cv::GRunArgs cv::gapi::python::GPythonKernel::operator()(const cv::gapi::python::GPythonContext& ctx)
{
return m_run(ctx);
}
cv::gapi::python::GPythonFunctor::GPythonFunctor(const char* id,
const cv::gapi::python::GPythonFunctor::Meta &meta,
const cv::gapi::python::Impl& impl)
: gapi::GFunctor(id), impl_{GPythonKernel{impl}, meta}
{
}
cv::GKernelImpl cv::gapi::python::GPythonFunctor::impl() const
{
return impl_;
}
cv::gapi::GBackend cv::gapi::python::GPythonFunctor::backend() const
{
return cv::gapi::python::backend();
}
namespace {
struct PythonUnit
{
static const char *name() { return "PythonUnit"; }
cv::gapi::python::GPythonKernel kernel;
};
using PythonModel = ade::TypedGraph
< cv::gimpl::Op
, PythonUnit
>;
using ConstPythonModel = ade::ConstTypedGraph
< cv::gimpl::Op
, PythonUnit
>;
class GPythonExecutable final: public cv::gimpl::GIslandExecutable
{
virtual void run(std::vector<InObj> &&,
std::vector<OutObj> &&) override;
virtual bool allocatesOutputs() const override { return true; }
// Return an empty RMat since we will reuse the input.
// There is no need to allocate and copy 4k image here.
virtual cv::RMat allocate(const cv::GMatDesc&) const override { return {}; }
virtual bool canReshape() const override { return true; }
virtual void reshape(ade::Graph&, const cv::GCompileArgs&) override {
// Do nothing here
}
public:
GPythonExecutable(const ade::Graph &,
const std::vector<ade::NodeHandle> &);
const ade::Graph& m_g;
cv::gimpl::GModel::ConstGraph m_gm;
cv::gapi::python::GPythonKernel m_kernel;
ade::NodeHandle m_op;
cv::GTypesInfo m_out_info;
cv::GMetaArgs m_in_metas;
cv::gimpl::Mag m_res;
};
static cv::GArg packArg(cv::gimpl::Mag& m_res, const cv::GArg &arg)
{
// No API placeholders allowed at this point
// FIXME: this check has to be done somewhere in compilation stage.
GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT
&& arg.kind != cv::detail::ArgKind::GSCALAR
&& arg.kind != cv::detail::ArgKind::GARRAY
&& arg.kind != cv::detail::ArgKind::GOPAQUE
&& arg.kind != cv::detail::ArgKind::GFRAME);
if (arg.kind != cv::detail::ArgKind::GOBJREF)
{
// All other cases - pass as-is, with no transformations to GArg contents.
return arg;
}
GAPI_Assert(arg.kind == cv::detail::ArgKind::GOBJREF);
// Wrap associated CPU object (either host or an internal one)
// FIXME: object can be moved out!!! GExecutor faced that.
const cv::gimpl::RcDesc &ref = arg.get<cv::gimpl::RcDesc>();
switch (ref.shape)
{
case cv::GShape::GMAT: return cv::GArg(m_res.slot<cv::Mat>() [ref.id]);
case cv::GShape::GSCALAR: return cv::GArg(m_res.slot<cv::Scalar>()[ref.id]);
// Note: .at() is intentional for GArray and GOpaque as objects MUST be already there
// (and constructed by either bindIn/Out or resetInternal)
case cv::GShape::GARRAY: return cv::GArg(m_res.slot<cv::detail::VectorRef>().at(ref.id));
case cv::GShape::GOPAQUE: return cv::GArg(m_res.slot<cv::detail::OpaqueRef>().at(ref.id));
case cv::GShape::GFRAME: return cv::GArg(m_res.slot<cv::MediaFrame>().at(ref.id));
default:
cv::util::throw_error(std::logic_error("Unsupported GShape type"));
break;
}
}
static void writeBack(cv::GRunArg& arg, cv::GRunArgP& out)
{
switch (arg.index())
{
case cv::GRunArg::index_of<cv::Mat>():
{
auto& rmat = *cv::util::get<cv::RMat*>(out);
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(cv::util::get<cv::Mat>(arg));
break;
}
case cv::GRunArg::index_of<cv::Scalar>():
{
*cv::util::get<cv::Scalar*>(out) = cv::util::get<cv::Scalar>(arg);
break;
}
case cv::GRunArg::index_of<cv::detail::OpaqueRef>():
{
auto& oref = cv::util::get<cv::detail::OpaqueRef>(arg);
cv::util::get<cv::detail::OpaqueRef>(out).mov(oref);
break;
}
case cv::GRunArg::index_of<cv::detail::VectorRef>():
{
auto& vref = cv::util::get<cv::detail::VectorRef>(arg);
cv::util::get<cv::detail::VectorRef>(out).mov(vref);
break;
}
default:
GAPI_Assert(false && "Unsupported output type");
}
}
void GPythonExecutable::run(std::vector<InObj> &&input_objs,
std::vector<OutObj> &&output_objs)
{
const auto &op = m_gm.metadata(m_op).get<cv::gimpl::Op>();
for (auto& it : input_objs) cv::gimpl::magazine::bindInArg(m_res, it.first, it.second);
using namespace std::placeholders;
cv::GArgs inputs;
ade::util::transform(op.args,
std::back_inserter(inputs),
std::bind(&packArg, std::ref(m_res), _1));
cv::gapi::python::GPythonContext ctx{inputs, m_in_metas, m_out_info};
auto outs = m_kernel(ctx);
for (auto&& it : ade::util::zip(outs, output_objs))
{
writeBack(std::get<0>(it), std::get<1>(it).second);
}
}
class GPythonBackendImpl final: public cv::gapi::GBackend::Priv
{
virtual void unpackKernel(ade::Graph &graph,
const ade::NodeHandle &op_node,
const cv::GKernelImpl &impl) override
{
PythonModel gm(graph);
const auto &kernel = cv::util::any_cast<cv::gapi::python::GPythonKernel>(impl.opaque);
gm.metadata(op_node).set(PythonUnit{kernel});
}
virtual EPtr compile(const ade::Graph &graph,
const cv::GCompileArgs &,
const std::vector<ade::NodeHandle> &nodes) const override
{
return EPtr{new GPythonExecutable(graph, nodes)};
}
virtual bool controlsMerge() const override
{
return true;
}
virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &,
const ade::NodeHandle &,
const ade::NodeHandle &,
const ade::NodeHandle &) const override
{
return false;
}
};
GPythonExecutable::GPythonExecutable(const ade::Graph& g,
const std::vector<ade::NodeHandle>& nodes)
: m_g(g), m_gm(m_g)
{
using namespace cv::gimpl;
const auto is_op = [this](const ade::NodeHandle &nh)
{
return m_gm.metadata(nh).get<NodeType>().t == NodeType::OP;
};
auto it = std::find_if(nodes.begin(), nodes.end(), is_op);
GAPI_Assert(it != nodes.end() && "No operators found for this island?!");
ConstPythonModel cag(m_g);
m_op = *it;
m_kernel = cag.metadata(m_op).get<PythonUnit>().kernel;
// Ensure this the only op in the graph
if (std::any_of(it+1, nodes.end(), is_op))
{
cv::util::throw_error
(std::logic_error
("Internal error: Python subgraph has multiple operations"));
}
m_out_info.reserve(m_op->outEdges().size());
for (const auto &e : m_op->outEdges())
{
const auto& out_data = m_gm.metadata(e->dstNode()).get<cv::gimpl::Data>();
m_out_info.push_back(cv::GTypeInfo{out_data.shape, out_data.kind, out_data.ctor});
}
const auto& op = m_gm.metadata(m_op).get<cv::gimpl::Op>();
m_in_metas.resize(op.args.size());
GAPI_Assert(m_op->inEdges().size() > 0);
for (const auto &in_eh : m_op->inEdges())
{
const auto& input_port = m_gm.metadata(in_eh).get<Input>().port;
const auto& input_nh = in_eh->srcNode();
const auto& input_meta = m_gm.metadata(input_nh).get<Data>().meta;
m_in_metas.at(input_port) = input_meta;
}
}
} // anonymous namespace
cv::gapi::GBackend cv::gapi::python::backend()
{
static cv::gapi::GBackend this_backend(std::make_shared<GPythonBackendImpl>());
return this_backend;
}
modules/gapi/src/backends/render/grenderocv.cpp
+76 -1
View File
@@ -114,8 +114,83 @@ GAPI_OCV_KERNEL_ST(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12, RenderOC
}
};
GAPI_OCV_KERNEL_ST(RenderFrameOCVImpl, cv::gapi::wip::draw::GRenderFrame, RenderOCVState)
{
static void run(const cv::MediaFrame & in,
const cv::gapi::wip::draw::Prims & prims,
cv::MediaFrame & out,
RenderOCVState & state)
{
GAPI_Assert(in.desc().fmt == cv::MediaFormat::NV12);
// FIXME: consider a better approach (aka native inplace operation)
// Non-intuitive logic with shared_ptr Priv class
out = in;
auto desc = out.desc();
auto w_out = out.access(cv::MediaFrame::Access::W);
auto out_y = cv::Mat(desc.size, CV_8UC1, w_out.ptr[0], w_out.stride[0]);
auto out_uv = cv::Mat(desc.size / 2, CV_8UC2, w_out.ptr[1], w_out.stride[1]);
auto r_in = in.access(cv::MediaFrame::Access::R);
auto in_y = cv::Mat(desc.size, CV_8UC1, r_in.ptr[0], r_in.stride[0]);
auto in_uv = cv::Mat(desc.size / 2, CV_8UC2, r_in.ptr[1], r_in.stride[1]);
/* FIXME How to render correctly on NV12 format ?
*
* Rendering on NV12 via OpenCV looks like this:
*
* y --------> 1)(NV12 -> YUV) -> yuv -> 2)draw -> yuv -> 3)split -------> out_y
* ^ |
* | |
* uv -------------- `----------> out_uv
*
*
* 1) Collect yuv mat from two planes, uv plain in two times less than y plane
* so, upsample uv in two times, with bilinear interpolation
*
* 2) Render primitives on YUV
*
* 3) Convert yuv to NV12 (using bilinear interpolation)
*
*/
// NV12 -> YUV
cv::Mat upsample_uv, yuv;
cv::resize(in_uv, upsample_uv, in_uv.size() * 2, cv::INTER_LINEAR);
cv::merge(std::vector<cv::Mat>{in_y, upsample_uv}, yuv);
cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, state.ftpr);
// YUV -> NV12
cv::Mat out_u, out_v, uv_plane;
std::vector<cv::Mat> chs = { out_y, out_u, out_v };
cv::split(yuv, chs);
cv::merge(std::vector<cv::Mat>{chs[1], chs[2]}, uv_plane);
cv::resize(uv_plane, out_uv, uv_plane.size() / 2, cv::INTER_LINEAR);
}
static void setup(const cv::GFrameDesc& /* in_nv12 */,
const cv::GArrayDesc& /* prims */,
std::shared_ptr<RenderOCVState>&state,
const cv::GCompileArgs & args)
{
using namespace cv::gapi::wip::draw;
auto has_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
state = std::make_shared<RenderOCVState>();
if (has_freetype_font)
{
state->ftpr = std::make_shared<FTTextRender>(has_freetype_font->path);
}
}
};
cv::gapi::GKernelPackage cv::gapi::render::ocv::kernels()
{
const static auto pkg = cv::gapi::kernels<RenderBGROCVImpl, RenderNV12OCVImpl>();
const static auto pkg = cv::gapi::kernels<RenderBGROCVImpl, RenderNV12OCVImpl, RenderFrameOCVImpl>();
return pkg;
}
modules/gapi/src/backends/streaming/gstreamingbackend.cpp
+438
View File
@@ -0,0 +1,438 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#include <mutex>
#if !defined(GAPI_STANDALONE)
#include <opencv2/imgproc.hpp>
#endif // !defined(GAPI_STANDALONE)
#include <opencv2/gapi/util/throw.hpp> // throw_error
#include <opencv2/gapi/streaming/format.hpp> // kernels
#include "logger.hpp"
#include "api/gbackend_priv.hpp"
#include "backends/common/gbackend.hpp"
#include "gstreamingbackend.hpp"
#include "gstreamingkernel.hpp"
namespace {
struct StreamingCreateFunction
{
static const char *name() { return "StreamingCreateFunction"; }
cv::gapi::streaming::CreateActorFunction createActorFunction;
};
using StreamingGraph = ade::TypedGraph
< cv::gimpl::Op
, StreamingCreateFunction
>;
using ConstStreamingGraph = ade::ConstTypedGraph
< cv::gimpl::Op
, StreamingCreateFunction
>;
class GStreamingIntrinExecutable final: public cv::gimpl::GIslandExecutable
{
virtual void run(std::vector<InObj> &&,
std::vector<OutObj> &&) override {
GAPI_Assert(false && "Not implemented");
}
virtual void run(GIslandExecutable::IInput &in,
GIslandExecutable::IOutput &out) override;
virtual bool allocatesOutputs() const override { return true; }
// Return an empty RMat since we will reuse the input.
// There is no need to allocate and copy 4k image here.
virtual cv::RMat allocate(const cv::GMatDesc&) const override { return {}; }
virtual bool canReshape() const override { return true; }
virtual void reshape(ade::Graph&, const cv::GCompileArgs&) override {
// Do nothing here
}
public:
GStreamingIntrinExecutable(const ade::Graph &,
const cv::GCompileArgs &,
const std::vector<ade::NodeHandle> &);
const ade::Graph& m_g;
cv::gimpl::GModel::ConstGraph m_gm;
cv::gapi::streaming::IActor::Ptr m_actor;
};
void GStreamingIntrinExecutable::run(GIslandExecutable::IInput &in,
GIslandExecutable::IOutput &out)
{
m_actor->run(in, out);
}
class GStreamingBackendImpl final: public cv::gapi::GBackend::Priv
{
virtual void unpackKernel(ade::Graph &graph,
const ade::NodeHandle &op_node,
const cv::GKernelImpl &impl) override
{
StreamingGraph gm(graph);
const auto &kimpl = cv::util::any_cast<cv::gapi::streaming::GStreamingKernel>(impl.opaque);
gm.metadata(op_node).set(StreamingCreateFunction{kimpl.createActorFunction});
}
virtual EPtr compile(const ade::Graph &graph,
const cv::GCompileArgs &args,
const std::vector<ade::NodeHandle> &nodes) const override
{
return EPtr{new GStreamingIntrinExecutable(graph, args, nodes)};
}
virtual bool controlsMerge() const override
{
return true;
}
virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &,
const ade::NodeHandle &,
const ade::NodeHandle &,
const ade::NodeHandle &) const override
{
return false;
}
};
GStreamingIntrinExecutable::GStreamingIntrinExecutable(const ade::Graph& g,
const cv::GCompileArgs& args,
const std::vector<ade::NodeHandle>& nodes)
: m_g(g), m_gm(m_g)
{
using namespace cv::gimpl;
const auto is_op = [this](const ade::NodeHandle &nh)
{
return m_gm.metadata(nh).get<NodeType>().t == NodeType::OP;
};
auto it = std::find_if(nodes.begin(), nodes.end(), is_op);
GAPI_Assert(it != nodes.end() && "No operators found for this island?!");
ConstStreamingGraph cag(m_g);
m_actor = cag.metadata(*it).get<StreamingCreateFunction>().createActorFunction(args);
// Ensure this the only op in the graph
if (std::any_of(it+1, nodes.end(), is_op))
{
cv::util::throw_error
(std::logic_error
("Internal error: Streaming subgraph has multiple operations"));
}
}
} // anonymous namespace
cv::gapi::GBackend cv::gapi::streaming::backend()
{
static cv::gapi::GBackend this_backend(std::make_shared<GStreamingBackendImpl>());
return this_backend;
}
struct Copy: public cv::detail::KernelTag
{
using API = cv::gimpl::streaming::GCopy;
static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
class Actor final: public cv::gapi::streaming::IActor
{
public:
explicit Actor(const cv::GCompileArgs&) {}
virtual void run(cv::gimpl::GIslandExecutable::IInput &in,
cv::gimpl::GIslandExecutable::IOutput &out) override;
};
static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
{
return cv::gapi::streaming::IActor::Ptr(new Actor(args));
}
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
};
void Copy::Actor::run(cv::gimpl::GIslandExecutable::IInput &in,
cv::gimpl::GIslandExecutable::IOutput &out)
{
const auto in_msg = in.get();
if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg))
{
out.post(cv::gimpl::EndOfStream{});
return;
}
const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg);
GAPI_Assert(in_args.size() == 1u);
const auto& in_arg = in_args[0];
auto out_arg = out.get(0);
using cv::util::get;
switch (in_arg.index()) {
case cv::GRunArg::index_of<cv::RMat>():
*get<cv::RMat*>(out_arg) = get<cv::RMat>(in_arg);
break;
case cv::GRunArg::index_of<cv::MediaFrame>():
*get<cv::MediaFrame*>(out_arg) = get<cv::MediaFrame>(in_arg);
break;
// FIXME: Add support for remaining types
default:
GAPI_Assert(false && "Copy: unsupported data type");
}
out.meta(out_arg, in_arg.meta);
out.post(std::move(out_arg));
}
cv::gapi::GKernelPackage cv::gimpl::streaming::kernels()
{
return cv::gapi::kernels<Copy>();
}
#if !defined(GAPI_STANDALONE)
class GAccessorActorBase : public cv::gapi::streaming::IActor {
public:
explicit GAccessorActorBase(const cv::GCompileArgs&) {}
virtual void run(cv::gimpl::GIslandExecutable::IInput &in,
cv::gimpl::GIslandExecutable::IOutput &out) override {
const auto in_msg = in.get();
if (cv::util::holds_alternative<cv::gimpl::EndOfStream>(in_msg))
{
out.post(cv::gimpl::EndOfStream{});
return;
}
const cv::GRunArgs &in_args = cv::util::get<cv::GRunArgs>(in_msg);
GAPI_Assert(in_args.size() == 1u);
auto frame = cv::util::get<cv::MediaFrame>(in_args[0]);
cv::GRunArgP out_arg = out.get(0);
auto& rmat = *cv::util::get<cv::RMat*>(out_arg);
extractRMat(frame, rmat);
out.meta(out_arg, in_args[0].meta);
out.post(std::move(out_arg));
}
virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) = 0;
protected:
std::once_flag m_warnFlag;
};
struct GOCVBGR: public cv::detail::KernelTag
{
using API = cv::gapi::streaming::GBGR;
static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
class Actor final: public GAccessorActorBase
{
public:
using GAccessorActorBase::GAccessorActorBase;
virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
};
static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
{
return cv::gapi::streaming::IActor::Ptr(new Actor(args));
}
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
};
void GOCVBGR::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
{
const auto& desc = frame.desc();
switch (desc.fmt)
{
case cv::MediaFormat::BGR:
{
rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
[](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 3, d.size); },
[](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
return cv::Mat(d.size, CV_8UC3, v.ptr[0], v.stride[0]);
});
break;
}
case cv::MediaFormat::NV12:
{
std::call_once(m_warnFlag,
[](){
GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from NV12 to BGR will happen.\n"
"Conversion may cost a lot for images with high resolution.\n"
"To retrieve cv::Mat-s from NV12 cv::MediaFrame for free, you may use "
"cv::gapi::streaming::Y and cv::gapi::streaming::UV accessors.\n");
});
cv::Mat bgr;
auto view = frame.access(cv::MediaFrame::Access::R);
cv::Mat y_plane (desc.size, CV_8UC1, view.ptr[0], view.stride[0]);
cv::Mat uv_plane(desc.size / 2, CV_8UC2, view.ptr[1], view.stride[1]);
cv::cvtColorTwoPlane(y_plane, uv_plane, bgr, cv::COLOR_YUV2BGR_NV12);
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(bgr);
break;
}
default:
cv::util::throw_error(
std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::BGR"));
}
}
struct GOCVY: public cv::detail::KernelTag
{
using API = cv::gapi::streaming::GY;
static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
class Actor final: public GAccessorActorBase
{
public:
using GAccessorActorBase::GAccessorActorBase;
virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
};
static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
{
return cv::gapi::streaming::IActor::Ptr(new Actor(args));
}
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
};
void GOCVY::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
{
const auto& desc = frame.desc();
switch (desc.fmt)
{
case cv::MediaFormat::BGR:
{
std::call_once(m_warnFlag,
[](){
GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from BGR to NV12 Y plane will "
"happen.\n"
"Conversion may cost a lot for images with high resolution.\n"
"To retrieve cv::Mat from BGR cv::MediaFrame for free, you may use "
"cv::gapi::streaming::BGR accessor.\n");
});
auto view = frame.access(cv::MediaFrame::Access::R);
cv::Mat tmp_bgr(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
cv::Mat yuv;
cvtColor(tmp_bgr, yuv, cv::COLOR_BGR2YUV_I420);
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(yuv.rowRange(0, desc.size.height));
break;
}
case cv::MediaFormat::NV12:
{
rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
[](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 1, d.size); },
[](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
return cv::Mat(d.size, CV_8UC1, v.ptr[0], v.stride[0]);
});
break;
}
default:
cv::util::throw_error(
std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::Y"));
}
}
struct GOCVUV: public cv::detail::KernelTag
{
using API = cv::gapi::streaming::GUV;
static cv::gapi::GBackend backend() { return cv::gapi::streaming::backend(); }
class Actor final: public GAccessorActorBase
{
public:
using GAccessorActorBase::GAccessorActorBase;
virtual void extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat) override;
};
static cv::gapi::streaming::IActor::Ptr create(const cv::GCompileArgs& args)
{
return cv::gapi::streaming::IActor::Ptr(new Actor(args));
}
static cv::gapi::streaming::GStreamingKernel kernel() { return {&create}; };
};
void GOCVUV::Actor::extractRMat(const cv::MediaFrame& frame, cv::RMat& rmat)
{
const auto& desc = frame.desc();
switch (desc.fmt)
{
case cv::MediaFormat::BGR:
{
std::call_once(m_warnFlag,
[](){
GAPI_LOG_WARNING(NULL, "\nOn-the-fly conversion from BGR to NV12 UV plane will "
"happen.\n"
"Conversion may cost a lot for images with high resolution.\n"
"To retrieve cv::Mat from BGR cv::MediaFrame for free, you may use "
"cv::gapi::streaming::BGR accessor.\n");
});
auto view = frame.access(cv::MediaFrame::Access::R);
cv::Mat tmp_bgr(desc.size, CV_8UC3, view.ptr[0], view.stride[0]);
cv::Mat yuv;
cvtColor(tmp_bgr, yuv, cv::COLOR_BGR2YUV_I420);
cv::Mat uv;
std::vector<int> dims = { desc.size.height / 2,
desc.size.width / 2 };
auto start = desc.size.height;
auto range_h = desc.size.height / 4;
std::vector<cv::Mat> uv_planes = {
yuv.rowRange(start, start + range_h).reshape(0, dims),
yuv.rowRange(start + range_h, start + range_h * 2).reshape(0, dims)
};
cv::merge(uv_planes, uv);
rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(uv);
break;
}
case cv::MediaFormat::NV12:
{
rmat = cv::make_rmat<cv::gimpl::RMatMediaFrameAdapter>(frame,
[](const cv::GFrameDesc& d){ return cv::GMatDesc(CV_8U, 2, d.size / 2); },
[](const cv::GFrameDesc& d, const cv::MediaFrame::View& v){
return cv::Mat(d.size / 2, CV_8UC2, v.ptr[1], v.stride[1]);
});
break;
}
default:
cv::util::throw_error(
std::logic_error("Unsupported MediaFormat for cv::gapi::streaming::UV"));
}
}
cv::gapi::GKernelPackage cv::gapi::streaming::kernels()
{
return cv::gapi::kernels<GOCVBGR, GOCVY, GOCVUV>();
}
#else
cv::gapi::GKernelPackage cv::gapi::streaming::kernels()
{
// Still provide this symbol to avoid linking issues
util::throw_error(std::runtime_error("cv::gapi::streaming::kernels() isn't supported in standalone"));
}
#endif // !defined(GAPI_STANDALONE)
cv::GMat cv::gapi::copy(const cv::GMat& in) {
return cv::gimpl::streaming::GCopy::on<cv::GMat>(in);
}
cv::GFrame cv::gapi::copy(const cv::GFrame& in) {
return cv::gimpl::streaming::GCopy::on<cv::GFrame>(in);
}
modules/gapi/src/backends/streaming/gstreamingbackend.hpp
+38
View File
@@ -0,0 +1,38 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#ifndef OPENCV_GAPI_GSTREAMINGBACKEND_HPP
#define OPENCV_GAPI_GSTREAMINGBACKEND_HPP
#include <opencv2/gapi/gkernel.hpp>
#include <opencv2/gapi/streaming/format.hpp>
#include "gstreamingkernel.hpp"
namespace cv {
namespace gimpl {
namespace streaming {
cv::gapi::GKernelPackage kernels();
struct GCopy final : public cv::detail::NoTag
{
static constexpr const char* id() { return "org.opencv.streaming.copy"; }
static GMetaArgs getOutMeta(const GMetaArgs &in_meta, const GArgs&) {
GAPI_Assert(in_meta.size() == 1u);
return in_meta;
}
template<typename T> static T on(const T& arg) {
return cv::GKernelType<GCopy, std::function<T(T)>>::on(arg);
}
};
} // namespace streaming
} // namespace gimpl
} // namespace cv
#endif // OPENCV_GAPI_GSTREAMINGBACKEND_HPP
modules/gapi/src/backends/streaming/gstreamingkernel.hpp
+39
View File
@@ -0,0 +1,39 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
//
// Copyright (C) 2020 Intel Corporation
#ifndef OPENCV_GAPI_GSTREAMINGKERNEL_HPP
#define OPENCV_GAPI_GSTREAMINGKERNEL_HPP
#include "compiler/gislandmodel.hpp"
namespace cv {
namespace gapi {
namespace streaming {
GAPI_EXPORTS cv::gapi::GBackend backend();
class IActor {
public:
using Ptr = std::shared_ptr<IActor>;
virtual void run(cv::gimpl::GIslandExecutable::IInput &in,
cv::gimpl::GIslandExecutable::IOutput &out) = 0;
virtual ~IActor() = default;
};
using CreateActorFunction = std::function<IActor::Ptr(const cv::GCompileArgs&)>;
struct GStreamingKernel
{
CreateActorFunction createActorFunction;
};
} // namespace streaming
} // namespace gapi
} // namespace cv
#endif // OPENCV_GAPI_GSTREAMINGKERNEL_HPP