potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 07:43:03 +04:00
Code Packages Releases Wiki Activity

Added NV12 support to fluid backend

Browse Source
This commit is contained in:
Ruslan Garnov
2019-02-18 19:27:48 +03:00
parent 322115bfcf
commit c0076b58cd
8 changed files with 306 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/gapi/src/backends/fluid/gfluidbackend.cpp
+105 -40
View File
@@ -67,7 +67,7 @@ namespace
{
GFluidModel fm(graph);
auto fluid_impl = cv::util::any_cast<cv::GFluidKernel>(impl.opaque);
fm.metadata(op_node).set(cv::gimpl::FluidUnit{fluid_impl, {}, 0, 0, 0.0});
fm.metadata(op_node).set(cv::gimpl::FluidUnit{fluid_impl, {}, 0, {}, 0.0});
}
virtual EPtr compile(const ade::Graph &graph,
@@ -138,8 +138,8 @@ private:
struct FluidFilterAgent : public FluidAgent
{
private:
virtual int firstWindow() const override;
virtual std::pair<int,int> linesReadAndnextWindow() const override;
virtual int firstWindow(std::size_t inPort) const override;
virtual std::pair<int,int> linesReadAndnextWindow(std::size_t inPort) const override;
virtual void setRatio(double) override { /* nothing */ }
public:
using FluidAgent::FluidAgent;
@@ -148,14 +148,24 @@ public:
struct FluidResizeAgent : public FluidAgent
{
private:
virtual int firstWindow() const override;
virtual std::pair<int,int> linesReadAndnextWindow() const override;
virtual int firstWindow(std::size_t inPort) const override;
virtual std::pair<int,int> linesReadAndnextWindow(std::size_t inPort) const override;
virtual void setRatio(double ratio) override;
std::unique_ptr<FluidMapper> m_mapper;
public:
using FluidAgent::FluidAgent;
};
struct FluidNV12toRGBAgent : public FluidAgent
{
private:
virtual int firstWindow(std::size_t inPort) const override;
virtual std::pair<int,int> linesReadAndnextWindow(std::size_t inPort) const override;
virtual void setRatio(double) override { /* nothing */ }
public:
using FluidAgent::FluidAgent;
};
}} // namespace cv::gimpl
cv::gimpl::FluidAgent::FluidAgent(const ade::Graph &g, ade::NodeHandle nh)
@@ -182,11 +192,13 @@ void cv::gimpl::FluidAgent::reset()
{
m_producedLines = 0;
auto lines = firstWindow();
for (auto &v : in_views)
for (const auto& it : ade::util::indexed(in_views))
{
auto& v = ade::util::value(it);
if (v)
{
auto idx = ade::util::index(it);
auto lines = firstWindow(idx);
v.priv().reset(lines);
}
}
@@ -240,7 +252,7 @@ static int calcResizeWindow(int inH, int outH)
}
}
static int maxLineConsumption(const cv::GFluidKernel& k, int inH, int outH, int lpi)
static int maxLineConsumption(const cv::GFluidKernel& k, int inH, int outH, int lpi, std::size_t inPort)
{
switch (k.m_kind)
{
@@ -260,6 +272,7 @@ static int maxLineConsumption(const cv::GFluidKernel& k, int inH, int outH, int
return (inH == 1) ? 1 : 2 + lpi - 1;
}
} break;
case cv::GFluidKernel::Kind::NV12toRGB: return inPort == 0 ? 2 : 1; break;
default: GAPI_Assert(false); return 0;
}
}
@@ -271,6 +284,7 @@ static int borderSize(const cv::GFluidKernel& k)
case cv::GFluidKernel::Kind::Filter: return (k.m_window - 1) / 2; break;
// Resize never reads from border pixels
case cv::GFluidKernel::Kind::Resize: return 0; break;
case cv::GFluidKernel::Kind::NV12toRGB: return 0; break;
default: GAPI_Assert(false); return 0;
}
}
@@ -354,31 +368,43 @@ std::pair<int,int> cv::gimpl::FluidUpscaleMapper::linesReadAndNextWindow(int out
return std::make_pair(lines_read, next_window);
}
int cv::gimpl::FluidFilterAgent::firstWindow() const
int cv::gimpl::FluidFilterAgent::firstWindow(std::size_t) const
{
return k.m_window + k.m_lpi - 1;
}
std::pair<int,int> cv::gimpl::FluidFilterAgent::linesReadAndnextWindow() const
std::pair<int,int> cv::gimpl::FluidFilterAgent::linesReadAndnextWindow(std::size_t) const
{
int lpi = std::min(k.m_lpi, m_outputLines - m_producedLines - k.m_lpi);
return std::make_pair(k.m_lpi, k.m_window - 1 + lpi);
}
int cv::gimpl::FluidResizeAgent::firstWindow() const
int cv::gimpl::FluidResizeAgent::firstWindow(std::size_t) const
{
auto outIdx = out_buffers[0]->priv().y();
auto lpi = std::min(m_outputLines - m_producedLines, k.m_lpi);
return m_mapper->firstWindow(outIdx, lpi);
}
std::pair<int,int> cv::gimpl::FluidResizeAgent::linesReadAndnextWindow() const
std::pair<int,int> cv::gimpl::FluidResizeAgent::linesReadAndnextWindow(std::size_t) const
{
auto outIdx = out_buffers[0]->priv().y();
auto lpi = std::min(m_outputLines - m_producedLines - k.m_lpi, k.m_lpi);
return m_mapper->linesReadAndNextWindow(outIdx, lpi);
}
int cv::gimpl::FluidNV12toRGBAgent::firstWindow(std::size_t inPort) const
{
// 2 lines for Y, 1 for UV
return inPort == 0 ? 2 : 1;
}
std::pair<int,int> cv::gimpl::FluidNV12toRGBAgent::linesReadAndnextWindow(std::size_t inPort) const
{
// 2 lines for Y, 1 for UV
return inPort == 0 ? std::make_pair(2, 2) : std::make_pair(1, 1);
}
void cv::gimpl::FluidResizeAgent::setRatio(double ratio)
{
if (ratio >= 1.0)
@@ -438,11 +464,14 @@ void cv::gimpl::FluidAgent::doWork()
k.m_f(in_args, out_buffers);
for (auto& in_view : in_views)
for (const auto& it : ade::util::indexed(in_views))
{
auto& in_view = ade::util::value(it);
if (in_view)
{
auto pair = linesReadAndnextWindow();
auto idx = ade::util::index(it);
auto pair = linesReadAndnextWindow(idx);
in_view.priv().readDone(pair.first, pair.second);
};
}
@@ -554,14 +583,15 @@ void cv::gimpl::GFluidExecutable::initBufferRois(std::vector<int>& readStarts,
// only GMats participate in the process so it's valid to obtain GMatDesc
const auto& meta = util::get<GMatDesc>(data.meta);
for (const auto& inNode : oh->inNodes())
for (const auto& in_edge : oh->inEdges())
{
const auto& in_data = m_gm.metadata(inNode).get<Data>();
const auto& in_node = in_edge->srcNode();
const auto& in_data = m_gm.metadata(in_node).get<Data>();
if (in_data.shape == GShape::GMAT && fg.metadata(inNode).contains<FluidData>())
if (in_data.shape == GShape::GMAT && fg.metadata(in_node).contains<FluidData>())
{
const auto& in_meta = util::get<GMatDesc>(in_data.meta);
const auto& fd = fg.metadata(inNode).get<FluidData>();
const auto& fd = fg.metadata(in_node).get<FluidData>();
auto adjFilterRoi = [](cv::gapi::own::Rect produced, int b, int max_height) {
// Extend with border roi which should be produced, crop to logical image size
@@ -599,13 +629,29 @@ void cv::gimpl::GFluidExecutable::initBufferRois(std::vector<int>& readStarts,
return roi;
};
auto adjNV12Roi = [&](cv::gapi::own::Rect produced, std::size_t port) {
GAPI_Assert(produced.x % 2 == 0);
GAPI_Assert(produced.y % 2 == 0);
GAPI_Assert(produced.width % 2 == 0);
GAPI_Assert(produced.height % 2 == 0);
cv::gapi::own::Rect roi;
switch (port) {
case 0: roi = produced; break;
case 1: roi = cv::gapi::own::Rect{ produced.x/2, produced.y/2, produced.width/2, produced.height/2 }; break;
default: GAPI_Assert(false);
}
return roi;
};
cv::gapi::own::Rect produced = rois[m_id_map.at(data.rc)];
cv::gapi::own::Rect resized;
switch (fg.metadata(oh).get<FluidUnit>().k.m_kind)
{
case GFluidKernel::Kind::Filter: resized = produced; break;
case GFluidKernel::Kind::Resize: resized = adjResizeRoi(produced, in_meta.size, meta.size); break;
case GFluidKernel::Kind::Filter: resized = produced; break;
case GFluidKernel::Kind::Resize: resized = adjResizeRoi(produced, in_meta.size, meta.size); break;
case GFluidKernel::Kind::NV12toRGB: resized = adjNV12Roi(produced, m_gm.metadata(in_edge).get<Input>().port); break;
default: GAPI_Assert(false);
}
@@ -618,7 +664,7 @@ void cv::gimpl::GFluidExecutable::initBufferRois(std::vector<int>& readStarts,
readStarts[in_id] = readStart;
rois[in_id] = roi;
// Continue traverse on internal (w.r.t Island) data nodes only.
if (fd.internal) nodesToVisit.push(inNode);
if (fd.internal) nodesToVisit.push(in_node);
}
else
{
@@ -626,7 +672,7 @@ void cv::gimpl::GFluidExecutable::initBufferRois(std::vector<int>& readStarts,
GAPI_Assert(rois[in_id] == roi);
}
} // if (in_data.shape == GShape::GMAT)
} // for (const auto& inNode : oh->inNodes())
} // for (const auto& in_edge : oh->inEdges())
} // if (!startNode->inNodes().empty())
} // while (!nodesToVisit.empty())
}
@@ -666,8 +712,9 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g,
const auto& fu = fg.metadata(nh).get<FluidUnit>();
switch (fu.k.m_kind)
{
case GFluidKernel::Kind::Filter: m_agents.emplace_back(new FluidFilterAgent(m_g, nh)); break;
case GFluidKernel::Kind::Resize: m_agents.emplace_back(new FluidResizeAgent(m_g, nh)); break;
case GFluidKernel::Kind::Filter: m_agents.emplace_back(new FluidFilterAgent(m_g, nh)); break;
case GFluidKernel::Kind::Resize: m_agents.emplace_back(new FluidResizeAgent(m_g, nh)); break;
case GFluidKernel::Kind::NV12toRGB: m_agents.emplace_back(new FluidNV12toRGBAgent(m_g, nh)); break;
default: GAPI_Assert(false);
}
// NB.: in_buffer_ids size is equal to Arguments size, not Edges size!!!
@@ -849,21 +896,35 @@ namespace
}
}
GAPI_Assert(in_hs.size() == 1 && out_ws.size() == 1 && out_hs.size() == 1);
auto &fu = fg.metadata(node).get<FluidUnit>();
GAPI_Assert((out_ws.size() == 1 && out_hs.size() == 1) &&
((in_hs.size() == 1) ||
((in_hs.size() == 2) && fu.k.m_kind == cv::GFluidKernel::Kind::NV12toRGB)));
const auto &op = g.metadata(node).get<Op>();
fu.line_consumption.resize(op.args.size(), 0);
auto in_h = *in_hs .cbegin();
auto out_h = *out_hs.cbegin();
auto &fu = fg.metadata(node).get<FluidUnit>();
fu.ratio = (double)in_h / out_h;
int line_consumption = maxLineConsumption(fu.k, in_h, out_h, fu.k.m_lpi);
int border_size = borderSize(fu.k);
// Set line consumption for each image (GMat) input
for (const auto& in_edge : node->inEdges())
{
const auto& d = g.metadata(in_edge->srcNode()).get<Data>();
if (d.shape == cv::GShape::GMAT)
{
auto port = g.metadata(in_edge).get<Input>().port;
fu.line_consumption[port] = maxLineConsumption(fu.k, in_h, out_h, fu.k.m_lpi, port);
fu.border_size = border_size;
fu.line_consumption = line_consumption;
GModel::log(g, node, "Line consumption (port " + std::to_string(port) + "): "
+ std::to_string(fu.line_consumption[port]));
}
}
GModel::log(g, node, "Line consumption: " + std::to_string(fu.line_consumption));
fu.border_size = borderSize(fu.k);
GModel::log(g, node, "Border size: " + std::to_string(fu.border_size));
}
}
@@ -884,17 +945,20 @@ namespace
{
const auto &fu = fg.metadata(node).get<FluidUnit>();
for (const auto &in_data_node : node->inNodes())
for (const auto &in_edge : node->inEdges())
{
const auto &in_data_node = in_edge->srcNode();
auto port = g.metadata(in_edge).get<Input>().port;
auto &fd = fg.metadata(in_data_node).get<FluidData>();
// Update (not Set) fields here since a single data node may be
// accessed by multiple consumers
fd.max_consumption = std::max(fu.line_consumption, fd.max_consumption);
fd.max_consumption = std::max(fu.line_consumption[port], fd.max_consumption);
fd.border_size = std::max(fu.border_size, fd.border_size);
GModel::log(g, in_data_node, "Line consumption: " + std::to_string(fd.max_consumption)
+ " (upd by " + std::to_string(fu.line_consumption) + ")", node);
+ " (upd by " + std::to_string(fu.line_consumption[port]) + ")", node);
GModel::log(g, in_data_node, "Border size: " + std::to_string(fd.border_size), node);
}
}
@@ -914,17 +978,18 @@ namespace
{
const auto &fu = fg.metadata(node).get<FluidUnit>();
const int own_latency = fu.line_consumption - fu.border_size;
GModel::log(g, node, "LPI: " + std::to_string(fu.k.m_lpi));
// Output latency is max(input_latency) + own_latency
int in_latency = 0;
for (const auto &in_data_node : node->inNodes())
int out_latency = 0;
for (const auto &in_edge: node->inEdges())
{
// FIXME: ASSERT(DATA), ASSERT(FLUIDDATA)
in_latency = std::max(in_latency, fg.metadata(in_data_node).get<FluidData>().latency);
const auto port = g.metadata(in_edge).get<Input>().port;
const auto own_latency = fu.line_consumption[port] - fu.border_size;
const auto in_latency = fg.metadata(in_edge->srcNode()).get<FluidData>().latency;
out_latency = std::max(out_latency, in_latency + own_latency);
}
const int out_latency = in_latency + own_latency;
for (const auto &out_data_node : node->outNodes())
{
@@ -1018,7 +1083,7 @@ void cv::gimpl::GFluidExecutable::makeReshape(const std::vector<gapi::own::Rect>
if (buf_idx >= 0)
{
agent->in_views[in_idx].priv().allocate(fu.line_consumption, fu.border);
agent->in_views[in_idx].priv().allocate(fu.line_consumption[in_idx], fu.border);
}
}
modules/gapi/src/backends/fluid/gfluidbackend.hpp
+6 -4
View File
@@ -4,10 +4,12 @@
//
// Copyright (C) 2018 Intel Corporation
#ifndef OPENCV_GAPI_FLUID_BACKEND_HPP
#define OPENCV_GAPI_FLUID_BACKEND_HPP
// FIXME? Actually gfluidbackend.hpp is not included anywhere
// and can be placed in gfluidbackend.cpp
#include "opencv2/gapi/garg.hpp"
#include "opencv2/gapi/gproto.hpp"
#include "opencv2/gapi/fluid/gfluidkernel.hpp"
@@ -25,7 +27,7 @@ struct FluidUnit
GFluidKernel k;
gapi::fluid::BorderOpt border;
int border_size;
int line_consumption;
std::vector<int> line_consumption;
double ratio;
};
@@ -90,8 +92,8 @@ public:
private:
// FIXME!!!
// move to another class
virtual int firstWindow() const = 0;
virtual std::pair<int,int> linesReadAndnextWindow() const = 0;
virtual int firstWindow(std::size_t inPort) const = 0;
virtual std::pair<int,int> linesReadAndnextWindow(std::size_t inPort) const = 0;
};
class GFluidExecutable final: public GIslandExecutable
modules/gapi/src/backends/fluid/gfluidbuffer.cpp
+3
View File
@@ -519,6 +519,9 @@ void fluid::Buffer::Priv::allocate(BorderOpt border,
// Init physical buffer
// FIXME? combine line_consumption with skew?
// FIXME? This formula serves general case to avoid possible deadlock,
// in some cases this value can be smaller:
// 2 lines produced, 2 consumed, data_height can be 2, not 3
auto data_height = std::max(line_consumption, skew) + m_writer_lpi - 1;
m_storage = createStorage(data_height,