potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 23:33:05 +04:00
Code Packages Releases Wiki Activity

Merge pull request #10306 from dkurt:faster_rcnn

Browse Source
This commit is contained in:
Vadim Pisarevsky
2017-12-15 12:23:53 +00:00
parent c5ed507737 08112f3821
commit 62359f70ff
12 changed files with 2420 additions and 501 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/src/layers/blank_layer.cpp
+18 -2
View File
@@ -92,9 +92,25 @@ public:
}
};
Ptr<BlankLayer> BlankLayer::create(const LayerParams& params)
Ptr<Layer> BlankLayer::create(const LayerParams& params)
{
return Ptr<BlankLayer>(new BlankLayerImpl(params));
// In case of Caffe's Dropout layer from Faster-RCNN framework,
// https://github.com/rbgirshick/caffe-fast-rcnn/tree/faster-rcnn
// return Power layer.
if (!params.get<bool>("scale_train", true))
{
float scale = 1 - params.get<float>("dropout_ratio", 0.5f);
CV_Assert(scale > 0);
LayerParams powerParams;
powerParams.name = params.name;
powerParams.type = "Power";
powerParams.set("scale", scale);
return PowerLayer::create(powerParams);
}
else
return Ptr<BlankLayer>(new BlankLayerImpl(params));
}
}
modules/dnn/src/layers/detection_output_layer.cpp
+74 -61
View File
@@ -85,6 +85,8 @@ static inline bool SortScorePairDescend(const std::pair<float, T>& pair1,
static inline float caffe_box_overlap(const util::NormalizedBBox& a, const util::NormalizedBBox& b);
static inline float caffe_norm_box_overlap(const util::NormalizedBBox& a, const util::NormalizedBBox& b);
} // namespace
class DetectionOutputLayerImpl : public DetectionOutputLayer
@@ -106,6 +108,9 @@ public:
int _topK;
// Whenever predicted bounding boxes are respresented in YXHW instead of XYWH layout.
bool _locPredTransposed;
// It's true whenever predicted bounding boxes and proposals are normalized to [0, 1].
bool _bboxesNormalized;
bool _clip;
enum { _numAxes = 4 };
static const std::string _layerName;
@@ -172,6 +177,8 @@ public:
_confidenceThreshold = getParameter<float>(params, "confidence_threshold", 0, false, -FLT_MAX);
_topK = getParameter<int>(params, "top_k", 0, false, -1);
_locPredTransposed = getParameter<bool>(params, "loc_pred_transposed", 0, false, false);
_bboxesNormalized = getParameter<bool>(params, "normalized_bbox", 0, false, true);
_clip = getParameter<bool>(params, "clip", 0, false, false);
getCodeType(params);
@@ -182,20 +189,12 @@ public:
setParamsFrom(params);
}
void checkInputs(const std::vector<Mat*> &inputs)
{
for (size_t i = 1; i < inputs.size(); i++)
{
CV_Assert(inputs[i]->size == inputs[0]->size);
}
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const
{
CV_Assert(inputs.size() > 0);
CV_Assert(inputs.size() >= 3);
CV_Assert(inputs[0][0] == inputs[1][0]);
int numPriors = inputs[2][2] / 4;
@@ -398,12 +397,28 @@ public:
// Retrieve all prior bboxes
std::vector<util::NormalizedBBox> priorBBoxes;
std::vector<std::vector<float> > priorVariances;
GetPriorBBoxes(priorData, numPriors, priorBBoxes, priorVariances);
GetPriorBBoxes(priorData, numPriors, _bboxesNormalized, priorBBoxes, priorVariances);
// Decode all loc predictions to bboxes
util::NormalizedBBox clipBounds;
if (_clip)
{
CV_Assert(_bboxesNormalized || inputs.size() >= 4);
clipBounds.xmin = clipBounds.ymin = 0.0f;
if (_bboxesNormalized)
clipBounds.xmax = clipBounds.ymax = 1.0f;
else
{
// Input image sizes;
CV_Assert(inputs[3]->dims == 4);
clipBounds.xmax = inputs[3]->size[3] - 1;
clipBounds.ymax = inputs[3]->size[2] - 1;
}
}
DecodeBBoxesAll(allLocationPredictions, priorBBoxes, priorVariances, num,
_shareLocation, _numLocClasses, _backgroundLabelId,
_codeType, _varianceEncodedInTarget, false, allDecodedBBoxes);
_codeType, _varianceEncodedInTarget, _clip, clipBounds,
_bboxesNormalized, allDecodedBBoxes);
}
size_t numKept = 0;
@@ -489,8 +504,12 @@ public:
LabelBBox::const_iterator label_bboxes = decodeBBoxes.find(label);
if (label_bboxes == decodeBBoxes.end())
CV_ErrorNoReturn_(cv::Error::StsError, ("Could not find location predictions for label %d", label));
NMSFast_(label_bboxes->second, scores, _confidenceThreshold, _nmsThreshold, 1.0, _topK,
indices[c], util::caffe_box_overlap);
if (_bboxesNormalized)
NMSFast_(label_bboxes->second, scores, _confidenceThreshold, _nmsThreshold, 1.0, _topK,
indices[c], util::caffe_norm_box_overlap);
else
NMSFast_(label_bboxes->second, scores, _confidenceThreshold, _nmsThreshold, 1.0, _topK,
indices[c], util::caffe_box_overlap);
numDetections += indices[c].size();
}
if (_keepTopK > -1 && numDetections > (size_t)_keepTopK)
@@ -539,8 +558,7 @@ public:
// **************************************************************
// Compute bbox size
template<bool normalized>
static float BBoxSize(const util::NormalizedBBox& bbox)
static float BBoxSize(const util::NormalizedBBox& bbox, bool normalized)
{
if (bbox.xmax < bbox.xmin || bbox.ymax < bbox.ymin)
{
@@ -575,7 +593,8 @@ public:
static void DecodeBBox(
const util::NormalizedBBox& prior_bbox, const std::vector<float>& prior_variance,
const cv::String& code_type,
const bool clip_bbox, const util::NormalizedBBox& bbox,
const bool clip_bbox, const util::NormalizedBBox& clip_bounds,
const bool normalized_bbox, const util::NormalizedBBox& bbox,
util::NormalizedBBox& decode_bbox)
{
float bbox_xmin = variance_encoded_in_target ? bbox.xmin : prior_variance[0] * bbox.xmin;
@@ -592,11 +611,16 @@ public:
else if (code_type == "CENTER_SIZE")
{
float prior_width = prior_bbox.xmax - prior_bbox.xmin;
CV_Assert(prior_width > 0);
float prior_height = prior_bbox.ymax - prior_bbox.ymin;
if (!normalized_bbox)
{
prior_width += 1.0f;
prior_height += 1.0f;
}
CV_Assert(prior_width > 0);
CV_Assert(prior_height > 0);
float prior_center_x = (prior_bbox.xmin + prior_bbox.xmax) * .5;
float prior_center_y = (prior_bbox.ymin + prior_bbox.ymax) * .5;
float prior_center_x = prior_bbox.xmin + prior_width * .5;
float prior_center_y = prior_bbox.ymin + prior_height * .5;
float decode_bbox_center_x, decode_bbox_center_y;
float decode_bbox_width, decode_bbox_height;
@@ -614,14 +638,14 @@ public:
if (clip_bbox)
{
// Clip the util::NormalizedBBox such that the range for each corner is [0, 1]
decode_bbox.xmin = std::max(std::min(decode_bbox.xmin, 1.f), 0.f);
decode_bbox.ymin = std::max(std::min(decode_bbox.ymin, 1.f), 0.f);
decode_bbox.xmax = std::max(std::min(decode_bbox.xmax, 1.f), 0.f);
decode_bbox.ymax = std::max(std::min(decode_bbox.ymax, 1.f), 0.f);
// Clip the util::NormalizedBBox.
decode_bbox.xmin = std::max(std::min(decode_bbox.xmin, clip_bounds.xmax), clip_bounds.xmin);
decode_bbox.ymin = std::max(std::min(decode_bbox.ymin, clip_bounds.ymax), clip_bounds.ymin);
decode_bbox.xmax = std::max(std::min(decode_bbox.xmax, clip_bounds.xmax), clip_bounds.xmin);
decode_bbox.ymax = std::max(std::min(decode_bbox.ymax, clip_bounds.ymax), clip_bounds.ymin);
}
decode_bbox.clear_size();
decode_bbox.set_size(BBoxSize<true>(decode_bbox));
decode_bbox.set_size(BBoxSize(decode_bbox, normalized_bbox));
}
// Decode a set of bboxes according to a set of prior bboxes
@@ -629,7 +653,8 @@ public:
const std::vector<util::NormalizedBBox>& prior_bboxes,
const std::vector<std::vector<float> >& prior_variances,
const cv::String& code_type, const bool variance_encoded_in_target,
const bool clip_bbox, const std::vector<util::NormalizedBBox>& bboxes,
const bool clip_bbox, const util::NormalizedBBox& clip_bounds,
const bool normalized_bbox, const std::vector<util::NormalizedBBox>& bboxes,
std::vector<util::NormalizedBBox>& decode_bboxes)
{
CV_Assert(prior_bboxes.size() == prior_variances.size());
@@ -641,13 +666,15 @@ public:
{
for (int i = 0; i < num_bboxes; ++i)
DecodeBBox<true>(prior_bboxes[i], prior_variances[i], code_type,
clip_bbox, bboxes[i], decode_bboxes[i]);
clip_bbox, clip_bounds, normalized_bbox,
bboxes[i], decode_bboxes[i]);
}
else
{
for (int i = 0; i < num_bboxes; ++i)
DecodeBBox<false>(prior_bboxes[i], prior_variances[i], code_type,
clip_bbox, bboxes[i], decode_bboxes[i]);
clip_bbox, clip_bounds, normalized_bbox,
bboxes[i], decode_bboxes[i]);
}
}
@@ -658,7 +685,8 @@ public:
const int num, const bool share_location,
const int num_loc_classes, const int background_label_id,
const cv::String& code_type, const bool variance_encoded_in_target,
const bool clip, std::vector<LabelBBox>& all_decode_bboxes)
const bool clip, const util::NormalizedBBox& clip_bounds,
const bool normalized_bbox, std::vector<LabelBBox>& all_decode_bboxes)
{
CV_Assert(all_loc_preds.size() == num);
all_decode_bboxes.clear();
@@ -677,8 +705,8 @@ public:
if (label_loc_preds == loc_preds.end())
CV_ErrorNoReturn_(cv::Error::StsError, ("Could not find location predictions for label %d", label));
DecodeBBoxes(prior_bboxes, prior_variances,
code_type, variance_encoded_in_target, clip,
label_loc_preds->second, decode_bboxes[label]);
code_type, variance_encoded_in_target, clip, clip_bounds,
normalized_bbox, label_loc_preds->second, decode_bboxes[label]);
}
}
}
@@ -689,7 +717,7 @@ public:
// prior_bboxes: stores all the prior bboxes in the format of util::NormalizedBBox.
// prior_variances: stores all the variances needed by prior bboxes.
static void GetPriorBBoxes(const float* priorData, const int& numPriors,
std::vector<util::NormalizedBBox>& priorBBoxes,
bool normalized_bbox, std::vector<util::NormalizedBBox>& priorBBoxes,
std::vector<std::vector<float> >& priorVariances)
{
priorBBoxes.clear(); priorBBoxes.resize(numPriors);
@@ -702,7 +730,7 @@ public:
bbox.ymin = priorData[startIdx + 1];
bbox.xmax = priorData[startIdx + 2];
bbox.ymax = priorData[startIdx + 3];
bbox.set_size(BBoxSize<true>(bbox));
bbox.set_size(BBoxSize(bbox, normalized_bbox));
}
for (int i = 0; i < numPriors; ++i)
@@ -805,36 +833,16 @@ public:
const util::NormalizedBBox& bbox2)
{
util::NormalizedBBox intersect_bbox;
if (bbox2.xmin > bbox1.xmax || bbox2.xmax < bbox1.xmin ||
bbox2.ymin > bbox1.ymax || bbox2.ymax < bbox1.ymin)
{
// Return [0, 0, 0, 0] if there is no intersection.
intersect_bbox.xmin = 0;
intersect_bbox.ymin = 0;
intersect_bbox.xmax = 0;
intersect_bbox.ymax = 0;
}
else
{
intersect_bbox.xmin = std::max(bbox1.xmin, bbox2.xmin);
intersect_bbox.ymin = std::max(bbox1.ymin, bbox2.ymin);
intersect_bbox.xmax = std::min(bbox1.xmax, bbox2.xmax);
intersect_bbox.ymax = std::min(bbox1.ymax, bbox2.ymax);
}
intersect_bbox.xmin = std::max(bbox1.xmin, bbox2.xmin);
intersect_bbox.ymin = std::max(bbox1.ymin, bbox2.ymin);
intersect_bbox.xmax = std::min(bbox1.xmax, bbox2.xmax);
intersect_bbox.ymax = std::min(bbox1.ymax, bbox2.ymax);
float intersect_width, intersect_height;
intersect_width = intersect_bbox.xmax - intersect_bbox.xmin;
intersect_height = intersect_bbox.ymax - intersect_bbox.ymin;
if (intersect_width > 0 && intersect_height > 0)
float intersect_size = BBoxSize(intersect_bbox, normalized);
if (intersect_size > 0)
{
if (!normalized)
{
intersect_width++;
intersect_height++;
}
float intersect_size = intersect_width * intersect_height;
float bbox1_size = BBoxSize<true>(bbox1);
float bbox2_size = BBoxSize<true>(bbox2);
float bbox1_size = BBoxSize(bbox1, normalized);
float bbox2_size = BBoxSize(bbox2, normalized);
return intersect_size / (bbox1_size + bbox2_size - intersect_size);
}
else
@@ -845,6 +853,11 @@ public:
};
float util::caffe_box_overlap(const util::NormalizedBBox& a, const util::NormalizedBBox& b)
{
return DetectionOutputLayerImpl::JaccardOverlap<false>(a, b);
}
float util::caffe_norm_box_overlap(const util::NormalizedBBox& a, const util::NormalizedBBox& b)
{
return DetectionOutputLayerImpl::JaccardOverlap<true>(a, b);
}
modules/dnn/src/layers/pooling_layer.cpp
+37 -22
View File
@@ -88,6 +88,7 @@ public:
else if (params.has("pooled_w") || params.has("pooled_h") || params.has("spatial_scale"))
{
type = ROI;
computeMaxIdx = false;
}
setParamsFrom(params);
ceilMode = params.get<bool>("ceil_mode", true);
@@ -294,24 +295,17 @@ public:
int ystart, yend;
const float *srcData;
int xstartROI = 0;
float roiRatio = 0;
if (poolingType == ROI)
{
const float *roisData = rois->ptr<float>(n);
int ystartROI = scaleAndRoundRoi(roisData[2], spatialScale);
int yendROI = scaleAndRoundRoi(roisData[4], spatialScale);
int roiHeight = std::max(yendROI - ystartROI + 1, 1);
roiRatio = (float)roiHeight / height;
float roiRatio = (float)roiHeight / height;
ystart = ystartROI + y0 * roiRatio;
yend = ystartROI + std::ceil((y0 + 1) * roiRatio);
xstartROI = scaleAndRoundRoi(roisData[1], spatialScale);
int xendROI = scaleAndRoundRoi(roisData[3], spatialScale);
int roiWidth = std::max(xendROI - xstartROI + 1, 1);
roiRatio = (float)roiWidth / width;
CV_Assert(roisData[0] < src->size[0]);
srcData = src->ptr<float>(roisData[0], c);
}
@@ -331,22 +325,12 @@ public:
ofs0 += delta;
int x1 = x0 + delta;
if( poolingType == MAX || poolingType == ROI)
if( poolingType == MAX)
for( ; x0 < x1; x0++ )
{
int xstart, xend;
if (poolingType == ROI)
{
xstart = xstartROI + x0 * roiRatio;
xend = xstartROI + std::ceil((x0 + 1) * roiRatio);
}
else
{
xstart = x0 * stride_w - pad_w;
xend = xstart + kernel_w;
}
int xstart = x0 * stride_w - pad_w;
int xend = min(xstart + kernel_w, inp_width);
xstart = max(xstart, 0);
xend = min(xend, inp_width);
if (xstart >= xend || ystart >= yend)
{
dstData[x0] = 0;
@@ -493,7 +477,7 @@ public:
}
}
}
else
else if (poolingType == AVE)
{
for( ; x0 < x1; x0++ )
{
@@ -543,6 +527,37 @@ public:
}
}
}
else // ROI
{
const float *roisData = rois->ptr<float>(n);
int xstartROI = scaleAndRoundRoi(roisData[1], spatialScale);
int xendROI = scaleAndRoundRoi(roisData[3], spatialScale);
int roiWidth = std::max(xendROI - xstartROI + 1, 1);
float roiRatio = (float)roiWidth / width;
for( ; x0 < x1; x0++ )
{
int xstart = xstartROI + x0 * roiRatio;
int xend = xstartROI + std::ceil((x0 + 1) * roiRatio);
xstart = max(xstart, 0);
xend = min(xend, inp_width);
if (xstart >= xend || ystart >= yend)
{
dstData[x0] = 0;
if (compMaxIdx && dstMaskData)
dstMaskData[x0] = -1;
continue;
}
float max_val = -FLT_MAX;
for (int y = ystart; y < yend; ++y)
for (int x = xstart; x < xend; ++x)
{
const int index = y * inp_width + x;
float val = srcData[index];
max_val = std::max(max_val, val);
}
dstData[x0] = max_val;
}
}
}
}
};
modules/dnn/src/layers/prior_box_layer.cpp
+28 -10
View File
@@ -183,6 +183,7 @@ public:
_minSize = getParameter<float>(params, "min_size", 0, false, 0);
_flip = getParameter<bool>(params, "flip", 0, false, true);
_clip = getParameter<bool>(params, "clip", 0, false, true);
_bboxesNormalized = getParameter<bool>(params, "normalized_bbox", 0, false, true);
_scales.clear();
_aspectRatios.clear();
@@ -251,7 +252,7 @@ public:
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const
{
CV_Assert(inputs.size() == 2);
CV_Assert(!inputs.empty());
int layerHeight = inputs[0][2];
int layerWidth = inputs[0][3];
@@ -282,6 +283,8 @@ public:
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
CV_Assert(inputs.size() == 2);
size_t real_numPriors = _numPriors / pow(2, _offsetsX.size() - 1);
if (_scales.empty())
_scales.resize(real_numPriors, 1.0f);
@@ -323,7 +326,8 @@ public:
{
float center_x = (w + _offsetsX[i]) * stepX;
float center_y = (h + _offsetsY[i]) * stepY;
outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth, _imageHeight, outputPtr);
outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth,
_imageHeight, _bboxesNormalized, outputPtr);
}
if (_maxSize > 0)
{
@@ -333,7 +337,8 @@ public:
{
float center_x = (w + _offsetsX[i]) * stepX;
float center_y = (h + _offsetsY[i]) * stepY;
outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth, _imageHeight, outputPtr);
outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth,
_imageHeight, _bboxesNormalized, outputPtr);
}
}
@@ -349,7 +354,8 @@ public:
{
float center_x = (w + _offsetsX[i]) * stepX;
float center_y = (h + _offsetsY[i]) * stepY;
outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth, _imageHeight, outputPtr);
outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth,
_imageHeight, _bboxesNormalized, outputPtr);
}
}
@@ -363,7 +369,8 @@ public:
{
float center_x = (w + _offsetsX[j]) * stepX;
float center_y = (h + _offsetsY[j]) * stepY;
outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth, _imageHeight, outputPtr);
outputPtr = addPrior(center_x, center_y, _boxWidth, _boxHeight, _imageWidth,
_imageHeight, _bboxesNormalized, outputPtr);
}
}
}
@@ -437,6 +444,7 @@ private:
bool _flip;
bool _clip;
bool _explicitSizes;
bool _bboxesNormalized;
size_t _numPriors;
@@ -444,12 +452,22 @@ private:
static const std::string _layerName;
static float* addPrior(float center_x, float center_y, float width, float height,
float imgWidth, float imgHeight, float* dst)
float imgWidth, float imgHeight, bool normalized, float* dst)
{
dst[0] = (center_x - width * 0.5f) / imgWidth; // xmin
dst[1] = (center_y - height * 0.5f) / imgHeight; // ymin
dst[2] = (center_x + width * 0.5f) / imgWidth; // xmax
dst[3] = (center_y + height * 0.5f) / imgHeight; // ymax
if (normalized)
{
dst[0] = (center_x - width * 0.5f) / imgWidth; // xmin
dst[1] = (center_y - height * 0.5f) / imgHeight; // ymin
dst[2] = (center_x + width * 0.5f) / imgWidth; // xmax
dst[3] = (center_y + height * 0.5f) / imgHeight; // ymax
}
else
{
dst[0] = center_x - width * 0.5f; // xmin
dst[1] = center_y - height * 0.5f; // ymin
dst[2] = center_x + width * 0.5f - 1.0f; // xmax
dst[3] = center_y + height * 0.5f - 1.0f; // ymax
}
return dst + 4;
}
};
modules/dnn/src/layers/proposal_layer.cpp
+245
View File
@@ -0,0 +1,245 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2017, Intel Corporation, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "layers_common.hpp"
namespace cv { namespace dnn {
class ProposalLayerImpl : public ProposalLayer
{
public:
ProposalLayerImpl(const LayerParams& params)
{
setParamsFrom(params);
uint32_t featStride = params.get<uint32_t>("feat_stride", 16);
uint32_t baseSize = params.get<uint32_t>("base_size", 16);
// uint32_t minSize = params.get<uint32_t>("min_size", 16);
uint32_t keepTopBeforeNMS = params.get<uint32_t>("pre_nms_topn", 6000);
keepTopAfterNMS = params.get<uint32_t>("post_nms_topn", 300);
float nmsThreshold = params.get<float>("nms_thresh", 0.7);
DictValue ratios = params.get("ratio");
DictValue scales = params.get("scale");
{
LayerParams lp;
lp.set("step", featStride);
lp.set("flip", false);
lp.set("clip", false);
lp.set("normalized_bbox", false);
// Unused values.
float variance[] = {0.1f, 0.1f, 0.2f, 0.2f};
lp.set("variance", DictValue::arrayReal<float*>(&variance[0], 4));
// Compute widths and heights explicitly.
std::vector<float> widths, heights;
widths.reserve(ratios.size() * scales.size());
heights.reserve(ratios.size() * scales.size());
for (int i = 0; i < ratios.size(); ++i)
{
float ratio = ratios.get<float>(i);
for (int j = 0; j < scales.size(); ++j)
{
float scale = scales.get<float>(j);
float width = std::floor(baseSize / sqrt(ratio) + 0.5f);
float height = std::floor(width * ratio + 0.5f);
widths.push_back(scale * width);
heights.push_back(scale * height);
}
}
lp.set("width", DictValue::arrayReal<float*>(&widths[0], widths.size()));
lp.set("height", DictValue::arrayReal<float*>(&heights[0], heights.size()));
priorBoxLayer = PriorBoxLayer::create(lp);
}
{
int order[] = {0, 2, 3, 1};
LayerParams lp;
lp.set("order", DictValue::arrayInt<int*>(&order[0], 4));
deltasPermute = PermuteLayer::create(lp);
scoresPermute = PermuteLayer::create(lp);
}
{
LayerParams lp;
lp.set("code_type", "CENTER_SIZE");
lp.set("num_classes", 1);
lp.set("share_location", true);
lp.set("background_label_id", 1); // We won't pass background scores so set it out of range [0, num_classes)
lp.set("variance_encoded_in_target", true);
lp.set("keep_top_k", keepTopAfterNMS);
lp.set("top_k", keepTopBeforeNMS);
lp.set("nms_threshold", nmsThreshold);
lp.set("normalized_bbox", false);
lp.set("clip", true);
detectionOutputLayer = DetectionOutputLayer::create(lp);
}
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const
{
// We need to allocate the following blobs:
// - output priors from PriorBoxLayer
// - permuted priors
// - permuted scores
CV_Assert(inputs.size() == 3);
const MatShape& scores = inputs[0];
const MatShape& bboxDeltas = inputs[1];
std::vector<MatShape> layerInputs, layerOutputs, layerInternals;
// Prior boxes layer.
layerInputs.assign(1, scores);
priorBoxLayer->getMemoryShapes(layerInputs, 1, layerOutputs, layerInternals);
CV_Assert(layerOutputs.size() == 1);
CV_Assert(layerInternals.empty());
internals.push_back(layerOutputs[0]);
// Scores permute layer.
CV_Assert(scores.size() == 4);
MatShape objectScores = scores;
CV_Assert((scores[1] & 1) == 0); // Number of channels is even.
objectScores[1] /= 2;
layerInputs.assign(1, objectScores);
scoresPermute->getMemoryShapes(layerInputs, 1, layerOutputs, layerInternals);
CV_Assert(layerOutputs.size() == 1);
CV_Assert(layerInternals.empty());
internals.push_back(layerOutputs[0]);
// BBox predictions permute layer.
layerInputs.assign(1, bboxDeltas);
deltasPermute->getMemoryShapes(layerInputs, 1, layerOutputs, layerInternals);
CV_Assert(layerOutputs.size() == 1);
CV_Assert(layerInternals.empty());
internals.push_back(layerOutputs[0]);
outputs.resize(1, shape(keepTopAfterNMS, 5));
return false;
}
void finalize(const std::vector<Mat*> &inputs, std::vector<Mat> &outputs)
{
std::vector<Mat*> layerInputs;
std::vector<Mat> layerOutputs;
// Scores permute layer.
Mat scores = getObjectScores(*inputs[0]);
layerInputs.assign(1, &scores);
layerOutputs.assign(1, Mat(shape(scores.size[0], scores.size[2],
scores.size[3], scores.size[1]), CV_32FC1));
scoresPermute->finalize(layerInputs, layerOutputs);
// BBox predictions permute layer.
Mat* bboxDeltas = inputs[1];
CV_Assert(bboxDeltas->dims == 4);
layerInputs.assign(1, bboxDeltas);
layerOutputs.assign(1, Mat(shape(bboxDeltas->size[0], bboxDeltas->size[2],
bboxDeltas->size[3], bboxDeltas->size[1]), CV_32FC1));
deltasPermute->finalize(layerInputs, layerOutputs);
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr)
{
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
Layer::forward_fallback(inputs_arr, outputs_arr, internals_arr);
}
void forward(std::vector<Mat*> &inputs, std::vector<Mat> &outputs, std::vector<Mat> &internals)
{
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
CV_Assert(inputs.size() == 3);
CV_Assert(internals.size() == 3);
const Mat& scores = *inputs[0];
const Mat& bboxDeltas = *inputs[1];
const Mat& imInfo = *inputs[2];
Mat& priorBoxes = internals[0];
Mat& permuttedScores = internals[1];
Mat& permuttedDeltas = internals[2];
CV_Assert(imInfo.total() >= 2);
// We've chosen the smallest data type because we need just a shape from it.
fakeImageBlob.create(shape(1, 1, imInfo.at<float>(0), imInfo.at<float>(1)), CV_8UC1);
// Generate prior boxes.
std::vector<Mat> layerInputs(2), layerOutputs(1, priorBoxes);
layerInputs[0] = scores;
layerInputs[1] = fakeImageBlob;
priorBoxLayer->forward(layerInputs, layerOutputs, internals);
// Permute scores.
layerInputs.assign(1, getObjectScores(scores));
layerOutputs.assign(1, permuttedScores);
scoresPermute->forward(layerInputs, layerOutputs, internals);
// Permute deltas.
layerInputs.assign(1, bboxDeltas);
layerOutputs.assign(1, permuttedDeltas);
deltasPermute->forward(layerInputs, layerOutputs, internals);
// Sort predictions by scores and apply NMS. DetectionOutputLayer allocates
// output internally because of different number of objects after NMS.
layerInputs.resize(4);
layerInputs[0] = permuttedDeltas;
layerInputs[1] = permuttedScores;
layerInputs[2] = priorBoxes;
layerInputs[3] = fakeImageBlob;
layerOutputs[0] = Mat();
detectionOutputLayer->forward(layerInputs, layerOutputs, internals);
// DetectionOutputLayer produces 1x1xNx7 output where N might be less or
// equal to keepTopAfterNMS. We fill the rest by zeros.
const int numDets = layerOutputs[0].total() / 7;
CV_Assert(numDets <= keepTopAfterNMS);
Mat src = layerOutputs[0].reshape(1, numDets).colRange(3, 7);
Mat dst = outputs[0].rowRange(0, numDets);
src.copyTo(dst.colRange(1, 5));
dst.col(0).setTo(0); // First column are batch ids. Keep it zeros too.
if (numDets < keepTopAfterNMS)
outputs[0].rowRange(numDets, keepTopAfterNMS).setTo(0);
}
private:
// A first half of channels are background scores. We need only a second one.
static Mat getObjectScores(const Mat& m)
{
CV_Assert(m.dims == 4);
CV_Assert(m.size[0] == 1);
int channels = m.size[1];
CV_Assert((channels & 1) == 0);
return slice(m, Range::all(), Range(channels / 2, channels));
}
Ptr<PriorBoxLayer> priorBoxLayer;
Ptr<DetectionOutputLayer> detectionOutputLayer;
Ptr<PermuteLayer> deltasPermute;
Ptr<PermuteLayer> scoresPermute;
uint32_t keepTopAfterNMS;
Mat fakeImageBlob;
};
Ptr<ProposalLayer> ProposalLayer::create(const LayerParams& params)
{
return Ptr<ProposalLayer>(new ProposalLayerImpl(params));
}
} // namespace dnn
} // namespace cv