potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 15:53:03 +04:00
Code Packages Releases Wiki Activity

Merge pull request #9750 from dkurt:feature_dnn_tf_text_graph

Browse Source
This commit is contained in:
Vadim Pisarevsky
2017-10-10 10:06:24 +00:00
parent 7d55c09a9f e4aa39f9e5
commit 046045239c
10 changed files with 536 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/src/layers/detection_output_layer.cpp
+22 -6
View File
@@ -81,6 +81,8 @@ public:
float _nmsThreshold;
int _topK;
// Whenever predicted bounding boxes are respresented in YXHW instead of XYWH layout.
bool _locPredTransposed;
enum { _numAxes = 4 };
static const std::string _layerName;
@@ -148,6 +150,7 @@ public:
_keepTopK = getParameter<int>(params, "keep_top_k");
_confidenceThreshold = getParameter<float>(params, "confidence_threshold", 0, false, -FLT_MAX);
_topK = getParameter<int>(params, "top_k", 0, false, -1);
_locPredTransposed = getParameter<bool>(params, "loc_pred_transposed", 0, false, false);
getCodeType(params);
@@ -209,7 +212,7 @@ public:
// Retrieve all location predictions
std::vector<LabelBBox> allLocationPredictions;
GetLocPredictions(locationData, num, numPriors, _numLocClasses,
_shareLocation, allLocationPredictions);
_shareLocation, _locPredTransposed, allLocationPredictions);
// Retrieve all confidences
GetConfidenceScores(confidenceData, num, numPriors, _numClasses, allConfidenceScores);
@@ -540,11 +543,14 @@ public:
// num_loc_classes: number of location classes. It is 1 if share_location is
// true; and is equal to number of classes needed to predict otherwise.
// share_location: if true, all classes share the same location prediction.
// loc_pred_transposed: if true, represent four bounding box values as
// [y,x,height,width] or [x,y,width,height] otherwise.
// loc_preds: stores the location prediction, where each item contains
// location prediction for an image.
static void GetLocPredictions(const float* locData, const int num,
const int numPredsPerClass, const int numLocClasses,
const bool shareLocation, std::vector<LabelBBox>& locPreds)
const bool shareLocation, const bool locPredTransposed,
std::vector<LabelBBox>& locPreds)
{
locPreds.clear();
if (shareLocation)
@@ -566,10 +572,20 @@ public:
labelBBox[label].resize(numPredsPerClass);
}
caffe::NormalizedBBox& bbox = labelBBox[label][p];
bbox.set_xmin(locData[startIdx + c * 4]);
bbox.set_ymin(locData[startIdx + c * 4 + 1]);
bbox.set_xmax(locData[startIdx + c * 4 + 2]);
bbox.set_ymax(locData[startIdx + c * 4 + 3]);
if (locPredTransposed)
{
bbox.set_ymin(locData[startIdx + c * 4]);
bbox.set_xmin(locData[startIdx + c * 4 + 1]);
bbox.set_ymax(locData[startIdx + c * 4 + 2]);
bbox.set_xmax(locData[startIdx + c * 4 + 3]);
}
else
{
bbox.set_xmin(locData[startIdx + c * 4]);
bbox.set_ymin(locData[startIdx + c * 4 + 1]);
bbox.set_xmax(locData[startIdx + c * 4 + 2]);
bbox.set_ymax(locData[startIdx + c * 4 + 3]);
}
}
}
}
modules/dnn/src/layers/prior_box_layer.cpp
+29 -10
View File
@@ -124,6 +124,20 @@ public:
}
}
void getScales(const LayerParams &params)
{
DictValue scalesParameter;
bool scalesRetieved = getParameterDict(params, "scales", scalesParameter);
if (scalesRetieved)
{
_scales.resize(scalesParameter.size());
for (int i = 0; i < scalesParameter.size(); ++i)
{
_scales[i] = scalesParameter.get<float>(i);
}
}
}
void getVariance(const LayerParams &params)
{
DictValue varianceParameter;
@@ -169,13 +183,14 @@ public:
_flip = getParameter<bool>(params, "flip");
_clip = getParameter<bool>(params, "clip");
_scales.clear();
_aspectRatios.clear();
_aspectRatios.push_back(1.);
getAspectRatios(params);
getVariance(params);
getScales(params);
_numPriors = _aspectRatios.size();
_numPriors = _aspectRatios.size() + 1; // + 1 for an aspect ratio 1.0
_maxSize = -1;
if (params.has("max_size"))
@@ -231,6 +246,11 @@ public:
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
if (_scales.empty())
_scales.resize(_numPriors, 1.0f);
else
CV_Assert(_scales.size() == _numPriors);
int _layerWidth = inputs[0]->size[3];
int _layerHeight = inputs[0]->size[2];
@@ -256,7 +276,7 @@ public:
{
for (size_t w = 0; w < _layerWidth; ++w)
{
_boxWidth = _boxHeight = _minSize;
_boxWidth = _boxHeight = _minSize * _scales[0];
float center_x = (w + 0.5) * stepX;
float center_y = (h + 0.5) * stepY;
@@ -272,7 +292,7 @@ public:
if (_maxSize > 0)
{
// second prior: aspect_ratio = 1, size = sqrt(min_size * max_size)
_boxWidth = _boxHeight = sqrt(_minSize * _maxSize);
_boxWidth = _boxHeight = sqrt(_minSize * _maxSize) * _scales[1];
// xmin
outputPtr[idx++] = (center_x - _boxWidth / 2.) / _imageWidth;
// ymin
@@ -284,15 +304,13 @@ public:
}
// rest of priors
CV_Assert((_maxSize > 0 ? 2 : 1) + _aspectRatios.size() == _scales.size());
for (size_t r = 0; r < _aspectRatios.size(); ++r)
{
float ar = _aspectRatios[r];
if (fabs(ar - 1.) < 1e-6)
{
continue;
}
_boxWidth = _minSize * sqrt(ar);
_boxHeight = _minSize / sqrt(ar);
float scale = _scales[(_maxSize > 0 ? 2 : 1) + r];
_boxWidth = _minSize * sqrt(ar) * scale;
_boxHeight = _minSize / sqrt(ar) * scale;
// xmin
outputPtr[idx++] = (center_x - _boxWidth / 2.) / _imageWidth;
// ymin
@@ -363,6 +381,7 @@ public:
std::vector<float> _aspectRatios;
std::vector<float> _variance;
std::vector<float> _scales;
bool _flip;
bool _clip;
modules/dnn/src/tensorflow/tf_importer.cpp
+158 -31
View File
@@ -321,10 +321,10 @@ DictValue parseDims(const tensorflow::TensorProto &tensor) {
CV_Assert(tensor.dtype() == tensorflow::DT_INT32);
CV_Assert(dims == 1);
int size = tensor.tensor_content().size() / sizeof(int);
const int *data = reinterpret_cast<const int*>(tensor.tensor_content().c_str());
Mat values = getTensorContent(tensor);
CV_Assert(values.type() == CV_32SC1);
// TODO: add reordering shape if dims == 4
return DictValue::arrayInt(data, size);
return DictValue::arrayInt((int*)values.data, values.total());
}
void setKSize(LayerParams &layerParams, const tensorflow::NodeDef &layer)
@@ -448,7 +448,7 @@ void ExcludeLayer(tensorflow::GraphDef& net, const int layer_index, const int in
class TFImporter : public Importer {
public:
TFImporter(const char *model);
TFImporter(const char *model, const char *config = NULL);
void populateNet(Net dstNet);
~TFImporter() {}
@@ -463,13 +463,20 @@ private:
int input_blob_index = -1, int* actual_inp_blob_idx = 0);
tensorflow::GraphDef net;
// Binary serialized TensorFlow graph includes weights.
tensorflow::GraphDef netBin;
// Optional text definition of TensorFlow graph. More flexible than binary format
// and may be used to build the network using binary format only as a weights storage.
// This approach is similar to Caffe's `.prorotxt` and `.caffemodel`.
tensorflow::GraphDef netTxt;
};
TFImporter::TFImporter(const char *model)
TFImporter::TFImporter(const char *model, const char *config)
{
if (model && model[0])
ReadTFNetParamsFromBinaryFileOrDie(model, &net);
ReadTFNetParamsFromBinaryFileOrDie(model, &netBin);
if (config && config[0])
ReadTFNetParamsFromTextFileOrDie(config, &netTxt);
}
void TFImporter::kernelFromTensor(const tensorflow::TensorProto &tensor, Mat &dstBlob)
@@ -557,21 +564,23 @@ const tensorflow::TensorProto& TFImporter::getConstBlob(const tensorflow::NodeDe
*actual_inp_blob_idx = input_blob_index;
}
return net.node(const_layers.at(kernel_inp.name)).attr().at("value").tensor();
int nodeIdx = const_layers.at(kernel_inp.name);
if (nodeIdx < netBin.node_size() && netBin.node(nodeIdx).name() == kernel_inp.name)
{
return netBin.node(nodeIdx).attr().at("value").tensor();
}
else
{
CV_Assert(nodeIdx < netTxt.node_size(),
netTxt.node(nodeIdx).name() == kernel_inp.name);
return netTxt.node(nodeIdx).attr().at("value").tensor();
}
}
void TFImporter::populateNet(Net dstNet)
static void addConstNodes(const tensorflow::GraphDef& net, std::map<String, int>& const_layers,
std::set<String>& layers_to_ignore)
{
RemoveIdentityOps(net);
std::map<int, String> layers_to_ignore;
int layersSize = net.node_size();
// find all Const layers for params
std::map<String, int> value_id;
for (int li = 0; li < layersSize; li++)
for (int li = 0; li < net.node_size(); li++)
{
const tensorflow::NodeDef &layer = net.node(li);
String name = layer.name();
@@ -582,11 +591,27 @@ void TFImporter::populateNet(Net dstNet)
if (layer.attr().find("value") != layer.attr().end())
{
value_id.insert(std::make_pair(name, li));
CV_Assert(const_layers.insert(std::make_pair(name, li)).second);
}
layers_to_ignore[li] = name;
layers_to_ignore.insert(name);
}
}
void TFImporter::populateNet(Net dstNet)
{
RemoveIdentityOps(netBin);
RemoveIdentityOps(netTxt);
std::set<String> layers_to_ignore;
tensorflow::GraphDef& net = netTxt.ByteSize() != 0 ? netTxt : netBin;
int layersSize = net.node_size();
// find all Const layers for params
std::map<String, int> value_id;
addConstNodes(netBin, value_id, layers_to_ignore);
addConstNodes(netTxt, value_id, layers_to_ignore);
std::map<String, int> layer_id;
@@ -597,7 +622,7 @@ void TFImporter::populateNet(Net dstNet)
String type = layer.op();
LayerParams layerParams;
if(layers_to_ignore.find(li) != layers_to_ignore.end())
if(layers_to_ignore.find(name) != layers_to_ignore.end())
continue;
if (type == "Conv2D" || type == "SpaceToBatchND" || type == "DepthwiseConv2dNative")
@@ -627,7 +652,7 @@ void TFImporter::populateNet(Net dstNet)
StrIntVector next_layers = getNextLayers(net, name, "Conv2D");
CV_Assert(next_layers.size() == 1);
layer = net.node(next_layers[0].second);
layers_to_ignore[next_layers[0].second] = next_layers[0].first;
layers_to_ignore.insert(next_layers[0].first);
name = layer.name();
type = layer.op();
}
@@ -644,7 +669,7 @@ void TFImporter::populateNet(Net dstNet)
blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]);
ExcludeLayer(net, weights_layer_index, 0, false);
layers_to_ignore[weights_layer_index] = next_layers[0].first;
layers_to_ignore.insert(next_layers[0].first);
}
kernelFromTensor(getConstBlob(layer, value_id), layerParams.blobs[0]);
@@ -684,7 +709,7 @@ void TFImporter::populateNet(Net dstNet)
layerParams.set("pad_mode", ""); // We use padding values.
CV_Assert(next_layers.size() == 1);
ExcludeLayer(net, next_layers[0].second, 0, false);
layers_to_ignore[next_layers[0].second] = next_layers[0].first;
layers_to_ignore.insert(next_layers[0].first);
}
int id = dstNet.addLayer(name, "Convolution", layerParams);
@@ -748,7 +773,7 @@ void TFImporter::populateNet(Net dstNet)
int weights_layer_index = next_layers[0].second;
blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]);
ExcludeLayer(net, weights_layer_index, 0, false);
layers_to_ignore[weights_layer_index] = next_layers[0].first;
layers_to_ignore.insert(next_layers[0].first);
}
int kernel_blob_index = -1;
@@ -778,6 +803,30 @@ void TFImporter::populateNet(Net dstNet)
// one input only
connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0);
}
else if (type == "Flatten")
{
int id = dstNet.addLayer(name, "Flatten", layerParams);
layer_id[name] = id;
connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0);
}
else if (type == "Transpose")
{
Mat perm = getTensorContent(getConstBlob(layer, value_id, 1));
CV_Assert(perm.type() == CV_32SC1);
int* permData = (int*)perm.data;
if (perm.total() == 4)
{
for (int i = 0; i < 4; ++i)
permData[i] = toNCHW[permData[i]];
}
layerParams.set("order", DictValue::arrayInt<int*>(permData, perm.total()));
int id = dstNet.addLayer(name, "Permute", layerParams);
layer_id[name] = id;
// one input only
connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0);
}
else if (type == "Const")
{
}
@@ -807,7 +856,7 @@ void TFImporter::populateNet(Net dstNet)
{
int axisId = (type == "Concat" ? 0 : layer.input_size() - 1);
int axis = getConstBlob(layer, value_id, axisId).int_val().Get(0);
layerParams.set("axis", toNCHW[axis]);
layerParams.set("axis", 0 <= axis && axis < 4 ? toNCHW[axis] : axis);
int id = dstNet.addLayer(name, "Concat", layerParams);
layer_id[name] = id;
@@ -929,6 +978,19 @@ void TFImporter::populateNet(Net dstNet)
else // is a vector
{
layerParams.blobs.resize(1, scaleMat);
StrIntVector next_layers = getNextLayers(net, name, "Add");
if (!next_layers.empty())
{
layerParams.set("bias_term", true);
layerParams.blobs.resize(2);
int weights_layer_index = next_layers[0].second;
blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs.back());
ExcludeLayer(net, weights_layer_index, 0, false);
layers_to_ignore.insert(next_layers[0].first);
}
id = dstNet.addLayer(name, "Scale", layerParams);
}
layer_id[name] = id;
@@ -1037,7 +1099,7 @@ void TFImporter::populateNet(Net dstNet)
blobFromTensor(getConstBlob(net.node(weights_layer_index), value_id), layerParams.blobs[1]);
ExcludeLayer(net, weights_layer_index, 0, false);
layers_to_ignore[weights_layer_index] = next_layers[0].first;
layers_to_ignore.insert(next_layers[0].first);
}
kernelFromTensor(getConstBlob(layer, value_id, 1), layerParams.blobs[0]);
@@ -1148,6 +1210,71 @@ void TFImporter::populateNet(Net dstNet)
connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0);
}
else if (type == "PriorBox")
{
if (hasLayerAttr(layer, "min_size"))
layerParams.set("min_size", getLayerAttr(layer, "min_size").i());
if (hasLayerAttr(layer, "max_size"))
layerParams.set("max_size", getLayerAttr(layer, "max_size").i());
if (hasLayerAttr(layer, "flip"))
layerParams.set("flip", getLayerAttr(layer, "flip").b());
if (hasLayerAttr(layer, "clip"))
layerParams.set("clip", getLayerAttr(layer, "clip").b());
if (hasLayerAttr(layer, "offset"))
layerParams.set("offset", getLayerAttr(layer, "offset").f());
if (hasLayerAttr(layer, "variance"))
{
Mat variance = getTensorContent(getLayerAttr(layer, "variance").tensor());
layerParams.set("variance",
DictValue::arrayReal<float*>((float*)variance.data, variance.total()));
}
if (hasLayerAttr(layer, "aspect_ratio"))
{
Mat aspectRatios = getTensorContent(getLayerAttr(layer, "aspect_ratio").tensor());
layerParams.set("aspect_ratio",
DictValue::arrayReal<float*>((float*)aspectRatios.data, aspectRatios.total()));
}
if (hasLayerAttr(layer, "scales"))
{
Mat scales = getTensorContent(getLayerAttr(layer, "scales").tensor());
layerParams.set("scales",
DictValue::arrayReal<float*>((float*)scales.data, scales.total()));
}
int id = dstNet.addLayer(name, "PriorBox", layerParams);
layer_id[name] = id;
connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0);
connect(layer_id, dstNet, parsePin(layer.input(1)), id, 1);
}
else if (type == "DetectionOutput")
{
// op: "DetectionOutput"
// input_0: "locations"
// input_1: "classifications"
// input_2: "prior_boxes"
if (hasLayerAttr(layer, "num_classes"))
layerParams.set("num_classes", getLayerAttr(layer, "num_classes").i());
if (hasLayerAttr(layer, "share_location"))
layerParams.set("share_location", getLayerAttr(layer, "share_location").b());
if (hasLayerAttr(layer, "background_label_id"))
layerParams.set("background_label_id", getLayerAttr(layer, "background_label_id").i());
if (hasLayerAttr(layer, "nms_threshold"))
layerParams.set("nms_threshold", getLayerAttr(layer, "nms_threshold").f());
if (hasLayerAttr(layer, "top_k"))
layerParams.set("top_k", getLayerAttr(layer, "top_k").i());
if (hasLayerAttr(layer, "code_type"))
layerParams.set("code_type", getLayerAttr(layer, "code_type").s());
if (hasLayerAttr(layer, "keep_top_k"))
layerParams.set("keep_top_k", getLayerAttr(layer, "keep_top_k").i());
if (hasLayerAttr(layer, "confidence_threshold"))
layerParams.set("confidence_threshold", getLayerAttr(layer, "confidence_threshold").f());
if (hasLayerAttr(layer, "loc_pred_transposed"))
layerParams.set("loc_pred_transposed", getLayerAttr(layer, "loc_pred_transposed").b());
int id = dstNet.addLayer(name, "DetectionOutput", layerParams);
layer_id[name] = id;
for (int i = 0; i < 3; ++i)
connect(layer_id, dstNet, parsePin(layer.input(i)), id, i);
}
else if (type == "Abs" || type == "Tanh" || type == "Sigmoid" ||
type == "Relu" || type == "Elu" || type == "Softmax" ||
type == "Identity" || type == "Relu6")
@@ -1188,9 +1315,9 @@ Ptr<Importer> createTensorflowImporter(const String&)
#endif //HAVE_PROTOBUF
Net readNetFromTensorflow(const String &model)
Net readNetFromTensorflow(const String &model, const String &config)
{
TFImporter importer(model.c_str());
TFImporter importer(model.c_str(), config.c_str());
Net net;
importer.populateNet(net);
return net;
modules/dnn/src/tensorflow/tf_io.cpp
+15
View File
@@ -52,12 +52,27 @@ bool ReadProtoFromBinaryFileTF(const char* filename, Message* proto) {
return success;
}
bool ReadProtoFromTextFileTF(const char* filename, Message* proto) {
std::ifstream fs(filename, std::ifstream::in);
CHECK(fs.is_open()) << "Can't open \"" << filename << "\"";
IstreamInputStream input(&fs);
bool success = google::protobuf::TextFormat::Parse(&input, proto);
fs.close();
return success;
}
void ReadTFNetParamsFromBinaryFileOrDie(const char* param_file,
tensorflow::GraphDef* param) {
CHECK(ReadProtoFromBinaryFileTF(param_file, param))
<< "Failed to parse GraphDef file: " << param_file;
}
void ReadTFNetParamsFromTextFileOrDie(const char* param_file,
tensorflow::GraphDef* param) {
CHECK(ReadProtoFromTextFileTF(param_file, param))
<< "Failed to parse GraphDef file: " << param_file;
}
}
}
#endif
modules/dnn/src/tensorflow/tf_io.hpp
+3
View File
@@ -22,6 +22,9 @@ namespace dnn {
void ReadTFNetParamsFromBinaryFileOrDie(const char* param_file,
tensorflow::GraphDef* param);
void ReadTFNetParamsFromTextFileOrDie(const char* param_file,
tensorflow::GraphDef* param);
}
}