From 2e6a0cab65bce5b743509a4b8cbaf028b66eaad7 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Fri, 13 Jun 2025 12:57:23 +0300 Subject: [PATCH] Merge pull request #27439 from dkurt:tflite_fixes_for_new_engine Adoptions of TFLife fixes to new engine #27439 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request This PR covers updates from the following PRs: #27273, #27307 (ported to 5.x by #27370) resolves #27397 - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/tflite/tflite_importer.cpp | 96 +++++++++++++--------- modules/dnn/test/test_tflite_importer.cpp | 2 +- 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/modules/dnn/src/tflite/tflite_importer.cpp b/modules/dnn/src/tflite/tflite_importer.cpp index e44dd66927..dd74d6ead0 100644 --- a/modules/dnn/src/tflite/tflite_importer.cpp +++ b/modules/dnn/src/tflite/tflite_importer.cpp @@ -86,7 +86,7 @@ private: int addReshapeLayer(const std::vector& shape, int axis, int num_axes, const std::string& name, const std::pair& inpId, int dtype, int inpTensorId); int addFlattenLayer(int axis, int end_axis, const std::string& name, const std::pair& inpId, int dtype, int outTensorId); - int addConstLayer(const Mat& data, const std::string& name); + void addConstLayer(const Mat& data, int tensorIdx); inline bool isInt8(const Operator& op); inline void getQuantParams(const Operator& op, float& inpScale, int& inpZero, float& outScale, int& outZero); @@ -488,11 +488,13 @@ void TFLiteImporter::parseConvolution(const Operator& op, const std::string& opc layerParams.type = "Convolution"; int inpId = op.inputs()->Get(0); + bool additionalPreLayer = false; if (layouts[inpId] == DNN_LAYOUT_UNKNOWN && modelTensors->Get(inpId)->shape()->size() == 4) { int permId = addPermuteLayer({0, 3, 1, 2}, layerParams.name + "/permute_input", layerIds[inpId], isInt8(op) ? CV_8S : CV_32F, op.inputs()->Get(0)); // NHWC -> NCHW layerIds[inpId] = std::make_pair(permId, 0); layouts[op.outputs()->Get(0)] = DNN_LAYOUT_NHWC; + additionalPreLayer = true; } auto options = reinterpret_cast(op.builtin_options()); @@ -554,7 +556,7 @@ void TFLiteImporter::parseConvolution(const Operator& op, const std::string& opc std::string fusedActivationType = EnumNameActivationFunctionType(options->fused_activation_function()); bool haveFusedActivation = fusedActivationType != "NONE"; - addLayer(layerParams, op, false, haveFusedActivation); + addLayer(layerParams, op, additionalPreLayer, haveFusedActivation); parseFusedActivation(op, options->fused_activation_function()); } @@ -720,27 +722,33 @@ void TFLiteImporter::parseEltwise(const Operator& op, const std::string& opcode, if (layouts[op.inputs()->Get(0)] == DNN_LAYOUT_NHWC && blob.dims == 1) { blob = blob.reshape(1, {1, (int)blob.total(), 1, 1}); } - int constId = addConstLayer(blob, modelTensors->Get(idx)->name()->str()); - layerIds[idx] = std::make_pair(constId, 0); + addConstLayer(blob, idx); } std::string fusedActivationType = EnumNameActivationFunctionType(activ); bool haveFusedActivation = fusedActivationType != "NONE"; - addLayer(layerParams, op, false, haveFusedActivation); + addLayer(layerParams, op, false, haveFusedActivation || opcode == "SQUARED_DIFFERENCE" || opcode == "RSQRT"); parseFusedActivation(op, activ); - // Layers that split on multiple operations - if (opcode == "SQUARED_DIFFERENCE") { + if (opcode == "SQUARED_DIFFERENCE" || opcode == "RSQRT") { + if (haveFusedActivation) + CV_LOG_WARNING(NULL, format("%s with fused activation on new engine is not tested", opcode.c_str())); LayerParams lp; - lp.set("power", 2); - int id = dstNet.addLayerToPrev(layerParams.name + "/square", "Power", isOpInt8 ? CV_8S : CV_32F, lp); - layerIds[op.outputs()->Get(0)] = std::make_pair(id, 0); - } - else if (opcode == "RSQRT") { - LayerParams lp; - int id = dstNet.addLayerToPrev(layerParams.name + "/inv", "Reciprocal", isOpInt8 ? CV_8S : CV_32F, lp); - layerIds[op.outputs()->Get(0)] = std::make_pair(id, 0); + if (opcode == "RSQRT") + lp.type = "Reciprocal"; + else { + lp.type = "Power"; + lp.set("power", 2); + } + if (newEngine) { + std::string tensorName = modelTensors->Get(op.outputs()->Get(0))->name()->str(); + addLayer(lp, {tensorName + "_additional_post_layer"}, {tensorName}); + layerIds[op.outputs()->Get(0)] = std::make_pair(-1, -1); + } else { + int id = dstNet.addLayerToPrev(layerParams.name + "/post", lp.type, isOpInt8 ? CV_8S : CV_32F, lp); + layerIds[op.outputs()->Get(0)] = std::make_pair(id, 0); + } } } @@ -866,8 +874,7 @@ void TFLiteImporter::parseConcat(const Operator& op, const std::string& opcode, transposeND(blob, {0, 3, 1, 2}, nchwBlob); blob = nchwBlob; } - int constId = addConstLayer(blob, modelTensors->Get(idx)->name()->str()); - layerIds[idx] = std::make_pair(constId, 0); + addConstLayer(blob, idx); } std::string fusedActivationType = EnumNameActivationFunctionType(options->fused_activation_function()); @@ -1079,11 +1086,23 @@ int TFLiteImporter::addFlattenLayer(int axis, int end_axis, const std::string& n } } -int TFLiteImporter::addConstLayer(const Mat& blob, const std::string& name) +void TFLiteImporter::addConstLayer(const Mat& blob, int tensorIdx) { + const std::string& name = modelTensors->Get(tensorIdx)->name()->str(); LayerParams lp; lp.blobs.push_back(blob.u ? blob : blob.clone()); // some tensors are owned by OpenCV - return dstNet.addLayer(name, "Const", lp); + if (newEngine) + { + lp.type = "Const"; + lp.name = name; + addLayer(lp, {}, {name}); + layerIds[tensorIdx] = std::make_pair(-1, -1); + } + else + { + int constId = dstNet.addLayer(name, "Const", lp); + layerIds[tensorIdx] = std::make_pair(constId, 0); + } } void TFLiteImporter::parseDeconvolution(const Operator& op, const std::string& opcode, LayerParams& layerParams) { @@ -1218,21 +1237,31 @@ void TFLiteImporter::parseStridedSlice(const Operator& op, const std::string& op lastShrinkAxis = axis; } std::string layerName = layerParams.name; - if (lastShrinkAxis != -1) + if (!newEngine && lastShrinkAxis != -1) { layerParams.name += "/slice"; } - addLayer(layerParams, op); + addLayer(layerParams, op, false, lastShrinkAxis != -1); for (int axis = 0; axis < num; ++axis) { if (!(shrinkMask & (1 << axis))) continue; - std::string name = (axis == lastShrinkAxis) ? layerName : format("%s/shrink_axis_%d", layerName.c_str(), axis); - int layerId = addFlattenLayer(axis, axis + 1, name, - layerIds[op.outputs()->Get(0)], isInt8(op) ? CV_8S : CV_32F, op.inputs()->Get(0)); - layerIds[op.inputs()->Get(0)] = std::make_pair(layerId, 0); + if (newEngine) + { + if (axis != lastShrinkAxis) + CV_LOG_WARNING(NULL, "StridedSlice with multiple axes shrink in new engine is not tested"); + addFlattenLayer(axis, axis + 1, layerName, + layerIds[op.outputs()->Get(0)], isInt8(op) ? CV_8S : CV_32F, op.outputs()->Get(0)); + } + else + { + std::string name = (axis == lastShrinkAxis) ? layerName : format("%s/shrink_axis_%d", layerName.c_str(), axis); + int layerId = addFlattenLayer(axis, axis + 1, name, + layerIds[op.outputs()->Get(0)], isInt8(op) ? CV_8S : CV_32F, op.inputs()->Get(0)); + layerIds[op.inputs()->Get(0)] = std::make_pair(layerId, 0); + } } } @@ -1328,22 +1357,7 @@ void TFLiteImporter::parseDetectionPostProcess(const Operator& op, const std::st layerParams.set("variance_encoded_in_target", true); } - LayerParams priorsLP; - priorsLP.name = layerParams.name + "/priors"; - priorsLP.type = "Const"; - priorsLP.blobs.resize(1, priors); - - if (newEngine) - { - std::string outTensorName = modelTensors->Get(op.inputs()->Get(2))->name()->str(); - addLayer(priorsLP, {}, {outTensorName}); - layerIds[op.inputs()->Get(2)] = std::make_pair(-1, -1); - } - else - { - int priorsId = dstNet.addLayer(priorsLP.name, priorsLP.type, priorsLP); - layerIds[op.inputs()->Get(2)] = std::make_pair(priorsId, 0); - } + addConstLayer(priors, op.inputs()->Get(2)); addLayer(layerParams, op); } diff --git a/modules/dnn/test/test_tflite_importer.cpp b/modules/dnn/test/test_tflite_importer.cpp index 52e5ecef27..b4e89ee46c 100644 --- a/modules/dnn/test/test_tflite_importer.cpp +++ b/modules/dnn/test/test_tflite_importer.cpp @@ -283,7 +283,7 @@ TEST_P(Test_TFLite, StridedSlice) { testLayer("strided_slice"); } -TEST_P(Test_TFLite, DISABLED_face_blendshapes) +TEST_P(Test_TFLite, face_blendshapes) { Mat inp = blobFromNPY(findDataFile("dnn/tflite/face_blendshapes_inp.npy")); testModel("face_blendshapes", inp);