diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index 37156a71fa..bb0085201b 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1469,6 +1469,15 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS LayerNorm2Layer : public Layer + { + public: + int axis; + float epsilon; + + static Ptr create(const LayerParams& params); + }; + class CV_EXPORTS GemmLayer : public Layer { public: bool trans_a; diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index 9cd5bd84ce..81c5e48039 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -200,6 +200,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(GatherElements, GatherElementsLayer); CV_DNN_REGISTER_LAYER_CLASS(LayerNormalization, LayerNormLayer); CV_DNN_REGISTER_LAYER_CLASS(RMSNormalization, RMSNormLayer); + CV_DNN_REGISTER_LAYER_CLASS(LayerNormalization2, LayerNorm2Layer); CV_DNN_REGISTER_LAYER_CLASS(Expand, ExpandLayer); CV_DNN_REGISTER_LAYER_CLASS(InstanceNormalization, InstanceNormLayer); CV_DNN_REGISTER_LAYER_CLASS(Attention, AttentionLayer); diff --git a/modules/dnn/src/layer.cpp b/modules/dnn/src/layer.cpp index b19f0d137c..6d3cc11b6b 100644 --- a/modules/dnn/src/layer.cpp +++ b/modules/dnn/src/layer.cpp @@ -269,9 +269,9 @@ void Layer::getTypes(const std::vector&inputs, if (preferableTarget == DNN_TARGET_CUDA_FP16 || preferableTarget == DNN_TARGET_CUDA) CV_CheckTypeEQ(input, CV_32F, ""); else if (preferableTarget == DNN_TARGET_OPENCL_FP16) - CV_CheckType(input, input == CV_16F || input == CV_8S || input == CV_64F, ""); + CV_CheckType(input, input == CV_16F || input == CV_8S || input == CV_64F || input == CV_64S, ""); else - CV_CheckType(input, input == CV_32F || input == CV_64F || input == CV_8S, ""); + CV_CheckType(input, input == CV_32F || input == CV_64F || input == CV_8S || input == CV_64S, ""); } outputs.assign(requiredOutputs, inputs[0]); diff --git a/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp b/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp index 2710d85004..85e2794d51 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_norm.cpp @@ -42,6 +42,49 @@ void fastNorm(const Mat &input, Mat &output, float epsilon, size_t normalized_ax parallel_for_(Range(0, loops), fn, nstripes); } +void fastNormMeanInvStdDev(const Mat& input, Mat& mean, Mat& invStdDev, float epsilon, size_t normalized_axis) +{ + CV_Assert(input.type() == CV_32F); + CV_Assert(mean.type() == CV_32F); + CV_Assert(invStdDev.type() == CV_32F); + CV_Assert(input.isContinuous() && mean.isContinuous() && invStdDev.isContinuous()); + + const auto input_shape = shape(input); + CV_CheckLT(normalized_axis, input_shape.size(), "fastNormMeanInvStdDev: axis out of range"); + + const size_t loops = static_cast(total(input_shape, 0, static_cast(normalized_axis))); + const size_t norm_size = static_cast(total(input_shape, static_cast(normalized_axis))); + const float inv_norm_size = 1.0f / (float)norm_size; + + CV_CheckEQ((size_t)mean.total(), loops, "fastNormMeanInvStdDev: mean output size mismatch"); + CV_CheckEQ((size_t)invStdDev.total(), loops, "fastNormMeanInvStdDev: invStdDev output size mismatch"); + + auto fn = [&](const Range& r) { + const float* input_data = input.ptr(); + float* mean_data = mean.ptr(); + float* invstd_data = invStdDev.ptr(); + for (int i = r.start; i < r.end; ++i) + { + const float* x = input_data + norm_size * (size_t)i; + float m = 0.f, mean_square = 0.f; + for (size_t j = 0; j < norm_size; ++j) + { + float v = x[j]; + m += v; + mean_square += v * v; + } + m *= inv_norm_size; + const float var = std::max(0.f, mean_square * inv_norm_size - m * m); + const float stdev = std::sqrt(var + epsilon); + mean_data[i] = m; + invstd_data[i] = 1.f / stdev; + } + }; + + const double nstripes = loops * norm_size * (1 / 1024.0); + parallel_for_(Range(0, (int)loops), fn, nstripes); +} + void fastNorm(const Mat &input, const Mat &scale, Mat &output, float epsilon, size_t normalized_axis, bool recenter) { const auto input_shape = shape(input); CV_CheckLT(normalized_axis, input_shape.size(), "fastNorm: axis out of range"); diff --git a/modules/dnn/src/layers/cpu_kernels/fast_norm.hpp b/modules/dnn/src/layers/cpu_kernels/fast_norm.hpp index 21f0101d07..5a026b118c 100644 --- a/modules/dnn/src/layers/cpu_kernels/fast_norm.hpp +++ b/modules/dnn/src/layers/cpu_kernels/fast_norm.hpp @@ -12,6 +12,9 @@ namespace cv { namespace dnn { // Normalization speedup by multi-threading, mainly for Caffe MVN layer which has normalize_variance parameter. void fastNorm(const Mat &input, Mat &output, float epsilon, size_t normalized_axis = 0, bool normalize_variance = true); +// Compute mean and inverse standard deviation. +void fastNormMeanInvStdDev(const Mat& input, Mat& mean, Mat& invStdDev, float epsilon, size_t normalized_axis = 0); + // Normalization speedup by multi-threading with absent bias. Mainly for LayerNormalization. void fastNorm(const Mat &input, const Mat &scale, Mat &output, float epsilon, size_t normalized_axis = 0, bool recenter=true); diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 17755454f6..48bc2f2af9 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -93,6 +93,14 @@ using std::sin; using std::sinh; using std::tan; +struct PowerFunctor; + +template +struct ElementWiseIntDispatch +{ + static inline bool apply(const Func&, const Mat&, Mat&) { return false; } +}; + template class ElementWiseLayer : public Func::Layer { @@ -226,6 +234,9 @@ public: Mat &dst = outputs[i]; CV_Assert_N(src.size == dst.size, src.isContinuous(), dst.isContinuous()); + if (ElementWiseIntDispatch::apply(func, src, dst)) + continue; + if (src.type() == CV_32F && dst.type() == CV_32F) { const int nstripes = getNumThreads(); @@ -2454,6 +2465,49 @@ struct PowerFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return power == 1 ? 2 : 10; } }; +// This is required for ONNX Neg on integer tensors produced by Shape/Size subgraphs. +template<> +struct ElementWiseIntDispatch +{ + static inline bool apply(const PowerFunctor& func, const Mat& src, Mat& dst) + { + if (src.type() != dst.type()) + return false; + const int depth = src.depth(); + if (depth != CV_32S && depth != CV_64S) + return false; + + if (func.power != 1.f) + return false; + if (func.shift != 0.f) + return false; + + // scale must be an integer value (Neg uses scale=-1) + const double scale_d = (double)func.scale; + if (std::floor(scale_d) != scale_d) + return false; + const int64_t scale = (int64_t)scale_d; + + const size_t n = src.total(); + if (depth == CV_32S) + { + const int32_t* sp = src.ptr(); + int32_t* dp = dst.ptr(); + for (size_t i = 0; i < n; ++i) + dp[i] = (int32_t)((int64_t)sp[i] * scale); + return true; + } + else // CV_64S + { + const int64_t* sp = src.ptr(); + int64_t* dp = dst.ptr(); + for (size_t i = 0; i < n; ++i) + dp[i] = sp[i] * scale; + return true; + } + } +}; + struct ExpFunctor : public BaseDefaultFunctor { typedef ExpLayer Layer; diff --git a/modules/dnn/src/layers/layer_norm2.cpp b/modules/dnn/src/layers/layer_norm2.cpp new file mode 100644 index 0000000000..61f1277580 --- /dev/null +++ b/modules/dnn/src/layers/layer_norm2.cpp @@ -0,0 +1,115 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// Copyright (C) 2025, BigVision LLC, all rights reserved. +// Third party copyrights are property of their respective owners. + +#include "../precomp.hpp" +#include "layers_common.hpp" +#include "cpu_kernels/fast_norm.hpp" + +namespace cv { +namespace dnn { + +// ONNX LayerNormalization operator +// Spec: https://onnx.ai/onnx/operators/onnx__LayerNormalization.html +// Supported opsets: 17 + +class LayerNorm2LayerImpl CV_FINAL : public LayerNorm2Layer +{ +public: + int axis0; + + LayerNorm2LayerImpl(const LayerParams& params) + { + setParamsFrom(params); + axis = axis0 = params.get("axis", -1); + epsilon = params.get("epsilon", 1e-5f); + } + + virtual bool supportBackend(int backendId) CV_OVERRIDE + { + return backendId == DNN_BACKEND_OPENCV; + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE + { + int noutputs = std::max(requiredOutputs > 0 ? requiredOutputs : (int)this->outputs.size(), 1); + CV_Assert(noutputs >= 1 && noutputs <= 3); + + int num_inputs = inputs.size() + blobs.size(); + CV_Check(num_inputs, num_inputs >= 2 && num_inputs <= 3, "LayerNorm2: require two (x, weight) or three (x, weight, bias) inputs"); + + auto x_shape = inputs[0]; + int x_ndims = static_cast(x_shape.size()); + int axis_ = normalize_axis(axis0, x_shape.dims); + + auto w_shape = blobs.empty() ? inputs[1] : shape(blobs.front()); + int w_ndims = static_cast(w_shape.size()); + w_ndims = (axis_ == x_ndims - 1 && w_ndims == 2) ? (w_ndims - 1) : w_ndims; + CV_CheckEQ(x_ndims - axis_, w_ndims, "LayerNorm2: weight rank mismatch"); + for (int i = 0; i < w_ndims; ++i) + CV_CheckEQ(x_shape[axis_ + i], w_shape[i], "LayerNorm2: weight dims mismatch"); + if (num_inputs >= 3) + { + auto b_shape = blobs.empty() ? inputs[2] : shape(blobs.back()); + CV_CheckEQ(w_shape.size(), b_shape.size(), "LayerNorm2: bias rank mismatch"); + for (size_t i = 0; i < w_shape.size(); ++i) + CV_CheckEQ(w_shape[i], b_shape[i], "LayerNorm2: bias dims mismatch"); + } + + outputs.resize(noutputs, inputs[0]); + for (int i = 1; i < noutputs; i++) { + for (int j = axis_; j < x_ndims; j++) + outputs[i][j] = 1; + } + internals.clear(); + return false; + } + + void forward(InputArrayOfArrays inputs_arr, + OutputArrayOfArrays outputs_arr, + OutputArrayOfArrays internals_arr) CV_OVERRIDE + { + CV_TRACE_FUNCTION(); + CV_TRACE_ARG_VALUE(name, "name", name.c_str()); + + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + + const Mat& input = inputs[0]; + const Mat& scale = blobs.empty() ? inputs[1] : blobs.front(); + Mat& output = outputs[0]; + + int axis_ = normalize_axis(axis0, input.dims); + + if (outputs.size() >= 3) + { + Mat& mean = outputs[1]; + Mat& invStdDev = outputs[2]; + fastNormMeanInvStdDev(input, mean, invStdDev, epsilon, (size_t)axis_); + } + + if ((int)inputs.size() + (int)blobs.size() >= 3) + { + const Mat& bias = blobs.empty() ? inputs[2] : blobs.back(); + fastNorm(input, scale, bias, output, epsilon, (size_t)axis_); + } + else + { + fastNorm(input, scale, output, epsilon, (size_t)axis_); + } + } +}; + +Ptr LayerNorm2Layer::create(const LayerParams& params) +{ + return makePtr(params); +} + +} +} diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index 89a8491a6c..a8176cc9e6 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -1994,6 +1994,7 @@ void ONNXImporter2::parseLayerNorm(LayerParams& layerParams, const opencv_onnx:: } n_inputs = 1; } + layerParams.type = "LayerNormalization2"; addLayer(layerParams, node_proto, n_inputs); } diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp index 5bd3287845..fe8b2992d8 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp @@ -1201,43 +1201,119 @@ CASE(test_isnan) CASE(test_isnan_float16) SKIP; CASE(test_layer_normalization_2d_axis0) - // no filter + SKIP; +CASE(test_layer_normalization_2d_axis0_expanded) + SKIP; +CASE(test_layer_normalization_2d_axis0_expanded_ver18) + SKIP; CASE(test_layer_normalization_2d_axis1) - // no filter + SKIP; +CASE(test_layer_normalization_2d_axis1_expanded) + SKIP; +CASE(test_layer_normalization_2d_axis1_expanded_ver18) + SKIP; CASE(test_layer_normalization_2d_axis_negative_1) - // no filter + SKIP; +CASE(test_layer_normalization_2d_axis_negative_1_expanded) + SKIP; +CASE(test_layer_normalization_2d_axis_negative_1_expanded_ver18) + SKIP; CASE(test_layer_normalization_2d_axis_negative_2) - // no filter + SKIP; +CASE(test_layer_normalization_2d_axis_negative_2_expanded) + SKIP; +CASE(test_layer_normalization_2d_axis_negative_2_expanded_ver18) + SKIP; CASE(test_layer_normalization_3d_axis0_epsilon) - // no filter + SKIP; +CASE(test_layer_normalization_3d_axis0_epsilon_expanded) + SKIP; +CASE(test_layer_normalization_3d_axis0_epsilon_expanded_ver18) + SKIP; CASE(test_layer_normalization_3d_axis1_epsilon) - // no filter + SKIP; +CASE(test_layer_normalization_3d_axis1_epsilon_expanded) + SKIP; +CASE(test_layer_normalization_3d_axis1_epsilon_expanded_ver18) + SKIP; CASE(test_layer_normalization_3d_axis2_epsilon) - // no filter + SKIP; +CASE(test_layer_normalization_3d_axis2_epsilon_expanded) + SKIP; +CASE(test_layer_normalization_3d_axis2_epsilon_expanded_ver18) + SKIP; CASE(test_layer_normalization_3d_axis_negative_1_epsilon) - // no filter + SKIP; +CASE(test_layer_normalization_3d_axis_negative_1_epsilon_expanded) + SKIP; +CASE(test_layer_normalization_3d_axis_negative_1_epsilon_expanded_ver18) + SKIP; CASE(test_layer_normalization_3d_axis_negative_2_epsilon) - // no filter + SKIP; +CASE(test_layer_normalization_3d_axis_negative_2_epsilon_expanded) + SKIP; +CASE(test_layer_normalization_3d_axis_negative_2_epsilon_expanded_ver18) + SKIP; CASE(test_layer_normalization_3d_axis_negative_3_epsilon) - // no filter + SKIP; +CASE(test_layer_normalization_3d_axis_negative_3_epsilon_expanded) + SKIP; +CASE(test_layer_normalization_3d_axis_negative_3_epsilon_expanded_ver18) + SKIP; CASE(test_layer_normalization_4d_axis0) - // no filter + SKIP; +CASE(test_layer_normalization_4d_axis0_expanded) + SKIP; +CASE(test_layer_normalization_4d_axis0_expanded_ver18) + SKIP; CASE(test_layer_normalization_4d_axis1) - // no filter + SKIP; +CASE(test_layer_normalization_4d_axis1_expanded) + SKIP; +CASE(test_layer_normalization_4d_axis1_expanded_ver18) + SKIP; CASE(test_layer_normalization_4d_axis2) - // no filter + SKIP; +CASE(test_layer_normalization_4d_axis2_expanded) + SKIP; +CASE(test_layer_normalization_4d_axis2_expanded_ver18) + SKIP; CASE(test_layer_normalization_4d_axis3) - // no filter + SKIP; +CASE(test_layer_normalization_4d_axis3_expanded) + SKIP; +CASE(test_layer_normalization_4d_axis3_expanded_ver18) + SKIP; CASE(test_layer_normalization_4d_axis_negative_1) - // no filter + SKIP; +CASE(test_layer_normalization_4d_axis_negative_1_expanded) + SKIP; +CASE(test_layer_normalization_4d_axis_negative_1_expanded_ver18) + SKIP; CASE(test_layer_normalization_4d_axis_negative_2) - // no filter + SKIP; +CASE(test_layer_normalization_4d_axis_negative_2_expanded) + SKIP; +CASE(test_layer_normalization_4d_axis_negative_2_expanded_ver18) + SKIP; CASE(test_layer_normalization_4d_axis_negative_3) - // no filter + SKIP; +CASE(test_layer_normalization_4d_axis_negative_3_expanded) + SKIP; +CASE(test_layer_normalization_4d_axis_negative_3_expanded_ver18) + SKIP; CASE(test_layer_normalization_4d_axis_negative_4) - // no filter + SKIP; +CASE(test_layer_normalization_4d_axis_negative_4_expanded) + SKIP; +CASE(test_layer_normalization_4d_axis_negative_4_expanded_ver18) + SKIP; CASE(test_layer_normalization_default_axis) - // no filter + SKIP; +CASE(test_layer_normalization_default_axis_expanded) + SKIP; +CASE(test_layer_normalization_default_axis_expanded_ver18) + SKIP; CASE(test_leakyrelu) // no filter CASE(test_leakyrelu_default) diff --git a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp index 048a8ea8d0..6d1e283787 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp @@ -652,3 +652,60 @@ "test_blackmanwindow_expanded", "test_blackmanwindow_symmetric", "test_blackmanwindow_symmetric_expanded", +"test_layer_normalization_2d_axis0", +"test_layer_normalization_2d_axis0_expanded", +"test_layer_normalization_2d_axis0_expanded_ver18", +"test_layer_normalization_2d_axis1", +"test_layer_normalization_2d_axis1_expanded", +"test_layer_normalization_2d_axis1_expanded_ver18", +"test_layer_normalization_2d_axis_negative_1", +"test_layer_normalization_2d_axis_negative_1_expanded", +"test_layer_normalization_2d_axis_negative_1_expanded_ver18", +"test_layer_normalization_2d_axis_negative_2", +"test_layer_normalization_2d_axis_negative_2_expanded", +"test_layer_normalization_2d_axis_negative_2_expanded_ver18", +"test_layer_normalization_3d_axis0_epsilon", +"test_layer_normalization_3d_axis0_epsilon_expanded", +"test_layer_normalization_3d_axis0_epsilon_expanded_ver18", +"test_layer_normalization_3d_axis1_epsilon", +"test_layer_normalization_3d_axis1_epsilon_expanded", +"test_layer_normalization_3d_axis1_epsilon_expanded_ver18", +"test_layer_normalization_3d_axis2_epsilon", +"test_layer_normalization_3d_axis2_epsilon_expanded", +"test_layer_normalization_3d_axis2_epsilon_expanded_ver18", +"test_layer_normalization_3d_axis_negative_1_epsilon", +"test_layer_normalization_3d_axis_negative_1_epsilon_expanded", +"test_layer_normalization_3d_axis_negative_1_epsilon_expanded_ver18", +"test_layer_normalization_3d_axis_negative_2_epsilon", +"test_layer_normalization_3d_axis_negative_2_epsilon_expanded", +"test_layer_normalization_3d_axis_negative_2_epsilon_expanded_ver18", +"test_layer_normalization_3d_axis_negative_3_epsilon", +"test_layer_normalization_3d_axis_negative_3_epsilon_expanded", +"test_layer_normalization_3d_axis_negative_3_epsilon_expanded_ver18", +"test_layer_normalization_4d_axis0", +"test_layer_normalization_4d_axis0_expanded", +"test_layer_normalization_4d_axis0_expanded_ver18", +"test_layer_normalization_4d_axis1", +"test_layer_normalization_4d_axis1_expanded", +"test_layer_normalization_4d_axis1_expanded_ver18", +"test_layer_normalization_4d_axis2", +"test_layer_normalization_4d_axis2_expanded", +"test_layer_normalization_4d_axis2_expanded_ver18", +"test_layer_normalization_4d_axis3", +"test_layer_normalization_4d_axis3_expanded", +"test_layer_normalization_4d_axis3_expanded_ver18", +"test_layer_normalization_4d_axis_negative_1", +"test_layer_normalization_4d_axis_negative_1_expanded", +"test_layer_normalization_4d_axis_negative_1_expanded_ver18", +"test_layer_normalization_4d_axis_negative_2", +"test_layer_normalization_4d_axis_negative_2_expanded", +"test_layer_normalization_4d_axis_negative_2_expanded_ver18", +"test_layer_normalization_4d_axis_negative_3", +"test_layer_normalization_4d_axis_negative_3_expanded", +"test_layer_normalization_4d_axis_negative_3_expanded_ver18", +"test_layer_normalization_4d_axis_negative_4", +"test_layer_normalization_4d_axis_negative_4_expanded", +"test_layer_normalization_4d_axis_negative_4_expanded_ver18", +"test_layer_normalization_default_axis", +"test_layer_normalization_default_axis_expanded", +"test_layer_normalization_default_axis_expanded_ver18", diff --git a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp index 3c50616c74..3e2064fb16 100644 --- a/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp +++ b/modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp @@ -337,63 +337,6 @@ "test_l1normalization_axis_last", "test_l2normalization_axis_0", "test_l2normalization_axis_1", -"test_layer_normalization_2d_axis0", -"test_layer_normalization_2d_axis0_expanded", -"test_layer_normalization_2d_axis0_expanded_ver18", -"test_layer_normalization_2d_axis1", -"test_layer_normalization_2d_axis1_expanded", -"test_layer_normalization_2d_axis1_expanded_ver18", -"test_layer_normalization_2d_axis_negative_1", -"test_layer_normalization_2d_axis_negative_1_expanded", -"test_layer_normalization_2d_axis_negative_1_expanded_ver18", -"test_layer_normalization_2d_axis_negative_2", -"test_layer_normalization_2d_axis_negative_2_expanded", -"test_layer_normalization_2d_axis_negative_2_expanded_ver18", -"test_layer_normalization_3d_axis0_epsilon", -"test_layer_normalization_3d_axis0_epsilon_expanded", -"test_layer_normalization_3d_axis0_epsilon_expanded_ver18", -"test_layer_normalization_3d_axis1_epsilon", -"test_layer_normalization_3d_axis1_epsilon_expanded", -"test_layer_normalization_3d_axis1_epsilon_expanded_ver18", -"test_layer_normalization_3d_axis2_epsilon", -"test_layer_normalization_3d_axis2_epsilon_expanded", -"test_layer_normalization_3d_axis2_epsilon_expanded_ver18", -"test_layer_normalization_3d_axis_negative_1_epsilon", -"test_layer_normalization_3d_axis_negative_1_epsilon_expanded", -"test_layer_normalization_3d_axis_negative_1_epsilon_expanded_ver18", -"test_layer_normalization_3d_axis_negative_2_epsilon", -"test_layer_normalization_3d_axis_negative_2_epsilon_expanded", -"test_layer_normalization_3d_axis_negative_2_epsilon_expanded_ver18", -"test_layer_normalization_3d_axis_negative_3_epsilon", -"test_layer_normalization_3d_axis_negative_3_epsilon_expanded", -"test_layer_normalization_3d_axis_negative_3_epsilon_expanded_ver18", -"test_layer_normalization_4d_axis0", -"test_layer_normalization_4d_axis0_expanded", -"test_layer_normalization_4d_axis0_expanded_ver18", -"test_layer_normalization_4d_axis1", -"test_layer_normalization_4d_axis1_expanded", -"test_layer_normalization_4d_axis1_expanded_ver18", -"test_layer_normalization_4d_axis2", -"test_layer_normalization_4d_axis2_expanded", -"test_layer_normalization_4d_axis2_expanded_ver18", -"test_layer_normalization_4d_axis3", -"test_layer_normalization_4d_axis3_expanded", -"test_layer_normalization_4d_axis3_expanded_ver18", -"test_layer_normalization_4d_axis_negative_1", -"test_layer_normalization_4d_axis_negative_1_expanded", -"test_layer_normalization_4d_axis_negative_1_expanded_ver18", -"test_layer_normalization_4d_axis_negative_2", -"test_layer_normalization_4d_axis_negative_2_expanded", -"test_layer_normalization_4d_axis_negative_2_expanded_ver18", -"test_layer_normalization_4d_axis_negative_3", -"test_layer_normalization_4d_axis_negative_3_expanded", -"test_layer_normalization_4d_axis_negative_3_expanded_ver18", -"test_layer_normalization_4d_axis_negative_4", -"test_layer_normalization_4d_axis_negative_4_expanded", -"test_layer_normalization_4d_axis_negative_4_expanded_ver18", -"test_layer_normalization_default_axis", -"test_layer_normalization_default_axis_expanded", -"test_layer_normalization_default_axis_expanded_ver18", "test_loop11", // Issue::'Graph' is not supported in function 'getLayerParams' "test_loop13_seq", // Issue::typeProto.has_tensor_type() in function 'populateNet' "test_loop16_seq_none", // Issue::Failed to allocate 179812654996800 bytes in function 'OutOfMemoryError'