From 06a78c239077fcd6de28b9fc4e5a548dc7d9d614 Mon Sep 17 00:00:00 2001 From: nklskyoy <31723634+nklskyoy@users.noreply.github.com> Date: Thu, 17 Jul 2025 07:55:07 +0200 Subject: [PATCH] Merge pull request #27527 from nklskyoy:trilu-layer Trilu layer #27527 Trilu layer https://onnx.ai/onnx/operators/onnx__Trilu.html is needed for importing paligemma Merged with https://github.com/opencv/opencv_extra/pull/1264 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../dnn/include/opencv2/dnn/all_layers.hpp | 6 ++ modules/dnn/src/init.cpp | 1 + modules/dnn/src/layers/trilu_layer.cpp | 85 +++++++++++++++++++ modules/dnn/src/onnx/onnx_importer2.cpp | 10 +++ modules/dnn/test/test_onnx_importer.cpp | 81 ++++++++++++++++++ 5 files changed, 183 insertions(+) create mode 100644 modules/dnn/src/layers/trilu_layer.cpp diff --git a/modules/dnn/include/opencv2/dnn/all_layers.hpp b/modules/dnn/include/opencv2/dnn/all_layers.hpp index b154587392..10916122b3 100644 --- a/modules/dnn/include/opencv2/dnn/all_layers.hpp +++ b/modules/dnn/include/opencv2/dnn/all_layers.hpp @@ -1376,6 +1376,12 @@ CV__DNN_INLINE_NS_BEGIN static Ptr create(const LayerParams& params); }; + class CV_EXPORTS TriluLayer : public Layer + { + public: + static Ptr create(const LayerParams& params); + }; + //! @} //! @} CV__DNN_INLINE_NS_END diff --git a/modules/dnn/src/init.cpp b/modules/dnn/src/init.cpp index b2d6200af2..fae419d52a 100644 --- a/modules/dnn/src/init.cpp +++ b/modules/dnn/src/init.cpp @@ -187,6 +187,7 @@ void initializeLayerFactory() CV_DNN_REGISTER_LAYER_CLASS(DepthToSpaceInt8, DepthToSpaceLayer) CV_DNN_REGISTER_LAYER_CLASS(SpaceToDepthInt8, SpaceToDepthLayer) + CV_DNN_REGISTER_LAYER_CLASS(Trilu, TriluLayer); CV_DNN_REGISTER_LAYER_CLASS(Crop, CropLayer); CV_DNN_REGISTER_LAYER_CLASS(Eltwise, EltwiseLayer); CV_DNN_REGISTER_LAYER_CLASS(NaryEltwise, NaryEltwiseLayer); diff --git a/modules/dnn/src/layers/trilu_layer.cpp b/modules/dnn/src/layers/trilu_layer.cpp new file mode 100644 index 0000000000..4a33177633 --- /dev/null +++ b/modules/dnn/src/layers/trilu_layer.cpp @@ -0,0 +1,85 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "../precomp.hpp" +#include +using namespace std; +namespace cv { namespace dnn { + +class TriluLayerImpl CV_FINAL : public TriluLayer { + public: + TriluLayerImpl(const LayerParams ¶ms) { + setParamsFrom(params); + upperTri = params.get("upper", true); + } + + virtual bool getMemoryShapes(const std::vector &inputs, + const int requiredOutputs, + std::vector &outputs, + std::vector &internals) const CV_OVERRIDE { + outputs.assign(1, inputs[0]); + return false; + } + + + void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE { + std::vector inputs, outputs; + inputs_arr.getMatVector(inputs); + outputs_arr.getMatVector(outputs); + inputs[0].copyTo(outputs[0]); + if (inputs[0].empty()) + return; + + int k = inputs.size() > 1 ? inputs[1].at(0,0) : 0; + const auto shape_input = shape(inputs[0]); + const int cdims = std::max(int(shape_input.size()) - 2, 0); + const int w = inputs[0].size[shape_input.size() - 1]; + const int h = shape_input.size() >= 2 ? inputs[0].size[shape_input.size() - 2] : 1; + + const int m = std::min(h,w); + int loops = 1; + for (int i = 0; i < cdims; ++i) + loops *= shape_input[i]; + + float *dst = outputs[0].ptr(); + auto fn = [&](const Range &r) { + + for (int i = r.start; i < r.end; i++) { + for(int l=0; l < m; l+=1) { + int cmin = upperTri ? 0 : (l + k + 1); + cmin = std::max(cmin, 0); + const int cmax = upperTri ? min(l + k -1, w-1) : w-1; + const int num_zeros = cmax - cmin + 1; + auto *cur_dst = dst + ((w * h) * i + (w * l + cmin)); + if (cmin < w && num_zeros > 0) + std::memset(cur_dst, 0, sizeof(float) * num_zeros); + } + } + }; + + double nstripes = loops * h * w / 1024.0; + parallel_for_(Range(0, loops), fn, nstripes); + } + + + void getTypes(const std::vector& inputs, + const int requiredOutputs, + const int requiredInternals, + std::vector& outputs, + std::vector& internals) const CV_OVERRIDE + { + outputs.assign(1, CV_32F); + } + + private: + bool upperTri; +}; + + +Ptr TriluLayer::create(const LayerParams& params) +{ + return makePtr(params); +} + +}} diff --git a/modules/dnn/src/onnx/onnx_importer2.cpp b/modules/dnn/src/onnx/onnx_importer2.cpp index b2e2ddf9f7..45609a991b 100644 --- a/modules/dnn/src/onnx/onnx_importer2.cpp +++ b/modules/dnn/src/onnx/onnx_importer2.cpp @@ -209,6 +209,7 @@ protected: void parseRange (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseReduce (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); + void parseTrilu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseResize (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseReshape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); void parseScatter (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto); @@ -1589,6 +1590,14 @@ void ONNXImporter2::parseResize(LayerParams& layerParams, const opencv_onnx::Nod addLayer(layerParams, node_proto, ninputs); } +void ONNXImporter2::parseTrilu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) +{ + int ninputs = node_proto.input_size(); + layerParams.type = "Trilu"; + addLayer(layerParams, node_proto, ninputs); + +} + void ONNXImporter2::parseUpsample(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto) { int n_inputs = node_proto.input_size(); @@ -2400,6 +2409,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI(int opset_version) dispatch["Concat"] = &ONNXImporter2::parseConcat; dispatch["If"] = &ONNXImporter2::parseIf; dispatch["Resize"] = &ONNXImporter2::parseResize; + dispatch["Trilu"] = &ONNXImporter2::parseTrilu; dispatch["Upsample"] = &ONNXImporter2::parseUpsample; dispatch["SoftMax"] = dispatch["Softmax"] = dispatch["LogSoftmax"] = &ONNXImporter2::parseSoftMax; dispatch["DetectionOutput"] = &ONNXImporter2::parseDetectionOutput; diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp index 822e5b70e6..9185ea6830 100644 --- a/modules/dnn/test/test_onnx_importer.cpp +++ b/modules/dnn/test/test_onnx_importer.cpp @@ -1327,6 +1327,87 @@ TEST_P(Test_ONNX_layers, Split_EltwiseMax) #endif testONNXModels("split_max"); } + +TEST_P(Test_ONNX_layers, trilu_tril) +{ + testONNXModels("trilu_tril", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_tril_neg) +{ + testONNXModels("trilu_tril_neg", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_tril_one_row) +{ + testONNXModels("trilu_tril_one_row", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_tril_one_row1D) +{ + testONNXModels("trilu_tril_one_row1D", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_tril_out_neg) +{ + testONNXModels("trilu_tril_out_neg", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_tril_out_pos) +{ + testONNXModels("trilu_tril_out_pos", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_tril_pos) +{ + testONNXModels("trilu_tril_pos", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_tril_square) +{ + testONNXModels("trilu_tril_square", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_tril_square_neg) +{ + testONNXModels("trilu_tril_square_neg", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_triu) +{ + testONNXModels("trilu_triu", npy, 0, 0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_triu_one_row) +{ + testONNXModels("trilu_triu_one_row", npy, 0,0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_triu_out_neg) +{ + testONNXModels("trilu_triu_out_neg_out", npy, 0,0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_triu_out_pos) +{ + testONNXModels("trilu_triu_out_pos", npy, 0,0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_triu_pos) +{ + testONNXModels("trilu_triu_pos", npy, 0,0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_triu_square) +{ + testONNXModels("trilu_triu_square", npy, 0,0,false, true, 2); +} + +TEST_P(Test_ONNX_layers, trilu_triu_square_neg) +{ + testONNXModels("trilu_triu_square_neg", npy, 0,0,false, true, 2); +} + // Fails with the new engine. Output shape [?, N, OutputSize] not supported by new graph engine TEST_P(Test_ONNX_layers, LSTM_Activations) {