mirror of
https://github.com/opencv/opencv.git
synced 2026-07-29 15:23:05 +04:00
Merge pull request #28963 from abhishek-gola:custom_layers
Added custom layer support in new DNN engine #28963 Closes: https://github.com/opencv/opencv/issues/26200 Merge with: https://github.com/opencv/opencv_extra/pull/1358 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
GitHub
@@ -202,6 +202,36 @@ Next we register a layer and try to import the model.
|
||||
|
||||
@snippet dnn/custom_layers.hpp Register ResizeBilinearLayer
|
||||
|
||||
## Example: custom layer from ONNX
|
||||
ONNX groups operators into **domains**. The standard operators live in the
|
||||
default domain `ai.onnx`; vendors and exporters often place their own ops in a
|
||||
named domain such as `my.namespace`. When OpenCV imports an ONNX node, it looks
|
||||
the op up in cv::dnn::LayerFactory by:
|
||||
|
||||
- the op_type alone, for nodes in the default `ai.onnx` domain (or no domain), and
|
||||
- `"<domain>.<op_type>"`, for nodes in any non-default domain.
|
||||
|
||||
Node attributes are passed through to the layer constructor as
|
||||
cv::dnn::LayerParams entries with the same names. Consider an op `MyCustomOp`
|
||||
with attributes `scale` and `bias` that computes `y = scale * x + bias`. The
|
||||
implementation can look like:
|
||||
|
||||
@snippet dnn/custom_layer_onnx.cpp CustomScaleBiasLayer
|
||||
|
||||
To import a model that uses this op, register the layer **before** calling
|
||||
cv::dnn::readNetFromONNX. Use cv::dnn::LayerFactory::registerLayer for runtime
|
||||
registration (and cv::dnn::LayerFactory::unregisterLayer when done) — pick the
|
||||
right key for the domain of the op as described above:
|
||||
|
||||
@snippet dnn/custom_layer_onnx.cpp Register CustomScaleBiasLayer
|
||||
|
||||
A complete runnable example is available at
|
||||
[samples/dnn/custom_layer_onnx.cpp](https://github.com/opencv/opencv/tree/5.x/samples/dnn/custom_layer_onnx.cpp).
|
||||
Tiny ONNX models exercising both the default-domain and custom-domain registration
|
||||
paths can be generated with
|
||||
[generate_custom_layer_models.py](https://github.com/opencv/opencv_extra/tree/5.x/testdata/dnn/onnx/generate_custom_layer_models.py)
|
||||
in the opencv_extra repository.
|
||||
|
||||
## Define a custom layer in Python
|
||||
The following example shows how to customize OpenCV's layers in Python.
|
||||
|
||||
|
||||
@@ -894,11 +894,12 @@ void ONNXImporter2::parseNode(const opencv_onnx::NodeProto& node_proto)
|
||||
<< node_proto.output_size() << " outputs from domain '"
|
||||
<< layer_type_domain << "'");*/
|
||||
|
||||
// Unknown domain: still try parseCustomLayer; addLayer() handles the miss.
|
||||
if (dispatch.empty())
|
||||
{
|
||||
CV_LOG_ERROR(NULL, "DNN/ONNX: missing dispatch map for domain='" << layer_type_domain << "'");
|
||||
rememberMissingOp(layer_type);
|
||||
return;
|
||||
CV_LOG_DEBUG(NULL, "DNN/ONNX: no built-in dispatch map for domain='"
|
||||
<< layer_type_domain << "', trying custom layer for '"
|
||||
<< layer_type << "'");
|
||||
}
|
||||
|
||||
node_inputs.clear();
|
||||
@@ -990,8 +991,21 @@ void ONNXImporter2::parseNeg(LayerParams& layerParams, const opencv_onnx::NodePr
|
||||
addLayer(layerParams, node_proto);
|
||||
}
|
||||
|
||||
// Lookup unknown ONNX ops in the user-side LayerFactory; non-default
|
||||
// domains are prefixed (e.g. "my.namespace.MyOp"), matching the old engine.
|
||||
void ONNXImporter2::parseCustomLayer(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
|
||||
{
|
||||
const std::string& name = layerParams.name;
|
||||
std::string& layer_type = layerParams.type;
|
||||
const std::string& layer_type_domain = node_proto.has_domain() ? node_proto.domain() : std::string();
|
||||
|
||||
if (!LayerFactory::isLayerRegistered(layer_type))
|
||||
{
|
||||
CV_LOG_INFO(NULL, "DNN/ONNX: unknown node type '" << layer_type
|
||||
<< "' (domain '" << layer_type_domain << "', node '" << name
|
||||
<< "'). Register a handler via CV_DNN_REGISTER_LAYER_CLASS() or LayerFactory::registerLayer().");
|
||||
}
|
||||
|
||||
parseSimpleLayers(layerParams, node_proto);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
//
|
||||
// Companion sample for tutorial:
|
||||
// doc/tutorials/dnn/dnn_custom_layers/dnn_custom_layers.md (ONNX section)
|
||||
//
|
||||
// Models used by this sample are generated by:
|
||||
// opencv_extra/testdata/dnn/onnx/generate_custom_layer_models.py
|
||||
// and live at:
|
||||
// opencv_extra/testdata/dnn/onnx/models/custom_layer_default_domain.onnx
|
||||
// opencv_extra/testdata/dnn/onnx/models/custom_layer_custom_domain.onnx
|
||||
|
||||
#include <opencv2/dnn.hpp>
|
||||
#include <opencv2/core.hpp>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
using namespace cv;
|
||||
using namespace cv::dnn;
|
||||
using namespace std;
|
||||
|
||||
//! [CustomScaleBiasLayer]
|
||||
// y = scale * x + bias, with scale/bias read from ONNX node attributes.
|
||||
class CustomScaleBiasLayer CV_FINAL : public Layer
|
||||
{
|
||||
public:
|
||||
CustomScaleBiasLayer(const LayerParams& params) : Layer(params)
|
||||
{
|
||||
scale = params.get<float>("scale", 1.f);
|
||||
bias = params.get<float>("bias", 0.f);
|
||||
}
|
||||
|
||||
static Ptr<Layer> create(LayerParams& params)
|
||||
{
|
||||
return makePtr<CustomScaleBiasLayer>(params);
|
||||
}
|
||||
|
||||
bool getMemoryShapes(const vector<MatShape>& inpts,
|
||||
const int /*requiredOutputs*/,
|
||||
vector<MatShape>& outShapes,
|
||||
vector<MatShape>& /*internals*/) const CV_OVERRIDE
|
||||
{
|
||||
outShapes.assign(1, inpts[0]);
|
||||
return false;
|
||||
}
|
||||
|
||||
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr,
|
||||
OutputArrayOfArrays) CV_OVERRIDE
|
||||
{
|
||||
vector<Mat> inps, outs;
|
||||
inputs_arr.getMatVector(inps);
|
||||
outputs_arr.getMatVector(outs);
|
||||
inps[0].convertTo(outs[0], outs[0].type(), scale, bias);
|
||||
}
|
||||
|
||||
private:
|
||||
float scale, bias;
|
||||
};
|
||||
//! [CustomScaleBiasLayer]
|
||||
|
||||
static void printShape(const string& tag, const Mat& m)
|
||||
{
|
||||
cout << tag;
|
||||
for (int d = 0; d < m.dims; ++d)
|
||||
cout << (d ? "x" : "") << m.size[d];
|
||||
cout << "\n";
|
||||
}
|
||||
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
const string keys =
|
||||
"{ help h | | Print help message }"
|
||||
"{ model m | | Path to ONNX model containing the custom op }"
|
||||
"{ op o | MyCustomOp | Op key for registration. Default-domain: just the op_type "
|
||||
"(e.g. MyCustomOp). Custom-domain: <domain>.<op_type> "
|
||||
"(e.g. my.namespace.MyDomainOp) }";
|
||||
CommandLineParser parser(argc, argv, keys);
|
||||
parser.about("Demonstrates importing an ONNX model that contains a custom (non-standard) op "
|
||||
"by registering a user-defined layer with cv::dnn::LayerFactory.");
|
||||
if (parser.has("help") || !parser.has("model"))
|
||||
{
|
||||
parser.printMessage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
const string modelPath = parser.get<string>("model");
|
||||
const string opKey = parser.get<string>("op");
|
||||
|
||||
//! [Register CustomScaleBiasLayer]
|
||||
// ONNX op-type lookup: layers in the default `ai.onnx` domain are registered
|
||||
// under their op_type; layers in a non-default domain are registered under
|
||||
// "<domain>.<op_type>" (e.g. "my.namespace.MyDomainOp").
|
||||
LayerFactory::registerLayer(opKey, CustomScaleBiasLayer::create);
|
||||
//! [Register CustomScaleBiasLayer]
|
||||
|
||||
Net net = readNetFromONNX(modelPath);
|
||||
if (net.empty())
|
||||
{
|
||||
cerr << "Failed to load model: " << modelPath << "\n";
|
||||
LayerFactory::unregisterLayer(opKey);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// The companion ONNX models accept a 1x3x4x4 float input.
|
||||
Mat input(vector<int>{1, 3, 4, 4}, CV_32F, Scalar(1.0f));
|
||||
net.setInput(input);
|
||||
Mat out = net.forward();
|
||||
|
||||
cout << "Loaded " << modelPath << " using custom op key '" << opKey << "'.\n";
|
||||
printShape("Input shape: ", input);
|
||||
printShape("Output shape: ", out);
|
||||
cout << "Output[0,0,0,0] = " << out.ptr<float>()[0]
|
||||
<< " (= scale * 1.0 + bias for the registered op)\n";
|
||||
|
||||
LayerFactory::unregisterLayer(opKey);
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user