1
0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 15:53:03 +04:00

Merge pull request #29579 from abhishek-gola:cumprod_causalconv_layers

Added Cumprod and Causalconv layers in new dnn engine #29579

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
2026-07-25 14:05:48 +05:30
committed by GitHub
parent b671e1e56f
commit b83e561526
9 changed files with 412 additions and 24 deletions
@@ -1920,6 +1920,16 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<AttentionOnnxAiLayer> create(const LayerParams &params);
};
class CV_EXPORTS CausalConvWithStateLayer : public Layer {
public:
static Ptr<CausalConvWithStateLayer> create(const LayerParams &params);
};
class CV_EXPORTS CumProdLayer : public Layer {
public:
static Ptr<CumProdLayer> create(const LayerParams &params);
};
class CV_EXPORTS GroupNormLayer : public Layer {
public:
static Ptr<GroupNormLayer> create(const LayerParams &params);
+2
View File
@@ -216,6 +216,7 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(Attention, AttentionLayer);
CV_DNN_REGISTER_LAYER_CLASS(SDPA, SDPALayer);
CV_DNN_REGISTER_LAYER_CLASS(AttentionOnnxAi, AttentionOnnxAiLayer);
CV_DNN_REGISTER_LAYER_CLASS(CausalConvWithState, CausalConvWithStateLayer);
CV_DNN_REGISTER_LAYER_CLASS(RotaryEmbedding, RotaryEmbeddingLayer);
CV_DNN_REGISTER_LAYER_CLASS(GroupNormalization, GroupNormLayer);
CV_DNN_REGISTER_LAYER_CLASS(Cast, CastLayer);
@@ -252,6 +253,7 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(LSTM2, LSTM2Layer);
CV_DNN_REGISTER_LAYER_CLASS(GRU, GRULayer);
CV_DNN_REGISTER_LAYER_CLASS(CumSum, CumSumLayer);
CV_DNN_REGISTER_LAYER_CLASS(CumProd, CumProdLayer);
CV_DNN_REGISTER_LAYER_CLASS(Einsum, EinsumLayer);
CV_DNN_REGISTER_LAYER_CLASS(Hardmax, HardmaxLayer);
CV_DNN_REGISTER_LAYER_CLASS(GatherND, GatherNDLayer);
@@ -0,0 +1,146 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "layers_common.hpp"
#include <opencv2/dnn/shape_utils.hpp>
#include <cmath>
namespace cv {
namespace dnn {
/*
Implementation of CausalConvWithState, as defined in ONNX specification:
https://onnx.ai/onnx/operators/onnx__CausalConvWithState.html
Opset 27 is covered.
*/
class CausalConvWithStateLayerImpl CV_FINAL : public CausalConvWithStateLayer
{
public:
CausalConvWithStateLayerImpl(const LayerParams& params)
{
setParamsFrom(params);
std::string act = params.get<std::string>("activation", "");
silu = (act == "silu" || act == "swish");
CV_Check(act, act.empty() || silu, "CausalConvWithState: unsupported activation");
}
bool supportBackend(int backendId) CV_OVERRIDE { return backendId == DNN_BACKEND_OPENCV; }
static bool present(const std::vector<Mat>& in, size_t i) { return in.size() > i && !in[i].empty(); }
void getTypes(const std::vector<MatType>& inputs, const int requiredOutputs, const int,
std::vector<MatType>& outputs, std::vector<MatType>& internals) const CV_OVERRIDE
{
CV_CheckType(inputs[0], inputs[0] == CV_32F || inputs[0] == CV_16F, "");
outputs.assign(requiredOutputs, inputs[0]);
internals.clear();
}
bool getMemoryShapes(const std::vector<MatShape>& inputs, const int,
std::vector<MatShape>& outputs, std::vector<MatShape>&) const CV_OVERRIDE
{
CV_CheckGE(inputs.size(), (size_t)2, "CausalConvWithState needs input and weight");
CV_CheckEQ(inputs[0].dims, 3, "input must be [batch, channels, seq]");
CV_CheckEQ(inputs[1].dims, 3, "weight must be [channels, 1, kernel]");
const int B = inputs[0][0], C = inputs[0][1], T = inputs[0][2], K = inputs[1][2];
outputs.assign(1, MatShape{B, C, T});
outputs.push_back(MatShape{B, C, K - 1});
return false;
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays) CV_OVERRIDE
{
std::vector<Mat> rawIn, rawOut;
inputs_arr.getMatVector(rawIn);
outputs_arr.getMatVector(rawOut);
const bool fp16 = rawIn[0].depth() == CV_16F;
std::vector<Mat> in32, out32;
if (fp16)
{
in32.resize(rawIn.size());
for (size_t i = 0; i < rawIn.size(); ++i)
if (!rawIn[i].empty()) rawIn[i].convertTo(in32[i], CV_32F);
out32.resize(rawOut.size());
for (size_t i = 0; i < rawOut.size(); ++i)
out32[i].create(rawOut[i].dims, rawOut[i].size.p, CV_32F);
}
std::vector<Mat>& inputs = fp16 ? in32 : rawIn;
std::vector<Mat>& outputs = fp16 ? out32 : rawOut;
const Mat& input = inputs[0];
const Mat& weight = inputs[1];
const bool has_bias = present(inputs, 2);
const bool has_past = present(inputs, 3);
const int B = input.size[0], C = input.size[1], T = input.size[2];
const int K = weight.size[2];
const int P = K - 1; // state / left-pad width
const float* Ip = input.ptr<float>();
const float* Wp = weight.ptr<float>();
const float* Bp = has_bias ? inputs[2].ptr<float>() : nullptr;
const float* Sp = has_past ? inputs[3].ptr<float>() : nullptr;
float* Op = outputs[0].ptr<float>();
float* PSp = outputs[1].ptr<float>();
parallel_for_(Range(0, B * C), [&](const Range& r)
{
std::vector<float> pad(P + T);
for (int bc = r.start; bc < r.end; ++bc)
{
const int c = bc % C;
const float* x = Ip + (size_t)bc * T;
const float* w = Wp + (size_t)c * K;
for (int j = 0; j < P; ++j)
pad[j] = has_past ? Sp[(size_t)bc * P + j] : 0.f;
for (int t = 0; t < T; ++t)
pad[P + t] = x[t];
const float bias = has_bias ? Bp[c] : 0.f;
float* o = Op + (size_t)bc * T;
for (int t = 0; t < T; ++t)
{
float acc = bias;
for (int k = 0; k < K; ++k)
acc += w[k] * pad[t + k];
o[t] = acc;
}
if (silu)
for (int t = 0; t < T; ++t)
o[t] = o[t] / (1.f + std::exp(-o[t]));
float* ps = PSp + (size_t)bc * P;
for (int j = 0; j < P; ++j)
ps[j] = pad[T + j];
}
});
if (fp16)
for (size_t i = 0; i < rawOut.size(); ++i)
out32[i].convertTo(rawOut[i], CV_16F);
}
int64 getFLOPS(const std::vector<MatShape>& inputs, const std::vector<MatShape>&) const CV_OVERRIDE
{
const int64 B = inputs[0][0], C = inputs[0][1], T = inputs[0][2], K = inputs[1][2];
return B * C * T * (2 * K + 4);
}
private:
bool silu = false;
};
Ptr<CausalConvWithStateLayer> CausalConvWithStateLayer::create(const LayerParams& params)
{
return makePtr<CausalConvWithStateLayerImpl>(params);
}
}} // namespace cv::dnn
+152
View File
@@ -0,0 +1,152 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "layers_common.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace cv {
namespace dnn {
/*
Implementation of CumProd, as defined in ONNX specification:
https://onnx.ai/onnx/operators/onnx__CumProd.html
Opset 26 is covered.
*/
class CumProdLayerImpl CV_FINAL : public CumProdLayer
{
public:
CumProdLayerImpl(const LayerParams& params)
{
axis_raw = params.get<int>("axis", 0);
exclusive_raw = params.get<int>("exclusive", 0);
reverse_raw = params.get<int>("reverse", 0);
setParamsFrom(params);
}
bool supportBackend(int backendId) CV_OVERRIDE { return backendId == DNN_BACKEND_OPENCV; }
bool getMemoryShapes(const std::vector<MatShape>& inputs, const int,
std::vector<MatShape>& outputs, std::vector<MatShape>&) const CV_OVERRIDE
{
outputs.assign(1, inputs[0]);
return false;
}
void getTypes(const std::vector<MatType>& inputs, const int, const int,
std::vector<MatType>& outputs, std::vector<MatType>&) const CV_OVERRIDE
{
CV_CheckType(inputs[0], inputs[0] == CV_32F || inputs[0] == CV_64F ||
inputs[0] == CV_32S || inputs[0] == CV_64S || inputs[0] == CV_16F, "");
outputs.assign(1, inputs[0]);
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
{
if (inputs_arr.depth() == CV_16F)
{
forward_fallback(inputs_arr, outputs_arr, internals_arr);
return;
}
std::vector<Mat> inputs, outputs;
inputs_arr.getMatVector(inputs);
outputs_arr.getMatVector(outputs);
CV_CheckTypeEQ(inputs[0].depth(), outputs[0].depth(), "");
switch (inputs[0].depth())
{
case CV_32F: forwardImpl<float>(inputs, outputs); break;
case CV_32S: forwardImpl<int32_t>(inputs, outputs); break;
case CV_64S: forwardImpl<int64_t>(inputs, outputs); break;
case CV_64F: forwardImpl<double>(inputs, outputs); break;
default: CV_Error(Error::BadDepth, "");
}
}
template <typename T>
void forwardImpl(const std::vector<Mat>& inputs, std::vector<Mat>& outputs)
{
const Mat& src_mat = inputs[0];
const T* src_ptr = src_mat.ptr<T>();
int axis = inputs.size() > 1 ? parseAxis(inputs[1]) : axis_raw;
axis = normalize_axis(axis, src_mat.dims);
Mat& dst_mat = outputs[0];
T* dst_ptr = dst_mat.ptr<T>();
const bool exclusive = exclusive_raw == 1;
const bool reverse = reverse_raw == 1;
// View data as [outer_size, target_size, inner_size] around the scan axis.
const size_t outer_size = src_mat.total(0, axis);
const size_t target_size = src_mat.size[axis];
const size_t inner_size = src_mat.total(axis + 1);
const size_t outer_step_length = target_size * inner_size;
const int target_start = reverse ? (int)target_size - 1 : 0;
const int target_stop = reverse ? -1 : (int)target_size;
const int target_delta = reverse ? -1 : 1;
const int target_step = target_delta * (int)inner_size;
const int exclusive_delta = exclusive ? target_step : 0;
// Each outer slice holds independent scans, so parallelize over it.
parallel_for_(Range(0, (int)outer_size), [&](const Range& range)
{
for (int outer_idx = range.start; outer_idx < range.end; outer_idx++)
{
const size_t target_offset = (size_t)outer_idx * outer_step_length;
// First element: multiplicative identity when exclusive, else the source value.
size_t first_inner_offset = target_offset + (size_t)target_start * inner_size;
if (exclusive)
for (size_t inner_idx = 0; inner_idx < inner_size; inner_idx++)
dst_ptr[first_inner_offset + inner_idx] = (T)1;
else
for (size_t inner_idx = 0; inner_idx < inner_size; inner_idx++)
dst_ptr[first_inner_offset + inner_idx] = src_ptr[first_inner_offset + inner_idx];
for (int target_idx = target_start + target_delta; target_idx != target_stop; target_idx += target_delta)
{
const size_t inner_offset = target_offset + (size_t)target_idx * inner_size;
for (size_t inner_idx = 0; inner_idx < inner_size; inner_idx++)
{
dst_ptr[inner_offset + inner_idx] = dst_ptr[inner_offset - target_step + inner_idx] *
src_ptr[inner_offset - exclusive_delta + inner_idx];
}
}
}
});
}
int64 getFLOPS(const std::vector<MatShape>& inputs, const std::vector<MatShape>&) const CV_OVERRIDE
{
return (int64)total(inputs[0]); // one multiply per element
}
int parseAxis(const Mat& axis_mat)
{
CV_CheckEQ(axis_mat.total(), 1u, "Axis tensor should contain single value");
if (axis_mat.type() == CV_32SC1)
return axis_mat.at<int32_t>(0);
Mat axis_mat_int;
axis_mat.convertTo(axis_mat_int, CV_32SC1);
return axis_mat_int.at<int32_t>(0);
}
int axis_raw;
int exclusive_raw;
int reverse_raw;
};
Ptr<CumProdLayer> CumProdLayer::create(const LayerParams& params)
{
return makePtr<CumProdLayerImpl>(params);
}
}} // namespace cv::dnn
+25
View File
@@ -208,6 +208,7 @@ protected:
void parseConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseConvTranspose (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseCumSum (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseCumProd (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseDepthSpaceOps (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseDetectionOutput (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parsePriorBox (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
@@ -285,6 +286,7 @@ protected:
// URL: https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md
void parseAttention (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseAttentionOnnxAi (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseCausalConvWithState (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseSDPA (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseDequantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQuantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
@@ -2120,6 +2122,22 @@ void ONNXImporter2::parseCumSum(LayerParams& layerParams, const opencv_onnx::Nod
addLayer(layerParams, node_proto, ninputs);
}
void ONNXImporter2::parseCumProd(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
int ninputs = node_proto.input_size();
CV_Assert(ninputs == 2);
layerParams.type = "CumProd";
if (net.isConstArg(node_inputs[1]))
{
Mat axisTensor;
net.argTensor(node_inputs[1]).convertTo(axisTensor, CV_32S);
CV_Assert(axisTensor.total() == 1);
layerParams.set("axis", axisTensor.at<int>(0));
ninputs = 1;
}
addLayer(layerParams, node_proto, ninputs);
}
// "Equal" "Greater" "Less" "Pow" "Add" "Sub" "Mul" "Div" "Sum" "Min" "Max" "GreaterOrEqual" "LessOrEqual" "And" "Or" "Xor"
void ONNXImporter2::parseElementWise(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_)
{
@@ -2632,6 +2650,11 @@ void ONNXImporter2::parseSDPA(LayerParams& params, const opencv_onnx::NodeProto&
addLayer(params, node_proto, 3);
}
void ONNXImporter2::parseCausalConvWithState(LayerParams& params, const opencv_onnx::NodeProto& node_proto) {
params.type = "CausalConvWithState";
addLayer(params, node_proto);
}
void ONNXImporter2::parseRoiAlign(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
layerParams.type = "RoiAlign";
@@ -2724,6 +2747,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI()
dispatch["DetectionOutput"] = &ONNXImporter2::parseDetectionOutput;
dispatch["PriorBox"] = &ONNXImporter2::parsePriorBox;
dispatch["CumSum"] = &ONNXImporter2::parseCumSum;
dispatch["CumProd"] = &ONNXImporter2::parseCumProd;
dispatch["SpaceToDepth"] = dispatch["DepthToSpace"] = &ONNXImporter2::parseDepthSpaceOps;
dispatch["ScatterElements"] = dispatch["Scatter"] = dispatch["ScatterND"] = &ONNXImporter2::parseScatter;
dispatch["Tile"] = &ONNXImporter2::parseTile;
@@ -2770,6 +2794,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI()
// Opset domain cannot be modified from onnx_graph_simplifier.cpp so this
// operator cannot be parsed if only added in buildDispatchMap_COM_MICROSOFT
dispatch["Attention"] = &ONNXImporter2::parseAttentionOnnxAi;
dispatch["CausalConvWithState"] = &ONNXImporter2::parseCausalConvWithState;
domain_dispatch_map[str_domain_ai_onnx] = dispatch;
}
@@ -2065,6 +2065,11 @@ TEST_P(Test_ONNX_conformance, Layer_Test)
default_l1 = std::max(default_l1, 2e-4);
default_lInf = std::max(default_lInf, 1e-3);
}
// fp16 CausalConvWithState keeps fp16 output precision (~8e-4 Inf) on fp32 targets.
if (name == "test_causal_conv_with_state_fp16" || name == "test_causal_conv_with_state_silu_fp16") {
default_l1 = std::max(default_l1, 2e-4);
default_lInf = std::max(default_lInf, 2e-3);
}
}
#ifdef HAVE_HALIDE
else if (backend == DNN_BACKEND_HALIDE)
@@ -2136,6 +2141,12 @@ TEST_P(Test_ONNX_conformance, Layer_Test)
default_l1 = std::max(default_l1, 2e-4);
default_lInf = std::max(default_lInf, 1e-3);
}
// fp16 CausalConvWithState keeps fp16 output precision (~8e-4 Inf) on fp32 targets
// (the layer falls back to the CPU path).
if (name == "test_causal_conv_with_state_fp16" || name == "test_causal_conv_with_state_silu_fp16") {
default_l1 = std::max(default_l1, 2e-4);
default_lInf = std::max(default_lInf, 2e-3);
}
}
#endif
else
@@ -3358,6 +3358,50 @@ CASE(test_causal_conv_with_state_with_bias_expanded)
SKIP;
CASE(test_causal_conv_with_state_with_past_state_expanded)
SKIP;
CASE(test_causal_conv_with_state_b1_c1_degenerate)
SKIP;
CASE(test_causal_conv_with_state_basic)
SKIP;
CASE(test_causal_conv_with_state_decode_step)
SKIP;
CASE(test_causal_conv_with_state_fp16)
SKIP;
CASE(test_causal_conv_with_state_kernel_size_one)
SKIP;
CASE(test_causal_conv_with_state_short_input_no_past_state)
SKIP;
CASE(test_causal_conv_with_state_silu)
SKIP;
CASE(test_causal_conv_with_state_silu_fp16)
SKIP;
CASE(test_causal_conv_with_state_silu_with_past_state)
SKIP;
CASE(test_causal_conv_with_state_swish_alias)
SKIP;
CASE(test_causal_conv_with_state_with_bias)
SKIP;
CASE(test_causal_conv_with_state_with_bias_and_past_state)
SKIP;
CASE(test_causal_conv_with_state_with_past_state)
SKIP;
CASE(test_cumprod_1d)
SKIP;
CASE(test_cumprod_1d_exclusive)
SKIP;
CASE(test_cumprod_1d_int32_exclusive)
SKIP;
CASE(test_cumprod_1d_reverse)
SKIP;
CASE(test_cumprod_1d_reverse_exclusive)
SKIP;
CASE(test_cumprod_2d_axis_0)
SKIP;
CASE(test_cumprod_2d_axis_1)
SKIP;
CASE(test_cumprod_2d_int32)
SKIP;
CASE(test_cumprod_2d_negative_axis)
SKIP;
CASE(test_flexattention_scaled_expanded_ver26)
SKIP;
CASE(test_range_bfloat16_type_positive_delta)
@@ -931,6 +931,28 @@
"test_causal_conv_with_state_with_bias_and_past_state_expanded",
"test_causal_conv_with_state_with_bias_expanded",
"test_causal_conv_with_state_with_past_state_expanded",
"test_causal_conv_with_state_b1_c1_degenerate",
"test_causal_conv_with_state_basic",
"test_causal_conv_with_state_decode_step",
"test_causal_conv_with_state_fp16",
"test_causal_conv_with_state_kernel_size_one",
"test_causal_conv_with_state_short_input_no_past_state",
"test_causal_conv_with_state_silu",
"test_causal_conv_with_state_silu_fp16",
"test_causal_conv_with_state_silu_with_past_state",
"test_causal_conv_with_state_swish_alias",
"test_causal_conv_with_state_with_bias",
"test_causal_conv_with_state_with_bias_and_past_state",
"test_causal_conv_with_state_with_past_state",
"test_cumprod_1d",
"test_cumprod_1d_exclusive",
"test_cumprod_1d_int32_exclusive",
"test_cumprod_1d_reverse",
"test_cumprod_1d_reverse_exclusive",
"test_cumprod_2d_axis_0",
"test_cumprod_2d_axis_1",
"test_cumprod_2d_int32",
"test_cumprod_2d_negative_axis",
"test_flexattention_scaled_expanded_ver26",
"test_range_bfloat16_type_positive_delta",
"test_range_float16_type_positive_delta",
@@ -379,30 +379,6 @@
"test_dequantizelinear_uint2",
"test_quantizelinear_int2",
"test_quantizelinear_uint2",
// CausalConvWithState op not supported
"test_causal_conv_with_state_b1_c1_degenerate",
"test_causal_conv_with_state_basic",
"test_causal_conv_with_state_decode_step",
"test_causal_conv_with_state_fp16",
"test_causal_conv_with_state_kernel_size_one",
"test_causal_conv_with_state_short_input_no_past_state",
"test_causal_conv_with_state_silu",
"test_causal_conv_with_state_silu_fp16",
"test_causal_conv_with_state_silu_with_past_state",
"test_causal_conv_with_state_swish_alias",
"test_causal_conv_with_state_with_bias",
"test_causal_conv_with_state_with_bias_and_past_state",
"test_causal_conv_with_state_with_past_state",
// CumProd op not supported
"test_cumprod_1d",
"test_cumprod_1d_exclusive",
"test_cumprod_1d_int32_exclusive",
"test_cumprod_1d_reverse",
"test_cumprod_1d_reverse_exclusive",
"test_cumprod_2d_axis_0",
"test_cumprod_2d_axis_1",
"test_cumprod_2d_int32",
"test_cumprod_2d_negative_axis",
// FlexAttention op not supported
"test_flexattention",
"test_flexattention_causal_mask",