1
0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 07:13:02 +04:00

Merge pull request #29577 from abhishek-gola:scan_layer

Add Scan layer to the new engine #29577

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
2026-07-25 14:09:47 +05:30
committed by GitHub
parent 859d550029
commit fe482bd575
10 changed files with 325 additions and 57 deletions
@@ -858,6 +858,28 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<LoopLayer> create(const LayerParams& params);
};
class CV_EXPORTS ScanLayer : public Layer
{
public:
/** Number of trailing inputs that are scanned (the rest are loop-carried state). */
virtual int numScanInputs() const = 0;
/** Per-scan-input axis to iterate over (empty => 0 for all). */
virtual const std::vector<int>& scanInputAxes() const = 0;
/** Per-scan-output axis to stack along (empty => 0 for all). */
virtual const std::vector<int>& scanOutputAxes() const = 0;
/** Per-scan-input direction, 1 = reverse (empty => forward for all). */
virtual const std::vector<int>& scanInputDirections() const = 0;
/** Per-scan-output direction, 1 = reverse (empty => forward for all). */
virtual const std::vector<int>& scanOutputDirections() const = 0;
/** ONNX-declared rank of each body scan output (-1 if unknown). A declared rank of
* 0 (scalar) marks an output that OpenCV stores as [1] but must stack into a rank-1
* tensor, not [T, 1]. Empty => unknown for all. */
virtual const std::vector<int>& scanOutputRanks() const = 0;
/** Factory: creates a ScanLayer implementation. */
static Ptr<ScanLayer> create(const LayerParams& params);
};
class CV_EXPORTS ConcatLayer : public Layer
{
public:
+48 -42
View File
@@ -137,6 +137,49 @@ struct BufferAllocator
releaseBuffer(toBuf);
}
// Allocate a Loop/Scan body, keeping its closure (outer-scope) args alive across it.
void assignSubgraphKeepingClosure(const Ptr<Graph>& body)
{
std::unordered_set<int> bodyDefined;
for (Arg ba : body->inputs())
bodyDefined.insert(ba.idx);
for (const Ptr<Layer>& blayer : body->prog()) {
if (!blayer) continue;
for (Arg bo : blayer->outputs)
bodyDefined.insert(bo.idx);
}
std::unordered_set<int> closureBumped;
for (const Ptr<Layer>& blayer : body->prog()) {
if (!blayer) continue;
for (Arg bi : blayer->inputs) {
if (bi.idx <= 0) continue;
if (bodyDefined.count(bi.idx)) continue;
if (netimpl->isConstArg(bi)) continue;
if (bufidxs[bi.idx] < 0) continue;
if (closureBumped.insert(bi.idx).second) {
usecounts[bi.idx]++;
buf_usecounts[bufidxs[bi.idx]]++;
}
}
}
std::vector<int> saved_freebufs = freebufs;
freebufs.clear();
assign(body);
freebufs = saved_freebufs;
for (int idx : closureBumped) {
int bidx = bufidxs[idx];
if (--usecounts[idx] == 0) {
if (bidx >= 0)
releaseBuffer(bidx);
} else if (bidx >= 0) {
CV_Assert(buf_usecounts[bidx] > 0);
--buf_usecounts[bidx];
}
}
}
template<typename _Tp> std::ostream&
dumpArgVec(std::ostream& strm, const std::string& name, const vector<_Tp>& vec) const
{
@@ -355,48 +398,11 @@ struct BufferAllocator
}
}
// The body reads names produced in the enclosing scope
// (closure references) without listing them in the Loop/If
// layer's inputs. Bump the outer-scope usecount of each
// such arg so its buffer survives until the subgraph runs.
std::unordered_set<int> bodyDefined;
for (Arg ba : body->inputs())
bodyDefined.insert(ba.idx);
for (const Ptr<Layer>& blayer : body->prog()) {
if (!blayer) continue;
for (Arg bo : blayer->outputs)
bodyDefined.insert(bo.idx);
}
std::unordered_set<int> closureBumped;
for (const Ptr<Layer>& blayer : body->prog()) {
if (!blayer) continue;
for (Arg bi : blayer->inputs) {
if (bi.idx <= 0) continue;
if (bodyDefined.count(bi.idx)) continue;
if (netimpl->isConstArg(bi)) continue;
if (bufidxs[bi.idx] < 0) continue;
if (closureBumped.insert(bi.idx).second) {
usecounts[bi.idx]++;
buf_usecounts[bufidxs[bi.idx]]++;
}
}
}
std::vector<int> saved_freebufs = freebufs;
freebufs.clear();
assign(body);
freebufs = saved_freebufs;
for (int idx : closureBumped) {
int bidx = bufidxs[idx];
if (--usecounts[idx] == 0) {
if (bidx >= 0)
releaseBuffer(bidx);
} else if (bidx >= 0) {
CV_Assert(buf_usecounts[bidx] > 0);
--buf_usecounts[bidx];
}
}
assignSubgraphKeepingClosure(body);
} else if (opname == "Scan") {
auto subgraphs = layer->subgraphs();
CV_Assert(subgraphs && subgraphs->size() == 1);
assignSubgraphKeepingClosure(subgraphs->at(0));
}
for (auto out: outputs) {
+1
View File
@@ -86,6 +86,7 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(If, IfLayer);
CV_DNN_REGISTER_LAYER_CLASS(Loop, LoopLayer);
CV_DNN_REGISTER_LAYER_CLASS(Scan, ScanLayer);
CV_DNN_REGISTER_LAYER_CLASS(Concat, ConcatLayer);
CV_DNN_REGISTER_LAYER_CLASS(Concat2, Concat2Layer);
CV_DNN_REGISTER_LAYER_CLASS(ConstantOfShape, ConstantOfShapeLayer);
+1 -1
View File
@@ -376,7 +376,7 @@ std::ostream& Layer::dump(std::ostream& strm, int indent, bool comma) const
std::vector<std::string> names;
if (opname == "If")
names = {"then", "else"};
else if (opname == "Loop")
else if (opname == "Loop" || opname == "Scan")
names = {"body"};
else {
CV_Error(Error::StsError,
+76
View File
@@ -0,0 +1,76 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "../net_impl.hpp"
#include "layers_common.hpp"
#include <opencv2/dnn.hpp>
namespace cv { namespace dnn {
CV__DNN_INLINE_NS_BEGIN
// ONNX op: Scan — https://onnx.ai/onnx/operators/onnx__Scan.html
// Supported opsets: 9 .. latest (opset-9 dataflow semantics).
// Opset-8 Scan (leading batch dimension + optional sequence_lens input) has different
// semantics and is NOT supported.
//
// Net::Impl::forwardGraph drives the loop; this layer only holds the body and its attributes.
class ScanLayerImpl CV_FINAL : public ScanLayer
{
public:
explicit ScanLayerImpl(const LayerParams& params)
{
setParamsFrom(params);
num_scan_inputs = params.get<int>("num_scan_inputs");
readIntList(params, "scan_input_axes", input_axes);
readIntList(params, "scan_output_axes", output_axes);
readIntList(params, "scan_input_directions", input_dirs);
readIntList(params, "scan_output_directions", output_dirs);
readIntList(params, "scan_output_ranks", output_ranks);
}
std::vector<Ptr<Graph> >* subgraphs() const CV_OVERRIDE { return &body_; }
bool dynamicOutputShapes() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape>&,
const int requiredOutputs,
std::vector<MatShape>& outputs,
std::vector<MatShape>& internals) const CV_OVERRIDE
{
outputs.assign(std::max(1, requiredOutputs), MatShape());
internals.clear();
return false;
}
int numScanInputs() const CV_OVERRIDE { return num_scan_inputs; }
const std::vector<int>& scanInputAxes() const CV_OVERRIDE { return input_axes; }
const std::vector<int>& scanOutputAxes() const CV_OVERRIDE { return output_axes; }
const std::vector<int>& scanInputDirections() const CV_OVERRIDE { return input_dirs; }
const std::vector<int>& scanOutputDirections() const CV_OVERRIDE { return output_dirs; }
const std::vector<int>& scanOutputRanks() const CV_OVERRIDE { return output_ranks; }
private:
static void readIntList(const LayerParams& params, const std::string& name, std::vector<int>& out)
{
out.clear();
if (!params.has(name)) return;
const DictValue& v = params.get(name);
out.resize(v.size());
for (int i = 0; i < v.size(); ++i) out[i] = v.get<int>(i);
}
int num_scan_inputs = 0;
std::vector<int> input_axes, output_axes, input_dirs, output_dirs, output_ranks;
mutable std::vector<Ptr<Graph> > body_;
};
Ptr<ScanLayer> ScanLayer::create(const LayerParams& params)
{
return makePtr<ScanLayerImpl>(params);
}
CV__DNN_INLINE_NS_END
}} // namespace cv::dnn
+111
View File
@@ -1127,6 +1127,49 @@ void Net::Impl::setGraphInput(Ptr<Graph>& graph, size_t idx, const Mat& m)
}
}
// Slice a Scan input at index `idx` along `axis`, removing that axis (contiguous result).
static Mat sliceScanAxis(const Mat& m, int axis, int idx)
{
std::vector<Range> r(m.dims, Range::all());
r[axis] = Range(idx, idx + 1);
Mat sub = m(r).clone();
std::vector<int> ns;
for (int d = 0; d < m.dims; d++)
if (d != axis) ns.push_back(m.size[d]);
if (ns.empty()) ns.push_back(1);
return sub.reshape(0, (int)ns.size(), &ns[0]);
}
// Stack per-iteration Scan outputs into one tensor with a new axis at `axis`.
static Mat stackScanAxis(const std::vector<Mat>& perIter, int axis, bool reverse)
{
if (perIter.empty()) return Mat();
const Mat& first = perIter[0];
const int e = first.dims, T = (int)perIter.size();
if (axis < 0) axis += e + 1;
CV_Assert(axis >= 0 && axis <= e);
std::vector<int> os, ss;
for (int d = 0; d < e; d++) {
if (d == axis) { os.push_back(T); ss.push_back(1); }
os.push_back(first.size[d]);
ss.push_back(first.size[d]);
}
if (axis == e) { os.push_back(T); ss.push_back(1); }
Mat stacked;
stacked.create((int)os.size(), &os[0], first.type());
for (int k = 0; k < T; k++) {
int idx = reverse ? (T - 1 - k) : k;
std::vector<Range> r(os.size(), Range::all());
r[axis] = Range(k, k + 1);
Mat dst = stacked(r);
Mat src = perIter[idx].reshape(0, (int)ss.size(), &ss[0]);
src.copyTo(dst);
}
return stacked;
}
void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
OutputArrayOfArrays outputs_, bool isMainGraph)
{
@@ -1214,6 +1257,7 @@ void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
else {
Ptr<IfLayer> iflayer = layer.dynamicCast<IfLayer>();
Ptr<LoopLayer> loopLayer = layer.dynamicCast<LoopLayer>();
Ptr<ScanLayer> scanLayer = layer.dynamicCast<ScanLayer>();
if (iflayer) {
int branch = iflayer->branch(inpMats[0]);
Ptr<Graph> subgraph = subgraphs->at(branch);
@@ -1306,6 +1350,73 @@ void Net::Impl::forwardGraph(Ptr<Graph>& graph, InputArrayOfArrays inputs_,
outMats[n_state + i] = stacked;
}
}
else if (scanLayer) {
CV_Assert(subgraphs->size() == 1);
Ptr<Graph> body = subgraphs->at(0);
const int M = scanLayer->numScanInputs();
const int bodyNIn = (int)body->inputs().size();
const int S = bodyNIn - M; // loop-carried state count
const int K = (int)body->outputs().size() - S; // scan output count
CV_Assert(M > 0 && S >= 0 && K >= 0);
// Reject opset-8 (batch dim + sequence_lens); those semantics are not implemented.
CV_CheckEQ((int)inpMats.size(), S + M,
"Scan: opset-8 form (batch dim + sequence_lens) is not supported; re-export with opset>=9");
const std::vector<int>& iax = scanLayer->scanInputAxes();
const std::vector<int>& oax = scanLayer->scanOutputAxes();
const std::vector<int>& idir = scanLayer->scanInputDirections();
const std::vector<int>& odir = scanLayer->scanOutputDirections();
const std::vector<int>& orank = scanLayer->scanOutputRanks();
std::vector<Mat> scanIn(M);
std::vector<int> inAxis(M);
std::vector<char> inRev(M);
int T = -1;
for (int j = 0; j < M; j++) {
scanIn[j] = inpMats[S + j];
int ax = iax.empty() ? 0 : iax[j];
if (ax < 0) ax += scanIn[j].dims;
inAxis[j] = ax;
inRev[j] = (char)(!idir.empty() && idir[j] != 0);
int len = scanIn[j].size[ax];
if (T < 0) T = len; else CV_Assert(T == len);
}
CV_Assert(T >= 0);
std::vector<Mat> state(S);
for (int i = 0; i < S; i++) state[i] = inpMats[i];
std::vector<std::vector<Mat> > history(K);
std::vector<Mat> inputs(bodyNIn), outputs;
for (int t = 0; t < T; t++) {
for (int i = 0; i < S; i++) inputs[i] = state[i];
for (int j = 0; j < M; j++) {
int idx = inRev[j] ? (T - 1 - t) : t;
inputs[S + j] = sliceScanAxis(scanIn[j], inAxis[j], idx);
}
forwardGraph(body, inputs, outputs, false);
// Deep-copy: body buffers are recycled across iterations.
// TODO: alias state in/out buffers like Loop to drop this copy.
for (int i = 0; i < S; i++) state[i] = outputs[i].clone();
for (int k = 0; k < K; k++) history[k].push_back(outputs[S + k].clone());
}
outMats.assign(state.begin(), state.end());
outMats.resize(S + K);
for (int k = 0; k < K; k++) {
int ax = oax.empty() ? 0 : oax[k];
bool rev = !odir.empty() && odir[k] != 0;
// 0-D scalar output is stored as [1]; drop it so T scalars stack to [T], not [T,1].
if (k < (int)orank.size() && orank[k] == 0) {
for (Mat& e : history[k])
e = e.reshape(0, 0, nullptr);
}
outMats[S + k] = stackScanAxis(history[k], ax, rev);
}
}
else {
CV_Error_(Error::StsNotImplemented,
("unknown layer type '%s' with subgraphs", layer->type.c_str()));
+56 -8
View File
@@ -202,6 +202,7 @@ protected:
void parseClip (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseLoop (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseScan (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseIf (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseConstant (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseConstantOfShape (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
@@ -742,19 +743,25 @@ Net ONNXImporter2::parseModel()
bool ONNXImporter2::parseValueInfo(const opencv_onnx::ValueInfoProto& valueInfoProto, ArgData& data)
{
CV_Assert(valueInfoProto.has_name());
CV_Assert(valueInfoProto.has_type());
// Subgraph body value-infos may omit type/shape; leave unset, inferred at runtime.
if (!valueInfoProto.has_type())
return true;
const opencv_onnx::TypeProto& typeProto = valueInfoProto.type();
CV_Assert(typeProto.has_tensor_type());
if (!typeProto.has_tensor_type())
return true;
const opencv_onnx::TypeProto::Tensor& tensor = typeProto.tensor_type();
CV_Assert(tensor.has_shape());
const opencv_onnx::TensorShapeProto& tensorShape = tensor.shape();
auto elem_type = tensor.elem_type();
data.type = dataType2cv(elem_type);
if (data.type < 0) {
CV_Error(Error::StsNotImplemented, format("unsupported datatype '%s'", dataType2str(elem_type).c_str()));
if (tensor.has_elem_type()) {
auto elem_type = tensor.elem_type();
data.type = dataType2cv(elem_type);
if (data.type < 0) {
CV_Error(Error::StsNotImplemented, format("unsupported datatype '%s'", dataType2str(elem_type).c_str()));
}
}
if (!tensor.has_shape())
return true;
const opencv_onnx::TensorShapeProto& tensorShape = tensor.shape();
int dim_size = tensorShape.dim_size();
CV_CheckGE(dim_size, 0, "");
MatShape shape(dim_size);
@@ -1723,6 +1730,46 @@ void ONNXImporter2::parseLoop(LayerParams& layerParams,
*loopLayer->subgraphs() = subgraphs;
}
void ONNXImporter2::parseScan(LayerParams& layerParams,
const opencv_onnx::NodeProto& node_proto)
{
CV_Assert(layerParams.has("num_scan_inputs"));
layerParams.type = "Scan";
std::vector<Arg> saved_inputs = node_inputs, saved_outputs = node_outputs;
std::vector<Ptr<Graph> > subgraphs(1);
for (int i = 0; i < node_proto.attribute_size(); ++i)
{
const auto& attr = node_proto.attribute(i);
if (attr.name() == "body")
{
opencv_onnx::GraphProto body = attr.g();
subgraphs[0] = parseGraph(&body, false);
}
}
CV_Assert(!subgraphs[0].empty());
node_inputs = saved_inputs;
node_outputs = saved_outputs;
const int num_scan_inputs = layerParams.get<int>("num_scan_inputs");
const int n_state = (int)subgraphs[0]->inputs().size() - num_scan_inputs;
const int n_scan_out = (int)subgraphs[0]->outputs().size() - n_state;
if (n_state >= 0 && n_scan_out > 0)
{
std::vector<int> scan_output_ranks(n_scan_out);
for (int k = 0; k < n_scan_out; ++k)
{
const Arg& a = subgraphs[0]->outputs()[n_state + k];
scan_output_ranks[k] = netimpl->args.at(a.idx).shape.dims;
}
layerParams.set("scan_output_ranks", DictValue::arrayInt(scan_output_ranks.data(), n_scan_out));
}
addLayer(layerParams, node_proto);
Ptr<Layer>& scanLayer = curr_prog.back();
*scanLayer->subgraphs() = subgraphs;
}
void ONNXImporter2::parseIf(LayerParams& layerParams,
const opencv_onnx::NodeProto& node_proto)
{
@@ -2718,6 +2765,7 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI()
dispatch["GatherElements"] = &ONNXImporter2::parseGatherElements;
dispatch["Concat"] = &ONNXImporter2::parseConcat;
dispatch["Loop"] = &ONNXImporter2::parseLoop;
dispatch["Scan"] = &ONNXImporter2::parseScan;
dispatch["If"] = &ONNXImporter2::parseIf;
dispatch["Resize"] = &ONNXImporter2::parseResize2;
dispatch["Size"] = &ONNXImporter2::parseSize;
@@ -2423,8 +2423,12 @@ CASE(test_rms_normalization_4d_axis_negative_4_expanded)
SKIP;
CASE(test_rms_normalization_default_axis_expanded)
SKIP;
CASE(test_scan9_multi_state)
SKIP;
CASE(test_scan9_scalar)
SKIP;
CASE(test_scan9_sum)
// no filter
SKIP;
CASE(test_scan_sum)
// no filter
CASE(test_scatter_elements_with_axis)
@@ -736,6 +736,9 @@
"test_reduce_sum_square_empty_set_expanded",
"test_reduce_log_sum_exp_empty_set_expanded",
"test_loop11",
"test_scan9_multi_state", // Scan supported only by the new graph engine
"test_scan9_scalar", // ---- same as above ---
"test_scan9_sum", // ---- same as above ---
"test_eyelike_populate_off_main_diagonal",
"test_eyelike_with_dtype",
"test_eyelike_without_dtype",
@@ -272,8 +272,8 @@
"test_reversesequence_batch", // Issue:: Parser: Can't create layer "onnx_node_output_0!y" of type "ReverseSequence" in function 'getLayerInstance'
"test_reversesequence_time", // ---- same as above ---
"test_rnn_seq_length", // Issue:: Parser: Can't create layer "onnx_node_output_1!Y_h" of type "RNN" in function 'getLayerInstance'
"test_scan9_sum", // Issue:: Parser: 'Graph' is not supported in function 'getLayerParams'
"test_scan_sum", // ---- same as above ---
// Scan edge cases beyond the opset-9+ dataflow the new engine supports:
"test_scan_sum", // opset-8 Scan (leading batch dim + sequence_lens, different semantics)
"test_sequence_insert_at_back", // Issue:: Parser: typeProto.has_tensor_type() in function 'populateNet'
"test_sequence_insert_at_front", // ---- same as above ---
"test_sequence_map_add_1_sequence_1_tensor",
@@ -420,9 +420,6 @@
"test_linear_attention_no_past_explicit_zeros_expanded",
"test_linear_attention_prefill_with_past",
"test_linear_attention_prefill_with_past_expanded",
// Scan op not supported
"test_scan9_multi_state",
"test_scan9_scalar",
// misc unsupported (expanded subgraphs / new ops)
"test_castlike_no_saturate_FLOAT_to_FLOAT8E4M3FNUZ_expanded",
"test_castlike_no_saturate_FLOAT_to_FLOAT8E4M3FN_expanded",