potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-30 07:43:03 +04:00
Code Packages Releases Wiki Activity

Merge pull request #28811 from abhishek-gola:qlinear_support

Browse Source 
Added QLinear layer support #28811

closes: https://github.com/opencv/opencv/issues/26310

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
2026-04-28 20:08:29 +05:30
committed by GitHub
parent a17db08c21
commit c83b86eb57
32 changed files with 1341 additions and 735 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/include/opencv2/dnn/all_layers.hpp
+97
View File
@@ -382,10 +382,28 @@ CV__DNN_INLINE_NS_BEGIN
bool ceil_mode;
};
struct Conv2Int8Params
{
String name;
std::vector<int> strides, dilations, pads;
int ngroups = 1;
AutoPadding auto_pad = AUTO_PAD_NONE;
bool ceil_mode = false;
float input_sc = 1.f;
int input_zp = 0;
float output_sc = 1.f;
int output_zp = 0;
bool per_channel = true;
bool input_is_u8 = false;
// blobs[0] = quantized weights, blobs[1] = fused bias, blobs[2] = output multiplier
Mat weights, bias, outputMultiplier;
};
class CV_EXPORTS Conv2Int8Layer : public Layer
{
public:
static Ptr<Conv2Int8Layer> create(const LayerParams& params);
static Ptr<Conv2Int8Layer> create(const Conv2Int8Params& params);
int input_zp, output_zp;
float input_sc, output_sc;
@@ -487,6 +505,21 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<PoolingLayerInt8> create(const LayerParams& params);
};
struct Pool2Int8Params
{
String name;
std::vector<int> kernel_shape, strides, dilations, pads;
AutoPadding auto_pad = AUTO_PAD_NONE;
bool ceil_mode = false;
bool is_global_pooling = false;
bool is_max_pool = true;
bool count_include_pad = false;
float input_sc = 1.f;
int input_zp = 0;
float output_sc = 1.f;
int output_zp = 0;
};
// New-engine int8 pooling with block memory layout (DATA_LAYOUT_BLOCK).
// Created by the QDQ graph fusion pass (graph_fusion_qdq.cpp) when it
// detects a DequantizeLinear -> Pooling -> QuantizeLinear pattern.
@@ -495,6 +528,7 @@ CV__DNN_INLINE_NS_BEGIN
{
public:
static Ptr<Pool2Int8Layer> create(const LayerParams& params);
static Ptr<Pool2Int8Layer> create(const Pool2Int8Params& params);
int input_zp, output_zp;
float input_sc, output_sc;
@@ -591,6 +625,21 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<InnerProductLayer> create(const LayerParams& params);
};
struct InnerProductInt8Params
{
String name;
int axis = 1;
int num_output = 0;
float input_sc = 1.f;
int input_zp = 0;
float output_sc = 1.f;
int output_zp = 0;
int output_type = CV_8S;
bool per_channel = true;
// blobs[0] = weights, blobs[1] = bias, blobs[2] = output multiplier
Mat weights, bias, outputMultiplier;
};
class CV_EXPORTS InnerProductLayerInt8 : public InnerProductLayer
{
public:
@@ -602,6 +651,7 @@ CV__DNN_INLINE_NS_BEGIN
// of per-Channel quantization. Otherwise, that means this layer contains per-Tensor quantized parameters.
bool per_channel;
static Ptr<InnerProductLayerInt8> create(const LayerParams& params);
static Ptr<InnerProductLayerInt8> create(const InnerProductInt8Params& params);
};
class CV_EXPORTS MVNLayer : public Layer
@@ -1239,6 +1289,17 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<ThresholdedReluLayer> create(const LayerParams &params);
};
struct ActivationInt8Params
{
String name;
String activationType; // "ReLUInt8", "SigmoidInt8", etc.
float input_sc = 1.f;
int input_zp = 0;
float output_sc = 1.f;
int output_zp = 0;
Mat activationLUT;
};
class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer
{
public:
@@ -1247,6 +1308,7 @@ CV__DNN_INLINE_NS_BEGIN
Mat activationLUT;
static Ptr<ActivationLayerInt8> create(const LayerParams &params);
static Ptr<ActivationLayerInt8> create(const ActivationInt8Params &params);
};
class CV_EXPORTS SignLayer : public ActivationLayer
@@ -1303,10 +1365,21 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<EltwiseLayerInt8> create(const LayerParams &params);
};
struct Eltwise2Int8Params
{
String name;
std::vector<float> input_scales;
std::vector<int> input_zeropoints;
float output_sc = 1.f;
int output_zp = 0;
bool with_relu = false;
};
class CV_EXPORTS Eltwise2Int8Layer : public Layer
{
public:
static Ptr<Eltwise2Int8Layer> create(const LayerParams& params);
static Ptr<Eltwise2Int8Layer> create(const Eltwise2Int8Params& params);
std::vector<float> scales;
std::vector<int> zeropoints;
@@ -1722,6 +1795,30 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<MatMulLayer> create(const LayerParams &params);
};
struct MatMulInt8Params
{
String name;
int num_output = 0;
int inp_dims = 2;
float input_sc = 1.f;
int input_zp = 0;
float output_sc = 1.f;
int output_zp = 0;
int output_type = CV_8S;
bool per_channel = true;
Mat weights, bias, outputMultiplier;
};
class CV_EXPORTS MatMulInt8Layer : public MatMulLayer
{
public:
int input_zp, output_zp;
float input_sc, output_sc;
int output_type;
bool per_channel;
static Ptr<MatMulInt8Layer> create(const MatMulInt8Params& params);
};
class CV_EXPORTS ExpandLayer : public Layer
{
public:
modules/dnn/include/opencv2/dnn/dnn.hpp
+7
View File
@@ -495,6 +495,13 @@ CV__DNN_INLINE_NS_BEGIN
// so that forward() can assume that the outputs are already allocated.
virtual bool dynamicOutputShapes() const;
// returns true if the layer only rearranges data without changing values.
// Examples: Flatten, Reshape, Transpose, Permute, Squeeze, Unsqueeze,
// Concat, Split, Slice, Tile, MaxPool.
// Used by QDQ fusion to elide redundant dequantize-quantize pairs
// when the scale and zero point are the same.
virtual bool isDataShuffling() const;
// dumps attributes of the layer (e.g. strides, dilations in Convolution, MaxPool)
virtual std::ostream& dumpAttrs(std::ostream& strm, int indent) const;
modules/dnn/src/graph_fusion_qdq.cpp
+473 -215
View File
@@ -78,6 +78,17 @@ struct ModelFusionQDQ
{ p->output_sc = sc; p->output_zp = zp; }
}
bool areDqArgsConst(const DequantizeLinearLayer* dq) const
{
if (!dq || dq->inputs.size() < 2)
return false;
if (!netimpl->isConstArg(dq->inputs[1]))
return false;
if (dq->inputs.size() >= 3 && !netimpl->isConstArg(dq->inputs[2]))
return false;
return true;
}
template<typename LayerT>
bool getQdqPatternContext(Layer* layer_ptr,
size_t ninputs,
@@ -96,6 +107,8 @@ struct ModelFusionQDQ
q_data_in = inputs[0];
out_scale = inputs[1];
out_zp = inputs[2];
if (!netimpl->isConstArg(out_scale) || !netimpl->isConstArg(out_zp))
return false;
mid_layer_idx = producer_of.at(q_data_in.idx);
mid_layer = getLayer<LayerT>(newprog, mid_layer_idx);
return mid_layer != 0;
@@ -152,7 +165,8 @@ struct ModelFusionQDQ
getLayer<DequantizeLinearLayer>(newprog, dq_idx);
if (!dq || dq->inputs.size() < 3 ||
usecounts.at(add_inp.idx) != 1) {
usecounts.at(add_inp.idx) != 1 ||
!areDqArgsConst(dq)) {
break;
}
dq_ptrs.push_back(dq);
@@ -184,27 +198,25 @@ struct ModelFusionQDQ
? (int)elt_out_zp_m.at<uint8_t>(0)
: (int)elt_out_zp_m.at<int8_t>(0);
LayerParams eltwiseParams = makeLayerParamsFromOriginal(add, "Eltwise2Int8");
eltwiseParams.blobs.clear();
eltwiseParams.set("input_scales", DictValue::arrayReal(in_scales.data(), (int)in_scales.size()));
eltwiseParams.set("input_zeropoints", DictValue::arrayInt(in_zps.data(), (int)in_zps.size()));
eltwiseParams.set("scales", out_scale_val);
eltwiseParams.set("zeropoints", out_zp_val);
Ptr<Layer> eltwiseInt8 = createFusedLayer(eltwiseParams);
if (!eltwiseInt8.empty()) {
CV_Assert(dynamic_cast<Eltwise2Int8Layer*>(eltwiseInt8.get()));
fused_layer_idx = add_layer_idx;
newprog[add_layer_idx] = eltwiseInt8;
fused_inputs.swap(int8_inputs);
removed_args.push_back(q_data_in); // float add_out
for (const Arg& add_inp : add->inputs)
removed_args.push_back(add_inp);
Eltwise2Int8Params ep;
ep.name = add->name;
ep.input_scales = in_scales;
ep.input_zeropoints = in_zps;
ep.output_sc = out_scale_val;
ep.output_zp = out_zp_val;
Ptr<Eltwise2Int8Layer> eltwiseInt8 = Eltwise2Int8Layer::create(ep);
eltwiseInt8->netimpl = netimpl;
fused_layer_idx = add_layer_idx;
newprog[add_layer_idx] = eltwiseInt8;
fused_inputs.swap(int8_inputs);
removed_args.push_back(q_data_in); // float add_out
for (const Arg& add_inp : add->inputs)
removed_args.push_back(add_inp);
for (int dq_prog_idx : dq_prog_indices)
newprog[dq_prog_idx] = Ptr<Layer>();
for (int dq_prog_idx : dq_prog_indices)
newprog[dq_prog_idx] = Ptr<Layer>();
break;
}
break;
}
}
@@ -222,6 +234,7 @@ struct ModelFusionQDQ
const int relu_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
const bool relu_in_int8 = (relu_in_type == CV_8S || relu_in_type == CV_8U);
if (dq && dq->inputs.size() >= 3 &&
areDqArgsConst(dq) &&
relu_in_int8 && relu_out_int8 &&
usecounts.at(relu_in.idx) == 1) {
const float inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
@@ -256,25 +269,23 @@ struct ModelFusionQDQ
}
}
LayerParams reluInt8Params = makeLayerParamsFromOriginal(relu, "ReLUInt8");
reluInt8Params.blobs.clear();
Ptr<Layer> reluInt8 = createFusedLayer(reluInt8Params);
if (!reluInt8.empty()) {
auto* reluInt8Layer = dynamic_cast<ActivationLayerInt8*>(reluInt8.get());
CV_Assert(reluInt8Layer);
reluInt8Layer->input_sc = inp_sc;
reluInt8Layer->input_zp = inp_zp;
reluInt8Layer->output_sc = out_sc;
reluInt8Layer->output_zp = out_zp_i;
reluInt8Layer->activationLUT = lookUpTable;
fused_layer_idx = relu_layer_idx;
newprog[relu_layer_idx] = reluInt8;
fused_inputs.assign(1, dq->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(relu_in);
newprog[dq_idx] = Ptr<Layer>();
break;
}
ActivationInt8Params ap;
ap.name = relu->name;
ap.activationType = "ReLUInt8";
ap.input_sc = inp_sc;
ap.input_zp = inp_zp;
ap.output_sc = out_sc;
ap.output_zp = out_zp_i;
ap.activationLUT = lookUpTable;
Ptr<ActivationLayerInt8> reluInt8 = ActivationLayerInt8::create(ap);
reluInt8->netimpl = netimpl;
fused_layer_idx = relu_layer_idx;
newprog[relu_layer_idx] = reluInt8;
fused_inputs.assign(1, dq->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(relu_in);
newprog[dq_idx] = Ptr<Layer>();
break;
}
}
}
@@ -299,8 +310,7 @@ struct ModelFusionQDQ
const Arg& add_inp = add2->inputs[k];
int dq_idx2 = producer_of.at(add_inp.idx);
DequantizeLinearLayer* dq2 = getLayer<DequantizeLinearLayer>(newprog, dq_idx2);
if (dq2 && dq2->inputs.size() >= 3) {
if (dq2 && dq2->inputs.size() >= 3 && areDqArgsConst(dq2)) {
dq_ptrs2.push_back(dq2);
dq_prog_indices2.push_back(dq_idx2);
int8_inputs2.push_back(dq2->inputs[0]);
@@ -326,7 +336,7 @@ struct ModelFusionQDQ
}
int elt_out_type2 = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
if (elt_out_type2 < 0) {
if (elt_out_type2 < 0 && netimpl->isConstArg(inputs[2])) {
const Mat& zp_tensor = netimpl->argTensor(inputs[2]);
elt_out_type2 = !zp_tensor.empty() ? zp_tensor.type() : CV_8S;
}
@@ -373,41 +383,38 @@ struct ModelFusionQDQ
: (int)out_zp_m2.at<int8_t>(0);
if (out_sc2 > 0.f) {
int relu_out_uc = usecounts.at(q_inp.idx);
LayerParams eltParams = makeLayerParamsFromOriginal(add2, "Eltwise2Int8");
eltParams.blobs.clear();
eltParams.set("input_scales", DictValue::arrayReal(in_scales2.data(), (int)in_scales2.size()));
eltParams.set("input_zeropoints", DictValue::arrayInt(in_zps2.data(), (int)in_zps2.size()));
eltParams.set("scales", out_sc2);
eltParams.set("zeropoints", out_zp_val2);
eltParams.set("with_relu", true);
Ptr<Layer> eltInt8 = createFusedLayer(eltParams);
if (!eltInt8.empty()) {
CV_Assert(dynamic_cast<Eltwise2Int8Layer*>(eltInt8.get()));
Eltwise2Int8Params ep2;
ep2.name = add2->name;
ep2.input_scales = in_scales2;
ep2.input_zeropoints = in_zps2;
ep2.output_sc = out_sc2;
ep2.output_zp = out_zp_val2;
ep2.with_relu = true;
Ptr<Eltwise2Int8Layer> eltInt8 = Eltwise2Int8Layer::create(ep2);
eltInt8->netimpl = netimpl;
fused_inputs = int8_inputs2;
if (relu_out_uc <= 1) {
fused_layer_idx = add_idx2;
newprog[add_idx2] = eltInt8;
newprog[relu_layer_idx2] = Ptr<Layer>();
removed_args.push_back(q_inp); // relu_out
removed_args.push_back(relu_in2); // add_out
for (size_t dk = 0; dk < add2->inputs.size(); dk++) {
removed_args.push_back(add2->inputs[dk]);
}
for (int dq_prog_idx : dq_prog_indices2) {
if (dq_prog_idx >= 0)
newprog[dq_prog_idx] = Ptr<Layer>();
}
} else {
int new_idx = (int)newprog.size();
newprog.push_back(eltInt8);
fused_layer_idx = new_idx;
usecounts.at(q_inp.idx) -= 1;
Eltwise2Int8Layer* elt2ptr = dynamic_cast<Eltwise2Int8Layer*>(eltInt8.get());
relu_to_eltwise[q_inp.idx] = {outputs[0], elt2ptr};
fused_inputs = int8_inputs2;
if (relu_out_uc <= 1) {
fused_layer_idx = add_idx2;
newprog[add_idx2] = eltInt8;
newprog[relu_layer_idx2] = Ptr<Layer>();
removed_args.push_back(q_inp); // relu_out
removed_args.push_back(relu_in2); // add_out
for (size_t dk = 0; dk < add2->inputs.size(); dk++) {
removed_args.push_back(add2->inputs[dk]);
}
break;
for (int dq_prog_idx : dq_prog_indices2) {
if (dq_prog_idx >= 0)
newprog[dq_prog_idx] = Ptr<Layer>();
}
} else {
int new_idx = (int)newprog.size();
newprog.push_back(eltInt8);
fused_layer_idx = new_idx;
usecounts.at(q_inp.idx) -= 1;
relu_to_eltwise[q_inp.idx] = {outputs[0], eltInt8.get()};
}
break;
}
}
}
@@ -431,6 +438,8 @@ struct ModelFusionQDQ
if (dq_x && dq_w &&
dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 &&
areDqArgsConst(dq_x) && areDqArgsConst(dq_w) &&
netimpl->isConstArg(dq_w->inputs[0]) &&
usecounts.at(conv_w.idx) == 1) {
float inp_sc = netimpl->argTensor(dq_x->inputs[1]).at<float>(0);
float out_sc = netimpl->argTensor(out_scale_arg).at<float>(0);
@@ -533,48 +542,41 @@ struct ModelFusionQDQ
outputMultiplier.at<float>(oc) = (inp_sc * wt_sc.at<float>(oc)) / out_sc;
}
LayerParams convInt8Params = makeLayerParamsFromOriginal(conv, "Conv2Int8");
{
if (!conv->strides.empty())
convInt8Params.set("stride", DictValue::arrayInt(conv->strides.data(), (int)conv->strides.size()));
if (!conv->dilations.empty())
convInt8Params.set("dilation", DictValue::arrayInt(conv->dilations.data(), (int)conv->dilations.size()));
if (!conv->pads.empty())
convInt8Params.set("pad", DictValue::arrayInt(conv->pads.data(), (int)conv->pads.size()));
Conv2Int8Params cp;
cp.name = conv->name;
cp.strides = conv->strides;
cp.dilations = conv->dilations;
cp.pads = conv->pads;
cp.ngroups = conv->ngroups;
cp.auto_pad = conv->auto_pad;
cp.ceil_mode = conv->ceil_mode;
cp.input_sc = inp_sc;
cp.input_zp = inp_zp;
cp.output_sc = out_sc;
cp.output_zp = out_zp;
cp.per_channel = per_channel;
cp.input_is_u8 = inputIsU8;
cp.weights = w_q;
cp.bias = biasFused;
cp.outputMultiplier = outputMultiplier;
Ptr<Conv2Int8Layer> convInt8 = Conv2Int8Layer::create(cp);
convInt8->netimpl = netimpl;
fused_layer_idx = conv_layer_idx;
newprog[conv_layer_idx] = convInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(conv_w);
if (conv->inputs.size() == 3) {
removed_args.push_back(conv->inputs[2]);
if (dq_bias_idx >= 0)
newprog[dq_bias_idx] = Ptr<Layer>();
}
convInt8Params.set("num_output", outCn);
convInt8Params.set("group", conv->ngroups);
convInt8Params.set("input_scale", inp_sc);
convInt8Params.set("input_zeropoint", inp_zp);
convInt8Params.set("scales", out_sc);
convInt8Params.set("zeropoints", out_zp);
convInt8Params.set("per_channel", per_channel);
convInt8Params.set("input_is_u8", inputIsU8);
convInt8Params.blobs.resize(3);
convInt8Params.blobs[0] = w_q;
convInt8Params.blobs[1] = biasFused;
convInt8Params.blobs[2] = outputMultiplier;
Ptr<Layer> convInt8 = createFusedLayer(convInt8Params);
if (!convInt8.empty()) {
auto* convInt8Layer = dynamic_cast<Conv2Int8Layer*>(convInt8.get());
CV_Assert(convInt8Layer);
fused_layer_idx = conv_layer_idx;
newprog[conv_layer_idx] = convInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(conv_w);
if (conv->inputs.size() == 3) {
removed_args.push_back(conv->inputs[2]);
if (dq_bias_idx >= 0)
newprog[dq_bias_idx] = Ptr<Layer>();
}
if (usecounts.at(conv_x.idx) == 1) {
removed_args.push_back(conv_x);
newprog[dq_x_idx] = Ptr<Layer>();
}
newprog[dq_w_idx] = Ptr<Layer>();
break;
if (usecounts.at(conv_x.idx) == 1) {
removed_args.push_back(conv_x);
newprog[dq_x_idx] = Ptr<Layer>();
}
newprog[dq_w_idx] = Ptr<Layer>();
break;
}
}
}
@@ -595,19 +597,22 @@ struct ModelFusionQDQ
float inp_sc = 0.f, out_sc = 0.f;
int inp_zp = 0, out_zp_i = 0;
int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
if (fc_out_type < 0) {
if (fc_out_type < 0 && netimpl->isConstArg(inputs[2])) {
const Mat& zp_t = netimpl->argTensor(inputs[2]);
fc_out_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U);
int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1;
if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3) {
if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3 &&
netimpl->isConstArg(dq_x->inputs[2])) {
const Mat& zp_t = netimpl->argTensor(dq_x->inputs[2]);
fc_in_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U);
if (dq_x && dq_w &&
dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 &&
areDqArgsConst(dq_x) && areDqArgsConst(dq_w) &&
netimpl->isConstArg(dq_w->inputs[0]) &&
fc_in_int8 && fc_out_int8 &&
usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) {
inp_sc = netimpl->argTensor(dq_x->inputs[1]).at<float>(0);
@@ -619,7 +624,7 @@ struct ModelFusionQDQ
out_sc = netimpl->argTensor(out_scale).at<float>(0);
const Mat& fc_out_zp_m = netimpl->argTensor(out_zp);
out_zp_i = fc_out_zp_m.depth() == CV_8U
? (int)fc_out_zp_m.at<uint8_t>(0)
? (int)fc_out_zp_m.at<uint8_t>(0) - 128
: (int)fc_out_zp_m.at<int8_t>(0);
if (!(inp_sc > 0.f && out_sc > 0.f))
break;
@@ -647,35 +652,31 @@ struct ModelFusionQDQ
outputMultiplier.at<float>(ioc) = (inp_sc * wt_sc.at<float>(ioc)) / out_sc;
}
int firstInpDims = (int)netimpl->argData(mm_x).shape.size();
int axis = std::max(1, firstInpDims - w_q.dims + 1);
LayerParams fcInt8Params = makeLayerParamsFromOriginal(mm, "InnerProductInt8");
fcInt8Params.set("num_output", outCn);
fcInt8Params.set("axis", axis);
fcInt8Params.blobs.resize(3);
fcInt8Params.blobs[0] = weights;
fcInt8Params.blobs[1] = bias;
fcInt8Params.blobs[2] = outputMultiplier;
Ptr<Layer> fcInt8 = createFusedLayer(fcInt8Params);
if (!fcInt8.empty()) {
auto* fcInt8Layer = dynamic_cast<InnerProductLayerInt8*>(fcInt8.get());
CV_Assert(fcInt8Layer);
fcInt8Layer->input_zp = inp_zp;
fcInt8Layer->input_sc = inp_sc;
fcInt8Layer->output_zp = out_zp_i;
fcInt8Layer->output_sc = out_sc;
fcInt8Layer->output_type = fc_out_type;
fcInt8Layer->per_channel = per_channel;
fused_layer_idx = mm_layer_idx;
newprog[mm_layer_idx] = fcInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(mm_x);
removed_args.push_back(mm_w);
newprog[dq_x_idx] = Ptr<Layer>();
newprog[dq_w_idx] = Ptr<Layer>();
break;
}
MatMulInt8Params fp;
fp.name = mm->name;
fp.num_output = outCn;
fp.inp_dims = firstInpDims;
fp.input_sc = inp_sc;
fp.input_zp = inp_zp;
fp.output_sc = out_sc;
fp.output_zp = out_zp_i;
fp.output_type = fc_out_type;
fp.per_channel = per_channel;
fp.weights = weights;
fp.bias = bias;
fp.outputMultiplier = outputMultiplier;
Ptr<MatMulInt8Layer> mmInt8 = MatMulInt8Layer::create(fp);
mmInt8->netimpl = netimpl;
fused_layer_idx = mm_layer_idx;
newprog[mm_layer_idx] = mmInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(mm_x);
removed_args.push_back(mm_w);
newprog[dq_x_idx] = Ptr<Layer>();
newprog[dq_w_idx] = Ptr<Layer>();
break;
}
}
}
@@ -711,19 +712,22 @@ struct ModelFusionQDQ
DequantizeLinearLayer* dq_x = getLayer<DequantizeLinearLayer>(newprog, dq_x_idx);
DequantizeLinearLayer* dq_w = getLayer<DequantizeLinearLayer>(newprog, dq_w_idx);
int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
if (fc_out_type < 0) {
if (fc_out_type < 0 && netimpl->isConstArg(inputs[2])) {
const Mat& zp_t = netimpl->argTensor(inputs[2]);
fc_out_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U);
int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1;
if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3) {
if (fc_in_type < 0 && dq_x && dq_x->inputs.size() >= 3 &&
netimpl->isConstArg(dq_x->inputs[2])) {
const Mat& zp_t = netimpl->argTensor(dq_x->inputs[2]);
fc_in_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U);
if (dq_x && dq_w &&
dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 &&
areDqArgsConst(dq_x) && areDqArgsConst(dq_w) &&
netimpl->isConstArg(dq_w->inputs[0]) &&
fc_in_int8 && fc_out_int8 &&
usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) {
float inp_sc = netimpl->argTensor(dq_x->inputs[1]).at<float>(0);
@@ -734,7 +738,7 @@ struct ModelFusionQDQ
float out_sc_val = netimpl->argTensor(out_scale).at<float>(0);
const Mat& fc_out_zp_m = netimpl->argTensor(out_zp);
int out_zp_i = fc_out_zp_m.depth() == CV_8U
? (int)fc_out_zp_m.at<uint8_t>(0)
? (int)fc_out_zp_m.at<uint8_t>(0) - 128
: (int)fc_out_zp_m.at<int8_t>(0);
if (inp_sc > 0.f && out_sc_val > 0.f) {
Mat w_q = netimpl->argTensor(dq_w->inputs[0]);
@@ -773,37 +777,162 @@ struct ModelFusionQDQ
}
if (biasOk) {
int firstInpDims = (int)netimpl->argData(mm_x).shape.size();
int fc_axis = std::max(1, firstInpDims - w_q.dims + 1);
LayerParams fcInt8Params = makeLayerParamsFromOriginal(mm2, "InnerProductInt8");
fcInt8Params.set("num_output", outCn);
fcInt8Params.set("axis", fc_axis);
fcInt8Params.blobs.resize(3);
fcInt8Params.blobs[0] = weights;
fcInt8Params.blobs[1] = bias;
fcInt8Params.blobs[2] = outputMultiplier;
Ptr<Layer> fcInt8 = createFusedLayer(fcInt8Params);
if (!fcInt8.empty()) {
auto* fcInt8Layer = dynamic_cast<InnerProductLayerInt8*>(fcInt8.get());
CV_Assert(fcInt8Layer);
fcInt8Layer->input_zp = inp_zp;
fcInt8Layer->input_sc = inp_sc;
fcInt8Layer->output_zp = out_zp_i;
fcInt8Layer->output_sc = out_sc_val;
fcInt8Layer->output_type = fc_out_type;
fcInt8Layer->per_channel = per_channel;
fused_layer_idx = add_bias_idx;
newprog[add_bias_idx] = fcInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(add_bias->inputs[mm_inp_k]); // matmul out
removed_args.push_back(mm_x);
removed_args.push_back(mm_w);
newprog[mm2_idx] = Ptr<Layer>();
newprog[dq_x_idx] = Ptr<Layer>();
newprog[dq_w_idx] = Ptr<Layer>();
break;
MatMulInt8Params fp2;
fp2.name = mm2->name;
fp2.num_output = outCn;
fp2.inp_dims = firstInpDims;
fp2.input_sc = inp_sc;
fp2.input_zp = inp_zp;
fp2.output_sc = out_sc_val;
fp2.output_zp = out_zp_i;
fp2.output_type = fc_out_type;
fp2.per_channel = per_channel;
fp2.weights = weights;
fp2.bias = bias;
fp2.outputMultiplier = outputMultiplier;
Ptr<MatMulInt8Layer> mmInt8 = MatMulInt8Layer::create(fp2);
mmInt8->netimpl = netimpl;
fused_layer_idx = add_bias_idx;
newprog[add_bias_idx] = mmInt8;
fused_inputs.assign(1, dq_x->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(add_bias->inputs[mm_inp_k]); // matmul out
removed_args.push_back(mm_x);
removed_args.push_back(mm_w);
newprog[mm2_idx] = Ptr<Layer>();
newprog[dq_x_idx] = Ptr<Layer>();
newprog[dq_w_idx] = Ptr<Layer>();
break;
}
}
}
}
}
}
}
// fuse DequantizeLinear -> Gemm -> QuantizeLinear into MatMulInt8
// Gemm: Y = alpha * A @ B + beta * C (with optional transA/transB)
{
int gemm_layer_idx = -1;
GemmLayer* gemm = 0;
if (getQdqPatternContext<GemmLayer>(layer_ptr, ninputs, inputs, producer_of,
newprog, q_data_in, out_scale, out_zp,
gemm_layer_idx, gemm) &&
gemm->inputs.size() >= 2 && !gemm->trans_a) {
// Gemm inputs: A, B, [C]
const Arg gemm_a = gemm->inputs[0];
const Arg gemm_b = gemm->inputs[1];
int dq_a_idx = producer_of.at(gemm_a.idx);
int dq_b_idx = producer_of.at(gemm_b.idx);
DequantizeLinearLayer* dq_a = getLayer<DequantizeLinearLayer>(newprog, dq_a_idx);
DequantizeLinearLayer* dq_b = getLayer<DequantizeLinearLayer>(newprog, dq_b_idx);
int g_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
if (g_out_type < 0 && netimpl->isConstArg(inputs[2])) {
const Mat& zp_t = netimpl->argTensor(inputs[2]);
g_out_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool g_out_int8 = (g_out_type == CV_8S || g_out_type == CV_8U);
int g_in_type = (dq_a && !dq_a->inputs.empty()) ? netimpl->argData(dq_a->inputs[0]).type : -1;
if (g_in_type < 0 && dq_a && dq_a->inputs.size() >= 3 &&
netimpl->isConstArg(dq_a->inputs[2])) {
const Mat& zp_t = netimpl->argTensor(dq_a->inputs[2]);
g_in_type = !zp_t.empty() ? zp_t.type() : CV_8S;
}
const bool g_in_int8 = (g_in_type == CV_8S || g_in_type == CV_8U);
if (dq_a && dq_b &&
dq_a->inputs.size() >= 3 && dq_b->inputs.size() >= 3 &&
areDqArgsConst(dq_a) && areDqArgsConst(dq_b) &&
netimpl->isConstArg(dq_b->inputs[0]) &&
g_in_int8 && g_out_int8 &&
usecounts.at(gemm_a.idx) == 1 && usecounts.at(gemm_b.idx) == 1) {
float inp_sc = netimpl->argTensor(dq_a->inputs[1]).at<float>(0);
const Mat& g_zp_m = netimpl->argTensor(dq_a->inputs[2]);
int inp_zp = g_zp_m.depth() == CV_8U
? (int)g_zp_m.at<uint8_t>(0) - 128
: (int)g_zp_m.at<int8_t>(0);
float out_sc_val = netimpl->argTensor(out_scale).at<float>(0);
const Mat& g_out_zp_m = netimpl->argTensor(out_zp);
int out_zp_i = g_out_zp_m.depth() == CV_8U
? (int)g_out_zp_m.at<uint8_t>(0) - 128
: (int)g_out_zp_m.at<int8_t>(0);
if (inp_sc > 0.f && out_sc_val > 0.f) {
Mat w_q = netimpl->argTensor(dq_b->inputs[0]);
Mat w_sc_m = netimpl->argTensor(dq_b->inputs[1]);
Mat w_zp_m = netimpl->argTensor(dq_b->inputs[2]);
if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims == 2) {
bool all_wzp_zero = true;
for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) {
int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at<int8_t>((int)t)
: (int)w_zp_m.at<uint8_t>((int)t);
if (wz != 0) all_wzp_zero = false;
}
if (all_wzp_zero) {
// Apply Gemm's transB: if transB, weights are already [N, K],
// otherwise [K, N] and we need to transpose.
Mat weights = gemm->trans_b ? w_q : w_q.t();
int outCn = weights.size[0];
float alpha_val = gemm->alpha;
Mat wt_sc = (w_sc_m.total() == (size_t)outCn)
? w_sc_m.reshape(1, 1)
: Mat(1, outCn, CV_32F, Scalar(w_sc_m.at<float>(0))).clone();
bool per_channel = w_sc_m.total() == (size_t)outCn;
// Fuse optional bias (C input of Gemm)
Mat float_bias;
if (gemm->inputs.size() >= 3)
float_bias = netimpl->argTensor(gemm->inputs[2]);
Mat bias(1, outCn, CV_32S);
Mat outputMult(1, outCn, CV_32F);
bool biasOk = true;
for (int ioc = 0; ioc < outCn && biasOk; ioc++) {
float denom = alpha_val * inp_sc * wt_sc.at<float>(ioc);
if (std::abs(denom) < 1e-12f) { biasOk = false; break; }
int zp_correction = -inp_zp * (int)cv::sum(weights.row(ioc))[0];
if (!float_bias.empty() && float_bias.total() == (size_t)outCn) {
float b_val = 0.f;
if (float_bias.depth() == CV_32F)
b_val = float_bias.at<float>(ioc);
else if (float_bias.depth() == CV_64F)
b_val = (float)float_bias.at<double>(ioc);
bias.at<int>(ioc) = cvRound(gemm->beta * b_val / denom) + zp_correction;
} else {
bias.at<int>(ioc) = zp_correction;
}
outputMult.at<float>(ioc) = denom / out_sc_val;
}
if (biasOk) {
int firstInpDims = (int)netimpl->argData(gemm_a).shape.size();
MatMulInt8Params gp;
gp.name = gemm->name;
gp.num_output = outCn;
gp.inp_dims = firstInpDims;
gp.input_sc = inp_sc;
gp.input_zp = inp_zp;
gp.output_sc = out_sc_val;
gp.output_zp = out_zp_i;
gp.output_type = g_out_type;
gp.per_channel = per_channel;
gp.weights = weights;
gp.bias = bias;
gp.outputMultiplier = outputMult;
Ptr<MatMulInt8Layer> gemmInt8 = MatMulInt8Layer::create(gp);
gemmInt8->netimpl = netimpl;
fused_layer_idx = gemm_layer_idx;
newprog[gemm_layer_idx] = gemmInt8;
fused_inputs.assign(1, dq_a->inputs[0]);
removed_args.push_back(q_data_in);
removed_args.push_back(gemm_a);
removed_args.push_back(gemm_b);
newprog[dq_a_idx] = Ptr<Layer>();
newprog[dq_b_idx] = Ptr<Layer>();
break;
}
}
}
@@ -816,44 +945,57 @@ struct ModelFusionQDQ
PoolingLayer* pool = 0;
if (getQdqPatternContext<PoolingLayer>(layer_ptr, ninputs, inputs, producer_of,
newprog, q_data_in, out_scale, out_zp,
pool_layer_idx, pool) && pool->inputs.size() == 1) {
Arg pool_in = pool->inputs[0];
int dq_idx = producer_of.at(pool_in.idx);
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
float inp_sc = 0.f, out_sc = 0.f;
int inp_zp = 0, out_zp_i = 0;
const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U);
const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U);
if (dq && dq->inputs.size() >= 3 && pool_in_int8 && pool_out_int8 && usecounts.at(pool_in.idx) == 1) {
inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]);
inp_zp = pool_zp_m.depth() == CV_8U
? (int)pool_zp_m.at<uint8_t>(0)
: (int)pool_zp_m.at<int8_t>(0);
out_sc = netimpl->argTensor(out_scale).at<float>(0);
const Mat& pool_out_zp_m = netimpl->argTensor(out_zp);
out_zp_i = pool_out_zp_m.depth() == CV_8U
? (int)pool_out_zp_m.at<uint8_t>(0)
: (int)pool_out_zp_m.at<int8_t>(0);
bool isGlobalAve = pool->globalPooling;
bool isMax = !isGlobalAve;
if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) ||
(isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) {
LayerParams poolInt8Params = makeLayerParamsFromOriginal(pool, "Pool2Int8");
poolInt8Params.blobs.clear();
poolInt8Params.set("input_scale", inp_sc);
poolInt8Params.set("input_zeropoint", inp_zp);
poolInt8Params.set("scales", out_sc);
poolInt8Params.set("zeropoints", out_zp_i);
poolInt8Params.set("is_max_pool", isMax);
poolInt8Params.set("global_pooling", isGlobalAve);
Ptr<Layer> poolInt8 = createFusedLayer(poolInt8Params);
if (!poolInt8.empty()) {
auto* poolInt8Layer = dynamic_cast<Pool2Int8Layer*>(poolInt8.get());
CV_Assert(poolInt8Layer);
pool_layer_idx, pool) &&
pool->inputs.size() == 1) {
Arg pool_in = pool->inputs[0];
int dq_idx = producer_of.at(pool_in.idx);
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
float inp_sc = 0.f, out_sc = 0.f;
int inp_zp = 0, out_zp_i = 0;
const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U);
const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U);
if (dq && dq->inputs.size() >= 3 &&
areDqArgsConst(dq) &&
pool_in_int8 && pool_out_int8 &&
usecounts.at(pool_in.idx) == 1) {
inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]);
inp_zp = pool_zp_m.depth() == CV_8U
? (int)pool_zp_m.at<uint8_t>(0)
: (int)pool_zp_m.at<int8_t>(0);
out_sc = netimpl->argTensor(out_scale).at<float>(0);
const Mat& pool_out_zp_m = netimpl->argTensor(out_zp);
out_zp_i = pool_out_zp_m.depth() == CV_8U
? (int)pool_out_zp_m.at<uint8_t>(0)
: (int)pool_out_zp_m.at<int8_t>(0);
bool isGlobalAve = pool->globalPooling;
bool isMax = !isGlobalAve;
if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) ||
(isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) {
Pool2Int8Params p;
p.name = pool->name;
p.kernel_shape.assign(pool->kernel_size.begin(), pool->kernel_size.end());
p.strides.assign(pool->strides.begin(), pool->strides.end());
if (!pool->pads_begin.empty()) {
p.pads.reserve(pool->pads_begin.size() + pool->pads_end.size());
for (size_t v : pool->pads_begin) p.pads.push_back((int)v);
for (size_t v : pool->pads_end) p.pads.push_back((int)v);
}
if (pool->padMode == "SAME")
p.auto_pad = AUTO_PAD_SAME_UPPER;
else if (pool->padMode == "VALID")
p.auto_pad = AUTO_PAD_VALID;
p.ceil_mode = pool->ceilMode;
p.is_global_pooling = isGlobalAve;
p.is_max_pool = isMax;
p.input_sc = inp_sc;
p.input_zp = inp_zp;
p.output_sc = out_sc;
p.output_zp = out_zp_i;
Ptr<Pool2Int8Layer> poolInt8 = Pool2Int8Layer::create(p);
poolInt8->netimpl = netimpl;
fused_layer_idx = pool_layer_idx;
newprog[pool_layer_idx] = poolInt8;
fused_inputs.assign(1, dq->inputs[0]);
@@ -864,6 +1006,59 @@ struct ModelFusionQDQ
}
}
}
// fuse DequantizeLinear -> DataShuffling/MaxPool -> QuantizeLinear
// when DQ and Q have same scale/zp, the op can work directly on int8 data.
// Uses Layer::isDataShuffling() to identify eligible layers.
{
QuantizeLinearLayer* ql_shuffle = dynamic_cast<QuantizeLinearLayer*>(layer_ptr);
if (ql_shuffle && ninputs == 3 && usecounts.at(inputs[0].idx) == 1) {
Arg ql_data = inputs[0];
Arg ql_scale = inputs[1];
Arg ql_zp = inputs[2];
int shuffle_idx = producer_of.at(ql_data.idx);
Layer* shuffle_layer = (shuffle_idx >= 0 && !newprog[shuffle_idx].empty())
? newprog[shuffle_idx].get() : nullptr;
bool is_shuffle = shuffle_layer && shuffle_layer->isDataShuffling();
if (is_shuffle && shuffle_layer->inputs.size() >= 1 &&
usecounts.at(ql_data.idx) == 1) {
Arg shuffle_inp = shuffle_layer->inputs[0];
int dq_idx = producer_of.at(shuffle_inp.idx);
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
if (dq && dq->inputs.size() >= 3 &&
usecounts.at(shuffle_inp.idx) == 1) {
float dq_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
float q_sc = netimpl->argTensor(ql_scale).at<float>(0);
const Mat& dq_zp_m = netimpl->argTensor(dq->inputs[2]);
const Mat& q_zp_m = netimpl->argTensor(ql_zp);
int dq_zp = dq_zp_m.depth() == CV_8U
? (int)dq_zp_m.at<uint8_t>(0) : (int)dq_zp_m.at<int8_t>(0);
int q_zp = q_zp_m.depth() == CV_8U
? (int)q_zp_m.at<uint8_t>(0) : (int)q_zp_m.at<int8_t>(0);
int in_type = netimpl->argData(dq->inputs[0]).type;
int out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
bool in_int8 = (in_type == CV_8S || in_type == CV_8U);
bool out_int8 = (out_type == CV_8S || out_type == CV_8U);
if (in_int8 && out_int8 &&
std::abs(dq_sc - q_sc) < 1e-6f && dq_zp == q_zp) {
fused_layer_idx = shuffle_idx;
fused_inputs.assign(1, dq->inputs[0]);
for (size_t si = 1; si < shuffle_layer->inputs.size(); si++)
fused_inputs.push_back(shuffle_layer->inputs[si]);
removed_args.push_back(ql_data);
removed_args.push_back(shuffle_inp);
newprog[dq_idx] = Ptr<Layer>();
break;
}
}
}
}
}
ActivationLayerInt8* activ_int8 = dynamic_cast<ActivationLayerInt8*>(layer_ptr);
@@ -913,6 +1108,69 @@ struct ModelFusionQDQ
}
}
for (size_t j = 0; j < newprog.size(); j++)
{
if (newprog[j].empty()) continue;
MaxPoolLayer* maxpool = dynamic_cast<MaxPoolLayer*>(newprog[j].get());
if (!maxpool || maxpool->inputs.size() != 1)
continue;
Arg mp_out = maxpool->outputs[0];
bool consumer_is_ql = false;
for (size_t j2 = j + 1; j2 < newprog.size(); j2++) {
if (newprog[j2].empty()) continue;
bool is_consumer = false;
for (const Arg& a : newprog[j2]->inputs)
if (a.idx == mp_out.idx) { is_consumer = true; break; }
if (is_consumer) {
consumer_is_ql = dynamic_cast<QuantizeLinearLayer*>(newprog[j2].get()) != nullptr;
break;
}
}
if (consumer_is_ql)
continue;
int cur = producer_of.at(maxpool->inputs[0].idx);
QuantizeLinearLayer* ql = nullptr;
while (cur >= 0 && !newprog[cur].empty()) {
ql = dynamic_cast<QuantizeLinearLayer*>(newprog[cur].get());
if (ql) break;
Layer* l = newprog[cur].get();
if (l->inputs.empty()) { ql = nullptr; break; }
cur = producer_of.at(l->inputs[0].idx);
}
if (!ql || ql->inputs.size() < 2 ||
!netimpl->isConstArg(ql->inputs[1]) ||
(ql->inputs.size() >= 3 && !netimpl->isConstArg(ql->inputs[2])))
continue;
float inp_sc = netimpl->argTensor(ql->inputs[1]).at<float>(0);
const Mat& inp_zp_m = ql->inputs.size() >= 3 ? netimpl->argTensor(ql->inputs[2]) : Mat();
int inp_zp = inp_zp_m.empty() ? 0 :
inp_zp_m.depth() == CV_8U ? (int)inp_zp_m.at<uint8_t>(0) :
(int)inp_zp_m.at<int8_t>(0);
Pool2Int8Params p;
p.name = maxpool->name;
p.kernel_shape = maxpool->kernel_shape;
p.strides = maxpool->strides;
p.dilations = maxpool->dilations;
p.pads = maxpool->pads;
p.auto_pad = maxpool->auto_pad;
p.ceil_mode = maxpool->ceil_mode;
p.is_max_pool = true;
p.input_sc = inp_sc;
p.input_zp = inp_zp;
p.output_sc = inp_sc;
p.output_zp = inp_zp;
Ptr<Pool2Int8Layer> poolInt8 = Pool2Int8Layer::create(p);
poolInt8->netimpl = netimpl;
poolInt8->inputs = maxpool->inputs;
poolInt8->outputs = maxpool->outputs;
newprog[j] = poolInt8;
modified = true;
}
if (modified) {
std::vector<int> uc(nargs, 0);
for (const auto& layer : newprog) {
modules/dnn/src/int8layers/conv2_int8_layer.cpp
+35
View File
@@ -212,6 +212,32 @@ public:
}
}
Conv2Int8LayerImpl(const Conv2Int8Params& p)
{
name = p.name;
type = "Conv2Int8";
strides = p.strides;
dilations = p.dilations;
pads = p.pads;
ngroups = p.ngroups;
auto_pad = p.auto_pad;
ceil_mode = p.ceil_mode;
input_sc = p.input_sc;
input_zp = p.input_zp;
output_sc = p.output_sc;
output_zp = p.output_zp;
per_channel = p.per_channel;
inputIsU8 = p.input_is_u8;
addResidual = false;
if (!p.weights.empty()) {
wshape0 = p.weights.shape();
biasInt32 = p.bias;
outMultiplier = p.outputMultiplier;
blobs = { p.weights, p.bias, p.outputMultiplier };
}
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs, const int,
std::vector<MatType>& outputs,
@@ -341,6 +367,10 @@ public:
cs.initConv(inpshape, wshape0, outshape, ngroups,
strides, dilations, pads, auto_pad, ceil_mode,
FAST_ACTIV_NONE, nullptr, {});
if (cs.depthwise) {
cs.wshape = getWpackShapeInt8(wshape0, ngroups, C0);
}
prevInpshape = inpshape;
}
@@ -367,5 +397,10 @@ Ptr<Conv2Int8Layer> Conv2Int8Layer::create(const LayerParams& params)
return Ptr<Conv2Int8Layer>(new Conv2Int8LayerImpl(params));
}
Ptr<Conv2Int8Layer> Conv2Int8Layer::create(const Conv2Int8Params& params)
{
return Ptr<Conv2Int8Layer>(new Conv2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/int8layers/elementwise_layers.cpp
+16 -1
View File
@@ -37,11 +37,21 @@ public:
{
slope = params.get<float>("slope");
}
}
ActivationLayerInt8Impl(const ActivationInt8Params &p)
{
name = p.name;
type = p.activationType;
input_sc = p.input_sc;
input_zp = p.input_zp;
output_sc = p.output_sc;
output_zp = p.output_zp;
activationLUT = p.activationLUT;
#ifdef HAVE_TIMVX
tvActType = getTimVXActType(type);
#endif
}
virtual bool supportBackend(int backendId) CV_OVERRIDE
@@ -360,5 +370,10 @@ Ptr<ActivationLayerInt8> ActivationLayerInt8::create(const LayerParams& params)
return Ptr<ActivationLayerInt8>(new ActivationLayerInt8Impl(params));
}
Ptr<ActivationLayerInt8> ActivationLayerInt8::create(const ActivationInt8Params& params)
{
return Ptr<ActivationLayerInt8>(new ActivationLayerInt8Impl(params));
}
}
}
modules/dnn/src/int8layers/eltwise2_int8_layer.cpp
+17
View File
@@ -57,6 +57,18 @@ public:
offset = 0.f;
}
Eltwise2Int8LayerImpl(const Eltwise2Int8Params& p)
{
name = p.name;
type = "Eltwise2Int8";
scales = p.input_scales;
zeropoints = p.input_zeropoints;
output_sc = p.output_sc;
output_zp = p.output_zp;
withRelu = p.with_relu;
offset = 0.f;
}
void ensureCoeffs()
{
if (!coeffs.empty() || scales.empty())
@@ -409,5 +421,10 @@ Ptr<Eltwise2Int8Layer> Eltwise2Int8Layer::create(const LayerParams& params)
return Ptr<Eltwise2Int8Layer>(new Eltwise2Int8LayerImpl(params));
}
Ptr<Eltwise2Int8Layer> Eltwise2Int8Layer::create(const Eltwise2Int8Params& params)
{
return Ptr<Eltwise2Int8Layer>(new Eltwise2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/int8layers/fully_connected_layer.cpp
+41
View File
@@ -57,6 +57,42 @@ public:
}
}
FullyConnectedLayerInt8Impl(const InnerProductInt8Params& p)
{
name = p.name;
type = "InnerProductInt8";
input_sc = p.input_sc;
input_zp = p.input_zp;
output_sc = p.output_sc;
output_zp = p.output_zp;
axis = p.axis;
per_channel = p.per_channel;
output_type = p.output_type;
if (!p.weights.empty()) {
int numOutput = p.num_output;
int innerSize = (int)p.weights.total() / numOutput;
CV_Assert(p.weights.dims >= 2 && (size_t)(innerSize * numOutput) == p.weights.total());
CV_Assert((size_t)numOutput == p.bias.total());
blobs = { p.weights.clone(), p.bias.clone(), p.outputMultiplier.clone() };
weightsMat = blobs[0] = blobs[0].reshape(1, numOutput);
int vecsize = weightsMat.cols;
if (vecsize % VEC_ALIGN != 0)
{
int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN);
Mat weightsBuf(weightsMat.rows, vecsize_aligned, weightsMat.type());
Mat wpadding = weightsBuf.colRange(vecsize, vecsize_aligned);
wpadding.setTo(Scalar::all(0));
weightsMat = weightsBuf.colRange(0, vecsize);
blobs[0].copyTo(weightsMat);
}
biasMat = blobs[1] = blobs[1].reshape(1, 1);
outputMultiplier = blobs[2];
}
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
@@ -508,5 +544,10 @@ Ptr<InnerProductLayerInt8> InnerProductLayerInt8::create(const LayerParams& para
return Ptr<InnerProductLayerInt8>(new FullyConnectedLayerInt8Impl(params));
}
Ptr<InnerProductLayerInt8> InnerProductLayerInt8::create(const InnerProductInt8Params& params)
{
return Ptr<InnerProductLayerInt8>(new FullyConnectedLayerInt8Impl(params));
}
}
}
modules/dnn/src/int8layers/matmul_int8_layer.cpp
+296
View File
@@ -0,0 +1,296 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
#include "../precomp.hpp"
#include "layers_common.hpp"
#include <opencv2/dnn/shape_utils.hpp>
namespace cv
{
namespace dnn
{
class MatMulInt8LayerImpl CV_FINAL : public MatMulInt8Layer
{
public:
enum { VEC_ALIGN = 32 };
MatMulInt8LayerImpl(const MatMulInt8Params& p)
{
name = p.name;
type = "MatMulInt8";
input_sc = p.input_sc;
input_zp = p.input_zp;
output_sc = p.output_sc;
output_zp = p.output_zp;
output_type = p.output_type;
per_channel = p.per_channel;
inp_dims = p.inp_dims;
num_output = p.num_output;
if (!p.weights.empty()) {
int numOutput = p.num_output;
int innerSize = (int)p.weights.total() / numOutput;
CV_Assert(p.weights.dims >= 2 && (size_t)(innerSize * numOutput) == p.weights.total());
CV_Assert((size_t)numOutput == p.bias.total());
blobs = { p.weights.clone(), p.bias.clone(), p.outputMultiplier.clone() };
weightsMat = blobs[0] = blobs[0].reshape(1, numOutput);
int vecsize = weightsMat.cols;
if (vecsize % VEC_ALIGN != 0)
{
int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN);
Mat weightsBuf(weightsMat.rows, vecsize_aligned, weightsMat.type());
Mat wpadding = weightsBuf.colRange(vecsize, vecsize_aligned);
wpadding.setTo(Scalar::all(0));
weightsMat = weightsBuf.colRange(0, vecsize);
blobs[0].copyTo(weightsMat);
}
biasMat = blobs[1] = blobs[1].reshape(1, 1);
outputMultiplier = blobs[2];
}
}
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &) const CV_OVERRIDE
{
CV_CheckEQ(inputs.size(), (size_t)1, "");
CV_CheckEQ(blobs[0].dims, 2, "");
int numOut = blobs[0].size[0];
CV_Assert((size_t)numOut == blobs[1].total());
// Preserve all batch dimensions, replace last dim with num_output.
// Input: [B1, B2, ..., M, K] → Output: [B1, B2, ..., M, num_output]
int ndims = inputs[0].dims;
CV_Assert(ndims >= 1);
MatShape outShape = inputs[0];
outShape[ndims - 1] = numOut;
outputs.resize(1, outShape);
return false;
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs,
const int requiredInternals,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
outputs.assign(requiredOutputs, output_type);
internals.clear();
}
virtual bool supportBackend(int backendId) CV_OVERRIDE
{
return backendId == DNN_BACKEND_OPENCV;
}
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr,
OutputArrayOfArrays internals_arr) CV_OVERRIDE
{
CV_TRACE_FUNCTION();
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
std::vector<Mat> input, output;
inputs_arr.getMatVector(input);
outputs_arr.getMatVector(output);
// Flatten all dimensions except the last (K) into a single batch dim.
// Input: [B1, B2, ..., M, K] → [B1*B2*...*M, K]
int ndims = input[0].dims;
int K = input[0].size[ndims - 1];
int outerSize = (int)(input[0].total() / K);
Mat srcMat0 = input[0].reshape(1, outerSize);
Mat srcMat;
if (srcMat0.type() == CV_8U) {
srcMat0.convertTo(srcMat, CV_8S, 1, -128);
} else {
srcMat = srcMat0;
}
Mat dstMat = output[0].reshape(1, outerSize);
Mat dstMatInt32 = Mat(shape(dstMat), CV_32S);
const int nstripes = outerSize <= 4 ? 1 : getNumThreads();
runInt8Gemm(srcMat, weightsMat, biasMat, outputMultiplier, dstMatInt32, nstripes, output_zp);
if (output_type == CV_8U) {
dstMatInt32.convertTo(dstMat, output_type, 1, 128);
} else {
dstMatInt32.convertTo(dstMat, output_type);
}
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const CV_OVERRIDE
{
CV_UNUSED(inputs);
long flops = 0;
int innerSize = blobs[0].size[1];
for (size_t i = 0; i < outputs.size(); i++) {
flops += total(outputs[i]) * innerSize;
}
return flops;
}
private:
// Int8 GEMM: C_int32 = A_int8 @ W_int8^T + bias_int32
// Then: output = clamp(round(C_int32 * multiplier) + output_zp)
static void runInt8Gemm(const Mat& srcMat, const Mat& weights, const Mat& biasMat,
const Mat& outputMultiplier, Mat& dstMat, int nstripes, int outZp)
{
CV_Assert(srcMat.dims == 2 && srcMat.cols == weights.cols &&
dstMat.rows == srcMat.rows && dstMat.cols == weights.rows &&
srcMat.type() == weights.type() && srcMat.type() == CV_8S &&
dstMat.type() == CV_32S && biasMat.type() == CV_32S &&
biasMat.isContinuous() && (int)biasMat.total() == dstMat.cols);
Int8GemmBody p;
p.srcMat = &srcMat;
p.weights = &weights;
p.biasMat = &biasMat;
p.outputMultiplier = &outputMultiplier;
p.dstMat = &dstMat;
p.nstripes = nstripes;
p.outZp = outZp;
p.useAVX2 = checkHardwareSupport(CPU_AVX2);
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX;
p.useLASX = checkHardwareSupport(CPU_LASX);
p.useRVV = checkHardwareSupport(CPU_RVV);
parallel_for_(Range(0, nstripes), p, nstripes);
}
class Int8GemmBody : public ParallelLoopBody
{
public:
Int8GemmBody() : srcMat(0), weights(0), biasMat(0), outputMultiplier(0),
dstMat(0), nstripes(0), outZp(0),
useAVX2(false), useAVX512(false), useLASX(false), useRVV(false) {}
void operator()(const Range& r) const CV_OVERRIDE
{
int valign = MatMulInt8LayerImpl::VEC_ALIGN;
int nsamples = srcMat->rows;
int nw0 = weights->rows;
int k, vecsize = srcMat->cols;
int vecsize_aligned = (int)alignSize(vecsize, VEC_ALIGN);
size_t total = (size_t)nsamples * nw0;
size_t stripeSize = (total + nstripes - 1) / nstripes;
size_t stripeStart = r.start * stripeSize;
size_t stripeEnd = r.end == nstripes ? total : std::min(r.end * stripeSize, total);
size_t wstep = weights->step1();
AutoBuffer<int8_t> srcbuf(vecsize_aligned + valign);
int8_t* sptr = alignPtr(srcbuf.data(), (int)(valign * sizeof(int8_t)));
for (k = vecsize; k < vecsize_aligned; k++)
sptr[k] = 0;
for (size_t ofs = stripeStart; ofs < stripeEnd; )
{
int sampleIdx = (int)(ofs / nw0);
int delta = (int)(ofs - (size_t)sampleIdx * nw0);
const int8_t* sptr_ = srcMat->ptr<int8_t>(sampleIdx);
const int8_t* wptr = weights->ptr<int8_t>(delta);
int* dptr = dstMat->ptr<int>(sampleIdx) + delta;
const int* biasptr = biasMat->ptr<int>() + delta;
const float* multptr = outputMultiplier->ptr<float>() + delta;
int nw = std::min(nw0 - delta, (int)(stripeEnd - ofs));
memcpy(sptr, sptr_, vecsize * sizeof(sptr[0]));
#if CV_TRY_AVX512_SKX
if (useAVX512)
opt_AVX512_SKX::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp);
else
#endif
#if CV_TRY_AVX2
if (useAVX2)
opt_AVX2::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp);
else
#endif
#if CV_TRY_LASX
if (useLASX)
opt_LASX::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp);
else
#endif
#if CV_TRY_RVV && CV_RVV
if (useRVV)
opt_RVV::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp);
else
#endif
#if CV_RVP052
if (1)
opt_RVP052::fastGEMM1T(sptr, wptr, wstep, biasptr, multptr, dptr, nw, vecsize, outZp);
else
#endif
{
int i = 0;
#if CV_SIMD128
for (; i <= nw - 4; i += 4, wptr += 4 * wstep)
{
v_int32x4 vs0 = v_setzero_s32(), vs1 = v_setzero_s32(),
vs2 = v_setzero_s32(), vs3 = v_setzero_s32();
v_int32x4 outzp = v_setall_s32(outZp), outmin = v_setall_s32(-128), outmax = v_setall_s32(127);
v_int32x4 s = v_load(biasptr + i);
v_float32x4 mult = v_load(multptr + i);
for (k = 0; k < vecsize; k += 16)
{
v_int8x16 v = v_load_aligned(sptr + k);
vs0 = v_dotprod_expand_fast(v, v_load_aligned(wptr + k), vs0);
vs1 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep + k), vs1);
vs2 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep * 2 + k), vs2);
vs3 = v_dotprod_expand_fast(v, v_load_aligned(wptr + wstep * 3 + k), vs3);
}
s = v_add(s, v_int32x4(v_reduce_sum(vs0), v_reduce_sum(vs1), v_reduce_sum(vs2), v_reduce_sum(vs3)));
v_int32x4 out = v_add(outzp, v_round(v_mul(v_cvt_f32(s), mult)));
v_store(dptr + i, v_min(v_max(out, outmin), outmax));
}
#endif
for (; i < nw; i++, wptr += wstep)
{
int s0 = biasptr[i];
float mult0 = multptr[i];
for (k = 0; k < vecsize; k++)
{
int8_t v = sptr[k];
s0 += (int)v * wptr[k];
}
int out0 = outZp + (int)std::round(s0 * mult0);
dptr[i] = std::min(std::max(out0, -128), 127);
}
}
ofs += nw;
}
}
const Mat *srcMat, *weights, *biasMat, *outputMultiplier;
Mat* dstMat;
int nstripes, outZp;
bool useAVX2;
bool useAVX512;
bool useLASX;
bool useRVV;
};
int inp_dims;
int num_output;
Mat weightsMat, biasMat, outputMultiplier;
};
Ptr<MatMulInt8Layer> MatMulInt8Layer::create(const MatMulInt8Params& params)
{
return Ptr<MatMulInt8Layer>(new MatMulInt8LayerImpl(params));
}
}
}
modules/dnn/src/int8layers/pool2_int8_layer.cpp
+24
View File
@@ -259,6 +259,25 @@ public:
output_zp = params.get<int>("zeropoints", 0);
}
Pool2Int8LayerImpl(const Pool2Int8Params& p)
{
name = p.name;
type = "Pool2Int8";
kernel_shape = p.kernel_shape;
strides = p.strides;
dilations = p.dilations;
pads = p.pads;
auto_pad = p.auto_pad;
ceil_mode = p.ceil_mode;
is_global_pooling = p.is_global_pooling;
is_max_pool = p.is_max_pool;
count_include_pad = p.count_include_pad;
input_sc = p.input_sc;
input_zp = p.input_zp;
output_sc = p.output_sc;
output_zp = p.output_zp;
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs, const int,
std::vector<MatType>& outputs,
@@ -377,5 +396,10 @@ Ptr<Pool2Int8Layer> Pool2Int8Layer::create(const LayerParams& params)
return Ptr<Pool2Int8Layer>(new Pool2Int8LayerImpl(params));
}
Ptr<Pool2Int8Layer> Pool2Int8Layer::create(const Pool2Int8Params& params)
{
return Ptr<Pool2Int8Layer>(new Pool2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/layer.cpp
+7 -2
View File
@@ -269,9 +269,9 @@ void Layer::getTypes(const std::vector<MatType>&inputs,
if (preferableTarget == DNN_TARGET_CUDA_FP16 || preferableTarget == DNN_TARGET_CUDA)
CV_CheckTypeEQ(input, CV_32F, "");
else if (preferableTarget == DNN_TARGET_OPENCL_FP16)
CV_CheckType(input, input == CV_16F || input == CV_8S || input == CV_64F || input == CV_64S, "");
CV_CheckType(input, input == CV_16F || input == CV_8S || input == CV_8U || input == CV_64F || input == CV_64S, "");
else
CV_CheckType(input, input == CV_32F || input == CV_64F || input == CV_8S || input == CV_64S, "");
CV_CheckType(input, input == CV_32F || input == CV_64F || input == CV_8S || input == CV_8U || input == CV_64S, "");
}
outputs.assign(requiredOutputs, inputs[0]);
@@ -314,6 +314,11 @@ bool Layer::dynamicOutputShapes() const
return false;
}
bool Layer::isDataShuffling() const
{
return false;
}
std::ostream& Layer::dumpAttrs(std::ostream& strm, int) const
{
return strm;
modules/dnn/src/layers/concat_layer.cpp
+2
View File
@@ -78,6 +78,8 @@ public:
scale = params.get<float>("scales", 1.0f);
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/dequantizelinear_layer.cpp
+5 -2
View File
@@ -170,12 +170,15 @@ static void dequantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp,
CV_Assert(zpshape == scshape);
}
axis = normalize_axis(axis, ndims);
if (ndims > 0)
axis = normalize_axis(axis, ndims);
else
axis = 0;
for (i = 0; i < axis; i++)
nslices *= inpshape[i];
for (i = axis+1; i < ndims; i++)
slice_size *= inpshape[i];
int sz_a = inpshape[axis];
int sz_a = ndims > 0 ? inpshape[axis] : 1;
if (block_size == 0) {
size_t sc_total = scshape.total();
modules/dnn/src/layers/flatten_layer.cpp
+2
View File
@@ -87,6 +87,8 @@ public:
backendId == DNN_BACKEND_CANN;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/maxpool_layer.cpp
+2
View File
@@ -456,6 +456,8 @@ public:
temptypes.clear();
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
virtual bool getMemoryShapes(const std::vector<MatShape>& inpshapes,
const int,
std::vector<MatShape> &outshapes,
modules/dnn/src/layers/permute_layer.cpp
+2
View File
@@ -147,6 +147,8 @@ public:
backendId == DNN_BACKEND_CANN;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/quantlizelinear_layer.cpp
+5 -2
View File
@@ -180,12 +180,15 @@ static void quantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp,
CV_Assert(zpshape == scshape);
}
axis = normalize_axis(axis, ndims);
if (ndims > 0)
axis = normalize_axis(axis, ndims);
else
axis = 0;
for (i = 0; i < axis; i++)
nslices *= inpshape[i];
for (i = axis+1; i < ndims; i++)
slice_size *= inpshape[i];
int sz_a = inpshape[axis];
int sz_a = ndims > 0 ? inpshape[axis] : 1;
if (block_size == 0) {
size_t sc_total = scshape.total();
modules/dnn/src/layers/reshape2_layer.cpp
+2
View File
@@ -57,6 +57,8 @@ public:
}
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
virtual bool dynamicOutputShapes() const CV_OVERRIDE
{
if (!dynamicShapeSpec)
modules/dnn/src/layers/reshape_layer.cpp
+2
View File
@@ -239,6 +239,8 @@ public:
backendId == DNN_BACKEND_CANN;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/slice2_layer.cpp
+2
View File
@@ -128,6 +128,8 @@ public:
return outShape;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/slice_layer.cpp
+2
View File
@@ -237,6 +237,8 @@ public:
return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_CANN;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/split2_layer.cpp
+2
View File
@@ -142,6 +142,8 @@ public:
}
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int noutputs,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/split_layer.cpp
+2
View File
@@ -78,6 +78,8 @@ public:
backendId == DNN_BACKEND_CUDA;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/squeeze_layer.cpp
+2
View File
@@ -81,6 +81,8 @@ public:
return outShape;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/tile2_layer.cpp
+2
View File
@@ -139,6 +139,8 @@ public:
return backendId == DNN_BACKEND_OPENCV;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
virtual bool dynamicOutputShapes() const CV_OVERRIDE
{
Net::Impl* netimpl_ = getNetImpl(this);
modules/dnn/src/layers/tile_layer.cpp
+2
View File
@@ -38,6 +38,8 @@ public:
backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int requiredOutputs,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/transpose_layer.cpp
+2
View File
@@ -148,6 +148,8 @@ public:
return outShape;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int,
std::vector<MatShape> &outputs,
modules/dnn/src/layers/unsqueeze_layer.cpp
+2
View File
@@ -78,6 +78,8 @@ public:
return outShape;
}
bool isDataShuffling() const CV_OVERRIDE { return true; }
bool getMemoryShapes(const std::vector<MatShape> &inputs,
const int,
std::vector<MatShape> &outputs,
modules/dnn/src/onnx/onnx_importer2.cpp
+269 -498
View File
@@ -266,15 +266,21 @@ protected:
void parseDequantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQuantizeLinear (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseCustomLayer (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQAvgPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQEltwise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQSigmoid (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
//void parseQSoftmax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQConv (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQMatMul (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQGemm (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQEltwise (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQLeakyRelu (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQSigmoid (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQAvgPool (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQConcat (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
void parseQSoftmax (LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto);
Arg addDequantize(const std::string& name, const std::vector<Arg>& inputs,
const opencv_onnx::NodeProto& node_proto, int axis = -1);
void addQuantize(const std::string& name, const Arg& data,
const std::vector<Arg>& scale_zp, const std::vector<Arg>& outputs,
const opencv_onnx::NodeProto& node_proto);
std::map<std::string, int> onnx_opset_map; // map from OperatorSetIdProto
void parseOperatorSet();
@@ -2073,559 +2079,323 @@ void ONNXImporter2::parseRandomNormalLike(LayerParams& layerParams, const opencv
addLayer(layerParams, node_proto);
}
// BUG: https://github.com/opencv/opencv/issues/26310
/*void ONNXImporter2::parseQConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_)
// All Q* parsers decompose quantized ops into: DequantizeLinear → float op → QuantizeLinear.
// The fused Int8 layers don't work with the new graph engine (issue #26310).
Arg ONNXImporter2::addDequantize(const std::string& name, const std::vector<Arg>& inputs,
const opencv_onnx::NodeProto& node_proto, int axis)
{
LayerParams lp;
lp.name = name;
lp.type = "DequantizeLinear";
if (axis >= 0)
lp.set("axis", axis);
Arg out = net.getArg(name);
node_inputs = inputs;
node_outputs = {out};
addLayer(lp, node_proto);
return out;
}
void ONNXImporter2::addQuantize(const std::string& name, const Arg& data,
const std::vector<Arg>& scale_zp,
const std::vector<Arg>& outputs,
const opencv_onnx::NodeProto& node_proto)
{
LayerParams lp;
lp.name = name;
lp.type = "QuantizeLinear";
node_inputs = {data};
node_inputs.insert(node_inputs.end(), scale_zp.begin(), scale_zp.end());
node_outputs = outputs;
addLayer(lp, node_proto);
}
static void copyAttrs(const LayerParams& src, LayerParams& dst)
{
for (auto it = src.begin(); it != src.end(); ++it)
{
const std::string& key = it->first;
if (key != "name" && key != "type")
dst.set(key, it->second);
}
}
void ONNXImporter2::parseQConv(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
opencv_onnx::NodeProto node_proto = node_proto_;
int ninputs = node_proto.input_size();
CV_Assert(ninputs == 8 || ninputs == 9);
float inp_sc = getScalarFromMat<float>(getBlob(node_proto, 1));
int inp_zp = (int)getScalarFromMat<int8_t>(getBlob(node_proto, 2));
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
if (layerParams.has("pad"))
Arg dq_x, dq_w;
dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto);
dq_w = addDequantize(bn + "/dequant_w", {inp[3], inp[4], inp[5]}, node_proto, 0);
Arg bias_arg;
bool has_bias = (ninputs == 9);
if (has_bias)
{
bool asymmetricPadding = false;
DictValue pads = layerParams.get("pad");
const int dims = pads.size() / 2;
for (int i = 0; i < dims; ++i)
if (net.isConstArg(inp[1]) && net.isConstArg(inp[4]) && net.isConstArg(inp[8]))
{
if (pads.get<int>(i) != pads.get<int>(i + dims))
{
asymmetricPadding = true;
break;
}
Mat x_scale_mat = net.argTensor(inp[1]);
Mat w_scale_mat = net.argTensor(inp[4]);
Mat bias_int32 = net.argTensor(inp[8]);
Mat bias_float;
bias_int32.convertTo(bias_float, CV_32F);
float x_sc = x_scale_mat.at<float>(0);
if (w_scale_mat.total() == 1)
bias_float *= x_sc * w_scale_mat.at<float>(0);
else
for (int i = 0; i < (int)w_scale_mat.total(); i++)
bias_float.at<float>(i) *= x_sc * w_scale_mat.at<float>(i);
bias_arg = netimpl->newConstArg(bn + "/bias_float", bias_float);
}
if (asymmetricPadding && pads.size() == 4)
else
{
layerParams.erase("pad");
std::vector<int> paddings(4, 0);
for (int i = 0; i < dims; ++i)
{
paddings.push_back(pads.get<int>(i));
paddings.push_back(pads.get<int>(dims + i));
}
LayerParams padLp;
padLp.name = layerParams.name + "/pad";
padLp.type = "PaddingInt8";
padLp.set("paddings", DictValue::arrayInt(&paddings[0], paddings.size()));
padLp.set("depth", CV_8S);
padLp.set<double>("value", (double)inp_zp);
opencv_onnx::NodeProto proto;
proto.add_input(node_proto.input(0));
proto.add_output(padLp.name);
addLayer(padLp, proto);
node_proto.set_input(0, padLp.name);
bias_arg = addDequantize(bn + "/dequant_bias", {inp[8], inp[1]}, node_proto);
}
}
Mat weights = getBlob(node_proto, 3);
int outCn = weights.size[0];
Mat w_scale = getBlob(node_proto, 4);
CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn);
bool per_channel = w_scale.total() == outCn;
Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at<float>(0)));
LayerParams convLp;
convLp.name = bn + "/conv";
convLp.type = "Conv2";
copyAttrs(layerParams, convLp);
Arg conv_out = net.getArg(convLp.name);
node_inputs = has_bias ? std::vector<Arg>{dq_x, dq_w, bias_arg} : std::vector<Arg>{dq_x, dq_w};
node_outputs = {conv_out};
addLayer(convLp, node_proto);
float out_sc = getScalarFromMat<float>(getBlob(node_proto, 6));
int8_t out_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 7));
Mat bias = (ninputs == 9) ? getBlob(node_proto, 8) : Mat::zeros(1, outCn, CV_32S);
Mat weights_2d = weights.reshape(1, outCn);
Mat biasFused(1, outCn, CV_32S);
Mat outputMultiplier(1, outCn, CV_32F);
for (int i = 0; i < outCn; i++)
{
biasFused.at<int>(i) = bias.at<int>(i) - inp_zp*(cv::sum(weights_2d.row(i))[0]);
outputMultiplier.at<float>(i) = (inp_sc * wt_sc.at<float>(i)) / out_sc;
}
layerParams.type = "ConvolutionInt8";
layerParams.set("num_output", outCn);
layerParams.set("input_zeropoint", inp_zp);
layerParams.set("input_scale",inp_sc);
layerParams.set("zeropoints", out_zp);
layerParams.set("scales", out_sc);
layerParams.set("per_channel", per_channel);
layerParams.blobs.push_back(weights);
layerParams.blobs.push_back(biasFused);
layerParams.blobs.push_back(outputMultiplier);
addLayer(layerParams, node_proto);
addQuantize(bn + "/quant_y", conv_out, {inp[6], inp[7]}, out, node_proto);
}
void ONNXImporter2::parseQMatMul(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
int ninputs = node_proto.input_size();
CV_Assert(ninputs == 8);
CV_Assert(node_proto.input_size() == 8);
if (constBlobs.find(node_proto.input(3)) == constBlobs.end())
CV_Error(Error::StsNotImplemented, "Variable weights is not supported");
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
int firstInpDims = outShapes[node_proto.input(0)].size();
Arg dq_a, dq_b;
dq_a = addDequantize(bn + "/dequant_a", {inp[0], inp[1], inp[2]}, node_proto);
dq_b = addDequantize(bn + "/dequant_b", {inp[3], inp[4], inp[5]}, node_proto);
float inp_sc = getScalarFromMat<float>(getBlob(node_proto, 1));
int8_t inp_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 2));
LayerParams mmLp;
mmLp.name = bn + "/matmul";
mmLp.type = "MatMul";
Arg mm_out = net.getArg(mmLp.name);
node_inputs = {dq_a, dq_b};
node_outputs = {mm_out};
addLayer(mmLp, node_proto);
Mat weights = getBlob(node_proto, 3).t();
int outCn = weights.size[0];
int secondInpDims = weights.dims;
Mat w_scale = getBlob(node_proto, 4);
CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn);
bool per_channel = w_scale.total() == outCn ? true : false;
Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at<float>(0)));
float out_sc = getScalarFromMat<float>(getBlob(node_proto, 6));
int8_t out_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 7));
Mat bias(1, outCn, CV_32S);
Mat outputMultiplier(1, outCn, CV_32F);
for (int i = 0; i < outCn; i++)
{
bias.at<int>(i) = -inp_zp*(cv::sum(weights.row(i))[0]);
outputMultiplier.at<float>(i) = (inp_sc * wt_sc.at<float>(i)) / out_sc;
}
layerParams.type = "InnerProductInt8";
layerParams.set("num_output", outCn);
layerParams.set("axis", firstInpDims - secondInpDims + 1);
layerParams.set("input_scale", inp_sc);
layerParams.set("input_zeropoint", inp_zp);
layerParams.set("zeropoints", out_zp);
layerParams.set("scales", out_sc);
layerParams.set("per_channel", per_channel);
layerParams.blobs.push_back(weights);
layerParams.blobs.push_back(bias);
layerParams.blobs.push_back(outputMultiplier);
addLayer(layerParams, node_proto);
addQuantize(bn + "/quant_y", mm_out, {inp[6], inp[7]}, out, node_proto);
}
// A * B + C = Y, we require that the dimension of A is [m, k], and the dimension of B is [n, k].
// And the dim of output Y is [m, n]
void ONNXImporter2::parseQGemm(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
int ninputs = node_proto.input_size();
CV_Assert(ninputs == 8 || ninputs == 9);
layerParams.type = "InnerProductInt8";
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
if (constBlobs.find(node_proto.input(3)) == constBlobs.end())
CV_Error(Error::StsNotImplemented, "Variable weights is not supported");
Arg dq_a, dq_b;
dq_a = addDequantize(bn + "/dequant_a", {inp[0], inp[1], inp[2]}, node_proto);
dq_b = addDequantize(bn + "/dequant_b", {inp[3], inp[4], inp[5]}, node_proto, 0);
Mat weights = getBlob(node_proto, 3);
LayerParams gemmLp;
gemmLp.name = bn + "/gemm";
gemmLp.type = "Gemm";
copyAttrs(layerParams, gemmLp);
Arg gemm_out = net.getArg(gemmLp.name);
node_inputs = (!node_proto.input(6).empty()) ? std::vector<Arg>{dq_a, dq_b, inp[6]}
: std::vector<Arg>{dq_a, dq_b};
node_outputs = {gemm_out};
addLayer(gemmLp, node_proto);
if (!layerParams.get<int>("transB", 0))
{
transpose(weights, weights);
}
CV_Assert(layerParams.get<float>("alpha", 1) == 1.0f);
CV_Assert(layerParams.get<int>("transA", 0) == 0);
int firstInpDims = outShapes[node_proto.input(0)].size();
float inp_sc = getScalarFromMat<float>(getBlob(node_proto, 1));
int8_t inp_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 2));
int outCn = weights.size[0];
int secondInpDims = weights.dims;
Mat w_scale = getBlob(node_proto, 4);
CV_Assert(w_scale.total() == 1 || w_scale.total() == outCn);
bool per_channel = w_scale.total() == outCn;
Mat wt_sc = (w_scale.total() == outCn) ? w_scale : Mat(1, outCn, CV_32F, Scalar(w_scale.at<float>(0)));
Mat w_zp = getBlob(node_proto, 5);
int8_t* ptrZp = w_zp.ptr<int8_t>(0);
for (int i = 0; i < w_zp.total(); i++)
{
if (ptrZp[i] != (int8_t)0)
CV_Error(Error::StsUnsupportedFormat, "The zero-point non-zero case of W is not supported!");
}
float out_sc = getScalarFromMat<float>(getBlob(node_proto, 7));
int8_t out_zp = ninputs == 9 ? getScalarFromMat<int8_t>(getBlob(node_proto, 8)) : 0;
Mat bias;
if (constBlobs.find(node_proto.input(6)) != constBlobs.end())
bias = getBlob(node_proto, 6);
if (bias.empty())
bias = Mat::zeros(1, outCn, CV_32S);
Mat biasFused(1, outCn, CV_32S);
Mat outputMultiplier(1, outCn, CV_32F);
for (int i = 0; i < outCn; i++)
{
biasFused.at<int>(i) = bias.at<int>(i) - inp_zp*(cv::sum(weights.row(i))[0]);
outputMultiplier.at<float>(i) = (inp_sc * wt_sc.at<float>(i)) / out_sc;
}
layerParams.type = "InnerProductInt8";
layerParams.set("num_output", outCn);
layerParams.set("axis", firstInpDims - secondInpDims + 1);
layerParams.set("input_scale", inp_sc);
layerParams.set("input_zeropoint", inp_zp);
layerParams.set("scales", out_sc);
layerParams.set("zeropoints", out_zp);
layerParams.set("per_channel", per_channel);
layerParams.blobs.push_back(weights);
layerParams.blobs.push_back(biasFused);
layerParams.blobs.push_back(outputMultiplier);
addLayer(layerParams, node_proto);
std::vector<Arg> q_sc_zp = {inp[7]};
if (ninputs == 9)
q_sc_zp.push_back(inp[8]);
addQuantize(bn + "/quant_y", gemm_out, q_sc_zp, out, node_proto);
}
void ONNXImporter2::parseQEltwise(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_)
void ONNXImporter2::parseQEltwise(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
opencv_onnx::NodeProto node_proto = node_proto_;
CV_Assert(node_proto.input_size() == 7 || node_proto.input_size() == 8);
std::string op = (node_proto.op_type() == "QLinearAdd") ? "sum" : "prod";
int constId = -1;
for (int i = 0; i < 4; i += 3)
{
if (constBlobs.find(node_proto.input(i)) != constBlobs.end())
constId = i;
}
int ninputs = node_proto.input_size();
CV_Assert(ninputs == 7 || ninputs == 8);
float inp_0_sc = getScalarFromMat<float>(getBlob(node_proto, 1));
int8_t inp_0_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 2));
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
float inp_1_sc = getScalarFromMat<float>(getBlob(node_proto, 4));
int8_t inp_1_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 5));
Arg dq_a, dq_b;
dq_a = addDequantize(bn + "/dequant_a", {inp[0], inp[1], inp[2]}, node_proto);
dq_b = addDequantize(bn + "/dequant_b", {inp[3], inp[4], inp[5]}, node_proto);
// Set 2nd input as the const input
if (constId == 0)
{
cv::swap(inp_0_sc, inp_1_sc);
cv::swap(inp_0_zp, inp_1_zp);
}
LayerParams eltLp;
eltLp.name = bn + "/eltwise";
eltLp.type = "NaryEltwise";
eltLp.set("operation", (node_proto.op_type() == "QLinearAdd") ? "add" : "mul");
Arg elt_out = net.getArg(eltLp.name);
node_inputs = {dq_a, dq_b};
node_outputs = {elt_out};
addLayer(eltLp, node_proto);
float out_sc = getScalarFromMat<float>(getBlob(node_proto, 6));
int8_t out_zp = 0;
if (node_proto.input_size() == 8)
out_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 7));
std::vector<float> inp_scales = {inp_0_sc, inp_1_sc};
std::vector<int8_t> inp_zps = {inp_0_zp, inp_1_zp};
std::vector<float> coeffs;
float offset;
if (op == "sum")
{
coeffs = {inp_scales[0]/out_sc, inp_scales[1]/out_sc};
offset = out_zp - coeffs[0]*inp_zps[0] - coeffs[1]*inp_zps[1];
}
else
{
coeffs = {inp_scales[0]/out_sc, inp_scales[1]};
offset = out_zp;
}
if (constId != -1)
{
Mat blob = getBlob(node_proto, constId);
if (blob.total() == 1)
{
float val = inp_scales[1] * (blob.at<int8_t>(0) - inp_zps[1]);
float scale = inp_scales[0] / out_sc;
if (op == "prod")
scale *= val;
float shift = out_zp - scale*inp_zps[0];
if (op == "sum")
shift += (val/out_sc);
LayerParams rescaleParams;
rescaleParams.name = layerParams.name;
rescaleParams.type = "Requantize";
rescaleParams.set("depth", CV_8S);
rescaleParams.set("scale", scale);
rescaleParams.set("shift", shift);
rescaleParams.set("isEltwise", true);
addLayer(rescaleParams, node_proto);
return;
}
else
{
MatShape inpShape = outShapes[node_proto.input(3 - constId)];
if (blob.dims == 2)
blob = blob.t();
if (shape(blob) == inpShape)
{
LayerParams constParams;
constParams.name = layerParams.name + "/const";
constParams.type = "ConstInt8";
constParams.set("depth", CV_8S);
constParams.set("scales", inp_1_sc);
constParams.set("zeropoints", inp_1_zp);
constParams.blobs.push_back(blob);
int id = net.addLayer(constParams.name, constParams.type, CV_8S, constParams);
layer_id.insert(std::make_pair(constParams.name, LayerInfo(id, 0, CV_8S)));
outShapes[constParams.name] = shape(blob);
node_proto.set_input(constId, constParams.name);
layerParams.type = "EltwiseInt8";
layerParams.set("operation", op);
layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size()));
layerParams.set("offset", offset);
}
else
{
layerParams.type = "ScaleInt8";
layerParams.set("bias_term", op == "sum");
int axis = 1;
for (int i = 0; i < graph_proto->initializer_size(); i++)
{
opencv_onnx::TensorProto tensor_proto = graph_proto->initializer(i);
if (tensor_proto.name() == node_proto.input(constId))
{
axis = inpShape.size() - tensor_proto.dims_size();
break;
}
}
layerParams.set("axis", axis);
blob = blob.reshape(1, 1);
Mat blob_dequantized;
blob.convertTo(blob_dequantized, CV_32F, inp_scales[1], -(inp_scales[1] * inp_zps[1]));
layerParams.blobs.push_back(blob_dequantized);
}
}
}
else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(3)])
{
layerParams.type = "EltwiseInt8";
layerParams.set("operation", op);
layerParams.set("coeff", DictValue::arrayReal(coeffs.data(), coeffs.size()));
layerParams.set("offset", offset);
}
else
{
layerParams.type = "ScaleInt8";
layerParams.set("bias_term", op == "sum");
}
layerParams.set("input_scales", DictValue::arrayReal(inp_scales.data(), inp_scales.size()));
layerParams.set("input_zeropoints", DictValue::arrayInt(inp_zps.data(), inp_zps.size()));
layerParams.set("scales", out_sc);
layerParams.set("zeropoints", out_zp);
addLayer(layerParams, node_proto);
std::vector<Arg> q_sc_zp = {inp[6]};
if (ninputs == 8)
q_sc_zp.push_back(inp[7]);
addQuantize(bn + "/quant_y", elt_out, q_sc_zp, out, node_proto);
}
void ONNXImporter2::parseQLeakyRelu(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
CV_Assert(node_proto.input_size() == 4 || node_proto.input_size() == 5);
int ninputs = node_proto.input_size();
CV_Assert(ninputs == 4 || ninputs == 5);
float slope = layerParams.get<float>("alpha");
float inp_sc = getScalarFromMat<float>(getBlob(node_proto, 1));
int8_t inp_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 2));
float out_sc = getScalarFromMat<float>(getBlob(node_proto, 3));
int8_t out_zp = node_proto.input_size() == 4 ? 0 : getScalarFromMat<int8_t>(getBlob(node_proto, 4));
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
Mat lookUpTable(1, 256, CV_8S);
int8_t* table = lookUpTable.ptr<int8_t>();
for (int i = -128; i < 128; i++)
{
float x = inp_sc*(i - inp_zp);
float y = x >= 0.f ? x : slope*x;
int quantized = out_zp + cvRound(y/out_sc);
table[i+128] = saturate_cast<int8_t>(quantized);
}
Arg dq_x;
dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto);
layerParams.type = "ReLUInt8";
layerParams.set("input_scale", inp_sc);
layerParams.set("input_zeropoint", inp_zp);
layerParams.set("scales", out_sc);
layerParams.set("zeropoints", out_zp);
layerParams.set("slope", slope);
layerParams.blobs.push_back(lookUpTable);
addLayer(layerParams, node_proto);
LayerParams reluLp;
reluLp.name = bn + "/leaky_relu";
reluLp.type = "ReLU";
reluLp.set("negative_slope", layerParams.get<float>("alpha", 0.01f));
Arg relu_out = net.getArg(reluLp.name);
node_inputs = {dq_x};
node_outputs = {relu_out};
addLayer(reluLp, node_proto);
std::vector<Arg> q_sc_zp = {inp[3]};
if (ninputs == 5)
q_sc_zp.push_back(inp[4]);
addQuantize(bn + "/quant_y", relu_out, q_sc_zp, out, node_proto);
}
void ONNXImporter2::parseQSigmoid(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
CV_Assert(node_proto.input_size() == 4 || node_proto.input_size() == 5);
int ninputs = node_proto.input_size();
CV_Assert(ninputs == 4 || ninputs == 5);
float inp_sc = getScalarFromMat<float>(getBlob(node_proto, 1));
int8_t inp_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 2));
float out_sc = getScalarFromMat<float>(getBlob(node_proto, 3));
int8_t out_zp = node_proto.input_size() == 4 ? 0 : getScalarFromMat<int8_t>(getBlob(node_proto, 4));
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
Mat lookUpTable(1, 256, CV_8S);
int8_t* table = lookUpTable.ptr<int8_t>();
for (int i = -128; i < 128; i++)
{
float x = inp_sc*(i - inp_zp);
float y = 1.f/(1.f + std::exp(-x));
int quantized = out_zp + cvRound(y/out_sc);
table[i+128] = saturate_cast<int8_t>(quantized);
}
Arg dq_x;
dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto);
layerParams.type = "SigmoidInt8";
layerParams.set("input_scale", inp_sc);
layerParams.set("input_zeropoint", inp_zp);
layerParams.set("scales", out_sc);
layerParams.set("zeropoints", out_zp);
layerParams.blobs.push_back(lookUpTable);
addLayer(layerParams, node_proto);
LayerParams sigLp;
sigLp.name = bn + "/sigmoid";
sigLp.type = "Sigmoid";
Arg sig_out = net.getArg(sigLp.name);
node_inputs = {dq_x};
node_outputs = {sig_out};
addLayer(sigLp, node_proto);
std::vector<Arg> q_sc_zp = {inp[3]};
if (ninputs == 5)
q_sc_zp.push_back(inp[4]);
addQuantize(bn + "/quant_y", sig_out, q_sc_zp, out, node_proto);
}
void ONNXImporter2::parseQAvgPool(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
CV_Assert(node_proto.input_size() == 4 || node_proto.input_size() == 5);
int ninputs = node_proto.input_size();
CV_Assert(ninputs == 4 || ninputs == 5);
float inp_sc = getScalarFromMat<float>(getBlob(node_proto, 1));
int8_t inp_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 2));
float out_sc = getScalarFromMat<float>(getBlob(node_proto, 3));
int8_t out_zp = node_proto.input_size() == 4 ? 0 : getScalarFromMat<int8_t>(getBlob(node_proto, 4));
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
layerParams.type = "PoolingInt8";
layerParams.set("pool", "ave");
layerParams.set("global_pooling", node_proto.op_type() == "QLinearGlobalAveragePool");
layerParams.set("multiplier", inp_sc/out_sc);
layerParams.set("input_scale", inp_sc);
layerParams.set("input_zeropoint", inp_zp);
layerParams.set("scales", out_sc);
layerParams.set("zeropoints", out_zp);
addLayer(layerParams, node_proto);
Arg dq_x;
dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto);
LayerParams poolLp;
poolLp.name = bn + "/avg_pool";
poolLp.type = "Pooling";
poolLp.set("pool", "ave");
poolLp.set("global_pooling", node_proto.op_type() == "QLinearGlobalAveragePool");
copyAttrs(layerParams, poolLp);
Arg pool_out = net.getArg(poolLp.name);
node_inputs = {dq_x};
node_outputs = {pool_out};
addLayer(poolLp, node_proto);
std::vector<Arg> q_sc_zp = {inp[3]};
if (ninputs == 5)
q_sc_zp.push_back(inp[4]);
addQuantize(bn + "/quant_y", pool_out, q_sc_zp, out, node_proto);
}
void ONNXImporter2::parseQConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto_)
void ONNXImporter2::parseQConcat(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
opencv_onnx::NodeProto node_proto = node_proto_;
layerParams.type = "ConcatInt8";
int num_inputs = node_proto.input_size();
int ninputs = node_proto.input_size();
CV_Assert(ninputs >= 5 && (ninputs - 2) % 3 == 0);
float out_scale = getScalarFromMat<float>(getBlob(node_proto, 0));
int8_t out_zp = getScalarFromMat<int8_t>(getBlob(node_proto, 1));
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
for (int i = 2; i < num_inputs; i += 3)
std::vector<Arg> dq_outs;
int n = (ninputs - 2) / 3;
for (int i = 0; i < n; i++)
{
float inp_scale = getScalarFromMat<float>(getBlob(node_proto, i + 1));
int8_t inp_zp = getScalarFromMat<int8_t>(getBlob(node_proto, i + 2));
if (inp_scale != out_scale || inp_zp != out_zp)
{
float scale = inp_scale/out_scale;
float shift = out_zp - scale*inp_zp;
if (constBlobs.find(node_proto.input(i)) != constBlobs.end())
{
Mat blob = getBlob(node_proto, i);
Mat blob_rescaled;
blob.convertTo(blob_rescaled, CV_8S, scale, shift);
constBlobs[node_proto.input(i)] = blob_rescaled;
}
else
{
LayerParams rescaleParams;
rescaleParams.name = node_proto.input(i) + "/rescale";
rescaleParams.type = "Requantize";
rescaleParams.set("depth", CV_8S);
rescaleParams.set("scale", scale);
rescaleParams.set("shift", shift);
rescaleParams.set("isEltwise", false);
opencv_onnx::NodeProto proto;
proto.add_input(node_proto.input(i));
proto.add_output(rescaleParams.name);
addLayer(rescaleParams, proto);
node_proto.set_input(i, rescaleParams.name);
}
}
int b = 2 + i * 3;
Arg dq = addDequantize(bn + "/dequant_" + std::to_string(i),
{inp[b], inp[b + 1], inp[b + 2]}, node_proto);
dq_outs.push_back(dq);
}
bool hasVariableInps = false;
for (int i = 2; i < num_inputs; i += 3)
{
if (layer_id.find(node_proto.input(i)) != layer_id.end())
{
hasVariableInps = true;
break;
}
}
LayerParams concatLp;
concatLp.name = bn + "/concat";
concatLp.type = "Concat";
concatLp.set("axis", layerParams.get<int>("axis", 1));
Arg concat_out = net.getArg(concatLp.name);
node_inputs = dq_outs;
node_outputs = {concat_out};
addLayer(concatLp, node_proto);
if (!hasVariableInps)
{
std::vector<Mat> inputs, concatenated;
MatShape inputShape;
for (size_t i = 2; i < num_inputs; i += 3)
{
Mat blob = getBlob(node_proto, i);
if (blob.size.dims() > inputShape.size())
{
inputShape = shape(blob);
}
inputs.push_back(blob);
}
int axis = layerParams.get<int>("axis", 1);
for (size_t i = 0; i < inputs.size(); ++i)
{
MatShape targetShape = inputShape;
targetShape[axis] = shape(inputs[i])[axis];
CV_CheckEQ(total(targetShape), total(shape(inputs[i])), "");
inputs[i] = inputs[i].reshape(0, targetShape);
}
runLayer(layerParams, inputs, concatenated);
CV_Assert(concatenated.size() == 1);
addConstant(layerParams.name, concatenated[0]);
return;
}
else
{
for (int i = 2; i < num_inputs; i += 3)
{
if (constBlobs.find(node_proto.input(i)) != constBlobs.end())
{
LayerParams constParams;
constParams.name = node_proto.input(i);
constParams.type = "ConstInt8";
constParams.blobs.push_back(getBlob(node_proto, i));
constParams.set("depth", CV_8S);
opencv_onnx::NodeProto proto;
proto.add_output(constParams.name);
addLayer(constParams, proto);
}
}
}
layerParams.set("scales", out_scale);
layerParams.set("zeropoints", out_zp);
addLayer(layerParams, node_proto);
addQuantize(bn + "/quant_y", concat_out, {inp[0], inp[1]}, out, node_proto);
}
void ONNXImporter2::parseQSoftmax(LayerParams& layerParams, const opencv_onnx::NodeProto& node_proto)
{
CV_CheckEQ(node_proto.input_size(), 5, "DNN/ONNX: QLinearSoftmax requires 5 inputs, X, X_scale, X_zero_point, Y_scale, Y_zero_point");
CV_CheckEQ(node_proto.input_size(), 5, "DNN/ONNX: QLinearSoftmax requires 5 inputs");
int opset = layerParams.get<int>("opset");
if (opset < 13) {
layerParams.set("coerced_2d", true);
std::string bn = layerParams.name;
std::vector<Arg> inp = node_inputs;
std::vector<Arg> out = node_outputs;
Arg dq_x;
dq_x = addDequantize(bn + "/dequant_x", {inp[0], inp[1], inp[2]}, node_proto);
LayerParams smLp;
smLp.name = bn + "/softmax";
smLp.type = "Softmax";
if (layerParams.has("axis"))
smLp.set("axis", layerParams.get<int>("axis"));
if (layerParams.has("opset"))
{
int opset = layerParams.get<int>("opset");
smLp.set("opset", opset);
if (opset < 13)
smLp.set("coerced_2d", true);
}
Arg sm_out = net.getArg(smLp.name);
node_inputs = {dq_x};
node_outputs = {sm_out};
addLayer(smLp, node_proto);
float x_scale = getScalarFromMat<float>(getBlob(node_proto, 1));
int8_t x_zero_point = getScalarFromMat<int8_t>(getBlob(node_proto, 2));
float y_scale = getScalarFromMat<float>(getBlob(node_proto, 3));
int8_t y_zero_point = getScalarFromMat<int8_t>(getBlob(node_proto, 4));
layerParams.type = "SoftmaxInt8";
// layerParams also has "axis" and "opset" attrs
layerParams.set("input_scale", x_scale);
layerParams.set("input_zeropoint", x_zero_point);
layerParams.set("scales", y_scale);
layerParams.set("zeropoints", y_zero_point);
addLayer(layerParams, node_proto);
}*/
addQuantize(bn + "/quant_y", sm_out, {inp[3], inp[4]}, out, node_proto);
}
void ONNXImporter2::parseRotaryEmbedding(LayerParams& params, const opencv_onnx::NodeProto& node_proto) {
int i, n_inputs = node_proto.input_size();
@@ -2819,8 +2589,8 @@ void ONNXImporter2::buildDispatchMap_ONNX_AI()
// ai.onnx: opset 10+
dispatch["DequantizeLinear"] = &ONNXImporter2::parseDequantizeLinear;
dispatch["QuantizeLinear"] = &ONNXImporter2::parseQuantizeLinear;
//dispatch["QLinearConv"] = &ONNXImporter2::parseQConv;
//dispatch["QLinearMatMul"] = &ONNXImporter2::parseQMatMul;
dispatch["QLinearConv"] = &ONNXImporter2::parseQConv;
dispatch["QLinearMatMul"] = &ONNXImporter2::parseQMatMul;
// com.microsft: This operator is added for compatibility via onnx graph simplifier.
// Opset domain cannot be modified from onnx_graph_simplifier.cpp so this
@@ -2836,14 +2606,15 @@ void ONNXImporter2::buildDispatchMap_COM_MICROSOFT()
{
DispatchMap dispatch;
// BUG: https://github.com/opencv/opencv/issues/26310
//dispatch["QLinearAdd"] = dispatch["QLinearMul"] = &ONNXImporter2::parseQEltwise;
//dispatch["QLinearAveragePool"] = dispatch["QLinearGlobalAveragePool"] = &ONNXImporter2::parseQAvgPool;
//dispatch["QLinearLeakyRelu"] = &ONNXImporter2::parseQLeakyRelu;
//dispatch["QLinearSigmoid"] = &ONNXImporter2::parseQSigmoid;
//dispatch["QLinearConcat"] = &ONNXImporter2::parseQConcat;
//dispatch["QGemm"] = &ONNXImporter2::parseQGemm;
//dispatch["QLinearSoftmax"] = &ONNXImporter2::parseQSoftmax;
dispatch["QLinearAdd"] = dispatch["QLinearMul"] = &ONNXImporter2::parseQEltwise;
dispatch["QLinearAveragePool"] = dispatch["QLinearGlobalAveragePool"] = &ONNXImporter2::parseQAvgPool;
dispatch["QLinearLeakyRelu"] = &ONNXImporter2::parseQLeakyRelu;
dispatch["QLinearSigmoid"] = &ONNXImporter2::parseQSigmoid;
dispatch["QLinearConcat"] = &ONNXImporter2::parseQConcat;
dispatch["QLinearConv"] = &ONNXImporter2::parseQConv;
dispatch["QLinearMatMul"] = &ONNXImporter2::parseQMatMul;
dispatch["QGemm"] = &ONNXImporter2::parseQGemm;
dispatch["QLinearSoftmax"] = &ONNXImporter2::parseQSoftmax;
dispatch["Attention"] = &ONNXImporter2::parseAttention;
domain_dispatch_map[str_domain_com_microsoft] = dispatch;
modules/dnn/test/test_onnx_conformance_layer_filter__openvino.inl.hpp
+3 -3
View File
@@ -1840,11 +1840,11 @@ CASE(test_prelu_broadcast_expanded)
CASE(test_prelu_example_expanded)
SKIP;
CASE(test_qlinearconv)
// no filter
SKIP;
CASE(test_qlinearmatmul_2D)
// no filter
SKIP;
CASE(test_qlinearmatmul_3D)
// no filter
SKIP;
CASE(test_quantizelinear)
SKIP;
CASE(test_quantizelinear_axis)
modules/dnn/test/test_onnx_conformance_layer_filter_opencv_classic_denylist.inl.hpp
+3
View File
@@ -739,3 +739,6 @@
"test_eyelike_populate_off_main_diagonal",
"test_eyelike_with_dtype",
"test_eyelike_without_dtype",
"test_qlinearconv",
"test_qlinearmatmul_2D",
"test_qlinearmatmul_3D",
modules/dnn/test/test_onnx_conformance_layer_parser_denylist.inl.hpp
+2 -5
View File
@@ -368,14 +368,11 @@
"test_optional_has_element_tensor_input",
"test_prelu_broadcast", // Issue::Parser:Blob slope not found in const blobs in function 'getBlob' (weights are required as inputs)
"test_prelu_example", // ---- same as above ---
"test_qlinearconv", // Issue::Parser: Blob x_scale not found in const blobs in function 'getBlob' (weights are required as inputs)
"test_qlinearmatmul_2D", // Issue:: Parser: Variable weights is not supported in function 'parseQMatMul'
"test_qlinearmatmul_2D_int8_float16",
"test_qlinearmatmul_2D_int8_float16", // Float output QLinearMatMul variants not supported
"test_qlinearmatmul_2D_int8_float32",
"test_qlinearmatmul_2D_uint8_float16",
"test_qlinearmatmul_2D_uint8_float32",
"test_qlinearmatmul_3D", // ---- same as above ---
"test_qlinearmatmul_3D_int8_float16",
"test_qlinearmatmul_3D_int8_float16", // Float output QLinearMatMul variants not supported
"test_qlinearmatmul_3D_int8_float32",
"test_qlinearmatmul_3D_uint8_float16",
"test_qlinearmatmul_3D_uint8_float32",
modules/dnn/test/test_onnx_importer.cpp
+11 -7
View File
@@ -2189,11 +2189,15 @@ TEST_P(Test_ONNX_layers, Gemm_External_Data)
TEST_P(Test_ONNX_layers, Quantized_MatMul_Variable_Weights)
{
// Unsupported
EXPECT_THROW(
auto engine_forced = static_cast<cv::dnn::EngineType>(
cv::utils::getConfigurationParameterSizeT("OPENCV_FORCE_DNN_ENGINE", cv::dnn::ENGINE_AUTO));
if (engine_forced == cv::dnn::ENGINE_CLASSIC)
{
testONNXModels("quantized_matmul_variable_inputs");
}, cv::Exception);
applyTestTag(CV_TEST_TAG_DNN_SKIP_PARSER);
return;
}
testONNXModels("quantized_matmul_variable_inputs", npy, 1.3, 1.3);
}
TEST_P(Test_ONNX_layers, Quantized_Eltwise)
@@ -2282,7 +2286,7 @@ TEST_P(Test_ONNX_layers, Quantized_Concat)
TEST_P(Test_ONNX_layers, Quantized_Constant)
{
testONNXModels("quantized_constant", npy, 0.008, 0.02);
testONNXModels("quantized_constant", npy, 0.02, 0.06);
}
TEST_P(Test_ONNX_layers, OutputRegistration)
@@ -2294,8 +2298,8 @@ TEST_P(Test_ONNX_layers, QLinearSoftmax)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);
testONNXModels("qlinearsoftmax_v11", npy, 0.002, 0.002); // 2D coerced
testONNXModels("qlinearsoftmax_v13", npy, 0.002, 0.002);
testONNXModels("qlinearsoftmax_v11", npy, 0.08, 0.16); // 2D coerced
testONNXModels("qlinearsoftmax_v13", npy, 0.08, 0.16);
}
TEST_P(Test_ONNX_layers, PriorBox_ONNX)