mirror of
https://github.com/opencv/opencv.git
synced 2026-07-21 19:33:03 +04:00
Merge pull request #28595 from abhishek-gola:qdq_support
* added qdq fusion * Added VNNI optimizations
This commit is contained in:
Abhishek Gola
GitHub
@@ -1153,6 +1153,10 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
class CV_EXPORTS ActivationLayerInt8 : public ActivationLayer
|
||||
{
|
||||
public:
|
||||
int input_zp, output_zp;
|
||||
float input_sc, output_sc;
|
||||
Mat activationLUT;
|
||||
|
||||
static Ptr<ActivationLayerInt8> create(const LayerParams ¶ms);
|
||||
};
|
||||
|
||||
@@ -1200,6 +1204,13 @@ CV__DNN_INLINE_NS_BEGIN
|
||||
class CV_EXPORTS EltwiseLayerInt8 : public Layer
|
||||
{
|
||||
public:
|
||||
std::vector<float> coeffs;
|
||||
std::vector<int> zeropoints;
|
||||
std::vector<float> scales;
|
||||
float output_sc;
|
||||
int output_zp;
|
||||
float offset;
|
||||
|
||||
static Ptr<EltwiseLayerInt8> create(const LayerParams ¶ms);
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,816 @@
|
||||
// This file is part of OpenCV project.
|
||||
// It is subject to the license terms in the LICENSE file found in the top-level directory
|
||||
// of this distribution and at http://opencv.org/license.html.
|
||||
// Copyright (C) 2026, BigVision LLC, all rights reserved.
|
||||
// Third party copyrights are property of their respective owners.
|
||||
|
||||
#include "precomp.hpp"
|
||||
#include "net_impl.hpp"
|
||||
|
||||
namespace cv { namespace dnn {
|
||||
CV__DNN_INLINE_NS_BEGIN
|
||||
|
||||
using std::vector;
|
||||
using std::string;
|
||||
|
||||
typedef std::pair<int, int> int_pair;
|
||||
typedef std::pair<int, Arg> int_arg_pair;
|
||||
|
||||
struct ModelFusionQDQ
|
||||
{
|
||||
ModelFusionQDQ(Net::Impl* netimpl_) : netimpl(netimpl_) {}
|
||||
|
||||
void fuse()
|
||||
{
|
||||
int i, niter = 10;
|
||||
netimpl->useCounts(usecounts);
|
||||
for (i = 0; i < niter; i++) {
|
||||
bool fused_any = fuseGraph(netimpl->mainGraph);
|
||||
if (!fused_any)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename _LayerType> _LayerType*
|
||||
getLayer(std::vector<Ptr<Layer> >& newprog, int op_idx) const
|
||||
{
|
||||
return op_idx >= 0 ? dynamic_cast<_LayerType*>(newprog.at(op_idx).get()) : 0;
|
||||
}
|
||||
|
||||
LayerParams makeLayerParamsFromOriginal(const Layer* layer, const String& newType) const
|
||||
{
|
||||
LayerParams params;
|
||||
int lid = netimpl->getLayerId(layer->name);
|
||||
if (lid >= 0)
|
||||
params = netimpl->getLayerData(lid).params;
|
||||
params.name = layer->name;
|
||||
params.type = newType;
|
||||
return params;
|
||||
}
|
||||
|
||||
Ptr<Layer> createFusedLayer(const LayerParams& src) const
|
||||
{
|
||||
LayerParams params = src;
|
||||
return LayerFactory::createLayerInstance(params.type, params);
|
||||
}
|
||||
|
||||
template<typename LayerT>
|
||||
bool getQdqPatternContext(Layer* layer_ptr,
|
||||
size_t ninputs,
|
||||
const std::vector<Arg>& inputs,
|
||||
const std::vector<int>& producer_of,
|
||||
std::vector<Ptr<Layer> >& newprog,
|
||||
Arg& q_data_in,
|
||||
Arg& out_scale,
|
||||
Arg& out_zp,
|
||||
int& mid_layer_idx,
|
||||
LayerT*& mid_layer) const
|
||||
{
|
||||
QuantizeLinearLayer* ql = dynamic_cast<QuantizeLinearLayer*>(layer_ptr);
|
||||
if (!(ql && ninputs == 3 && usecounts.at(inputs[0].idx) == 1))
|
||||
return false;
|
||||
q_data_in = inputs[0];
|
||||
out_scale = inputs[1];
|
||||
out_zp = inputs[2];
|
||||
mid_layer_idx = producer_of.at(q_data_in.idx);
|
||||
mid_layer = getLayer<LayerT>(newprog, mid_layer_idx);
|
||||
return mid_layer != 0;
|
||||
}
|
||||
|
||||
bool fuseGraph(Ptr<Graph>& graph)
|
||||
{
|
||||
vector<Arg> removed_args;
|
||||
bool modified = false;
|
||||
const std::vector<Ptr<Layer> >& prog = graph->prog();
|
||||
size_t i, nargs = netimpl->args.size(), nops = prog.size();
|
||||
std::vector<int> producer_of(nargs, -1);
|
||||
std::vector<Ptr<Layer> > newprog;
|
||||
std::vector<Arg> fused_inputs;
|
||||
|
||||
for (i = 0; i < nops; i++) {
|
||||
const Ptr<Layer>& layer = prog[i];
|
||||
Layer* layer_ptr = (Layer*)layer.get();
|
||||
int fused_layer_idx = -1;
|
||||
std::vector<Ptr<Graph> >* subgraphs = layer->subgraphs();
|
||||
if (subgraphs) {
|
||||
for (Ptr<Graph>& g: *subgraphs) {
|
||||
if (fuseGraph(g))
|
||||
modified = true;
|
||||
}
|
||||
}
|
||||
const std::vector<Arg>& inputs = layer->inputs;
|
||||
const std::vector<Arg>& outputs = layer->outputs;
|
||||
size_t ninputs = inputs.size();
|
||||
removed_args.clear();
|
||||
fused_inputs.clear(); // leave it empty to re-use original fused node inputs as-is.
|
||||
|
||||
for(;;) {
|
||||
Arg q_data_in, out_scale, out_zp;
|
||||
int add_layer_idx = -1;
|
||||
NaryEltwiseLayer* add = 0;
|
||||
if (getQdqPatternContext<NaryEltwiseLayer>(layer_ptr, ninputs, inputs, producer_of,
|
||||
newprog, q_data_in, out_scale, out_zp,
|
||||
add_layer_idx, add) &&
|
||||
add && add->inputs.size() >= 2) {
|
||||
vector<DequantizeLinearLayer*> dq_ptrs;
|
||||
vector<int> dq_prog_indices;
|
||||
vector<Arg> int8_inputs;
|
||||
|
||||
for (size_t k = 0; k < add->inputs.size(); k++) {
|
||||
const Arg& add_inp = add->inputs[k];
|
||||
int dq_idx = producer_of.at(add_inp.idx);
|
||||
DequantizeLinearLayer* dq =
|
||||
getLayer<DequantizeLinearLayer>(newprog, dq_idx);
|
||||
|
||||
if (!dq || dq->inputs.size() < 3 ||
|
||||
usecounts.at(add_inp.idx) != 1) {
|
||||
break;
|
||||
}
|
||||
dq_ptrs.push_back(dq);
|
||||
dq_prog_indices.push_back(dq_idx);
|
||||
int8_inputs.push_back(dq->inputs[0]); // int8 quantized tensor
|
||||
}
|
||||
|
||||
const int eltwise_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
|
||||
const bool eltwise_out_int8 = (eltwise_out_type == CV_8S || eltwise_out_type == CV_8U);
|
||||
if (dq_ptrs.size() == add->inputs.size() && eltwise_out_int8) {
|
||||
// Read per-input scale and zero-point from the DQ const args.
|
||||
vector<float> in_scales(2);
|
||||
vector<int> in_zps(2);
|
||||
bool eltwise_in_int8 = true;
|
||||
for (int k = 0; k < 2; k++) {
|
||||
int inp_type = netimpl->argData(dq_ptrs[k]->inputs[0]).type;
|
||||
eltwise_in_int8 = eltwise_in_int8 && (inp_type == CV_8S || inp_type == CV_8U);
|
||||
in_scales[k] = netimpl->argTensor(dq_ptrs[k]->inputs[1]).at<float>(0);
|
||||
const Mat& zp_m = netimpl->argTensor(dq_ptrs[k]->inputs[2]);
|
||||
in_zps[k] = zp_m.depth() == CV_8U
|
||||
? (int)zp_m.at<uint8_t>(0)
|
||||
: (int)zp_m.at<int8_t>(0);
|
||||
}
|
||||
if (!eltwise_in_int8)
|
||||
break;
|
||||
|
||||
float out_scale_val = netimpl->argTensor(out_scale).at<float>(0);
|
||||
const Mat& elt_out_zp_m = netimpl->argTensor(out_zp);
|
||||
int out_zp_val = elt_out_zp_m.depth() == CV_8U
|
||||
? (int)elt_out_zp_m.at<uint8_t>(0)
|
||||
: (int)elt_out_zp_m.at<int8_t>(0);
|
||||
|
||||
LayerParams eltwiseParams = makeLayerParamsFromOriginal(add, "EltwiseInt8");
|
||||
eltwiseParams.blobs.clear();
|
||||
Ptr<Layer> eltwiseInt8 = createFusedLayer(eltwiseParams);
|
||||
if (!eltwiseInt8.empty()) {
|
||||
auto* elt = dynamic_cast<EltwiseLayerInt8*>(eltwiseInt8.get());
|
||||
CV_Assert(elt);
|
||||
elt->scales = in_scales;
|
||||
elt->zeropoints = in_zps;
|
||||
elt->output_sc = out_scale_val;
|
||||
elt->output_zp = out_zp_val;
|
||||
fused_layer_idx = add_layer_idx;
|
||||
newprog[add_layer_idx] = eltwiseInt8;
|
||||
fused_inputs.swap(int8_inputs);
|
||||
removed_args.push_back(q_data_in); // float add_out
|
||||
for (const Arg& add_inp : add->inputs)
|
||||
removed_args.push_back(add_inp);
|
||||
|
||||
for (int dq_prog_idx : dq_prog_indices)
|
||||
newprog[dq_prog_idx] = Ptr<Layer>();
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int relu_layer_idx = -1;
|
||||
ReLULayer* relu = 0;
|
||||
if (getQdqPatternContext<ReLULayer>(layer_ptr, ninputs, inputs, producer_of,
|
||||
newprog, q_data_in, out_scale, out_zp,
|
||||
relu_layer_idx, relu) &&
|
||||
relu->inputs.size() == 1) {
|
||||
Arg relu_in = relu->inputs[0];
|
||||
int dq_idx = producer_of.at(relu_in.idx);
|
||||
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
|
||||
|
||||
const int relu_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
|
||||
const bool relu_out_int8 = (relu_out_type == CV_8S || relu_out_type == CV_8U);
|
||||
const int relu_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
|
||||
const bool relu_in_int8 = (relu_in_type == CV_8S || relu_in_type == CV_8U);
|
||||
if (dq && dq->inputs.size() >= 3 &&
|
||||
relu_in_int8 && relu_out_int8 &&
|
||||
usecounts.at(relu_in.idx) == 1) {
|
||||
const float inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
|
||||
const Mat& relu_zp_m = netimpl->argTensor(dq->inputs[2]);
|
||||
const int inp_zp = relu_zp_m.depth() == CV_8U
|
||||
? (int)relu_zp_m.at<uint8_t>(0)
|
||||
: (int)relu_zp_m.at<int8_t>(0);
|
||||
const float out_sc = netimpl->argTensor(out_scale).at<float>(0);
|
||||
const Mat& out_zp_relu_m = netimpl->argTensor(out_zp);
|
||||
const int out_zp_i = out_zp_relu_m.depth() == CV_8U
|
||||
? (int)out_zp_relu_m.at<uint8_t>(0)
|
||||
: (int)out_zp_relu_m.at<int8_t>(0);
|
||||
|
||||
if (inp_sc > 0.f && out_sc > 0.f) {
|
||||
const bool isU8 = (relu_in_type == CV_8U);
|
||||
Mat lookUpTable(1, 256, isU8 ? CV_8U : CV_8S);
|
||||
if (isU8) {
|
||||
uint8_t* table = lookUpTable.ptr<uint8_t>();
|
||||
for (int t = 0; t < 256; t++) {
|
||||
float x = inp_sc * (t - inp_zp);
|
||||
float y = std::max(0.0f, x);
|
||||
int quantized = out_zp_i + cvRound(y / out_sc);
|
||||
table[t] = saturate_cast<uint8_t>(quantized);
|
||||
}
|
||||
} else {
|
||||
int8_t* table = lookUpTable.ptr<int8_t>();
|
||||
for (int t = -128; t < 128; t++) {
|
||||
float x = inp_sc * (t - inp_zp);
|
||||
float y = std::max(0.0f, x);
|
||||
int quantized = out_zp_i + cvRound(y / out_sc);
|
||||
table[t + 128] = saturate_cast<int8_t>(quantized);
|
||||
}
|
||||
}
|
||||
|
||||
LayerParams reluInt8Params = makeLayerParamsFromOriginal(relu, "ReLUInt8");
|
||||
reluInt8Params.blobs.clear();
|
||||
Ptr<Layer> reluInt8 = createFusedLayer(reluInt8Params);
|
||||
if (!reluInt8.empty()) {
|
||||
auto* reluInt8Layer = dynamic_cast<ActivationLayerInt8*>(reluInt8.get());
|
||||
CV_Assert(reluInt8Layer);
|
||||
reluInt8Layer->input_sc = inp_sc;
|
||||
reluInt8Layer->input_zp = inp_zp;
|
||||
reluInt8Layer->output_sc = out_sc;
|
||||
reluInt8Layer->output_zp = out_zp_i;
|
||||
reluInt8Layer->activationLUT = lookUpTable;
|
||||
fused_layer_idx = relu_layer_idx;
|
||||
newprog[relu_layer_idx] = reluInt8;
|
||||
fused_inputs.assign(1, dq->inputs[0]);
|
||||
removed_args.push_back(q_data_in);
|
||||
removed_args.push_back(relu_in);
|
||||
newprog[dq_idx] = Ptr<Layer>();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compound pattern: DQ, DQ -> Add -> ReLU -> QuantizeLinear
|
||||
// Common in ResNet residual blocks. Fuses into EltwiseInt8 with activation LUT.
|
||||
{
|
||||
int relu_layer_idx2 = -1;
|
||||
ReLULayer* relu2 = 0;
|
||||
Arg q_data_in2, out_scale2, out_zp2;
|
||||
if (getQdqPatternContext<ReLULayer>(layer_ptr, ninputs, inputs, producer_of,
|
||||
newprog, q_data_in2, out_scale2, out_zp2,
|
||||
relu_layer_idx2, relu2) &&
|
||||
relu2->inputs.size() == 1) {
|
||||
Arg relu_in2 = relu2->inputs[0];
|
||||
int add_idx2 = producer_of.at(relu_in2.idx);
|
||||
NaryEltwiseLayer* add2 = getLayer<NaryEltwiseLayer>(newprog, add_idx2);
|
||||
if (add2 && add2->inputs.size() >= 2 &&
|
||||
usecounts.at(relu_in2.idx) == 1) {
|
||||
vector<DequantizeLinearLayer*> dq_ptrs2;
|
||||
vector<int> dq_prog_indices2;
|
||||
vector<Arg> int8_inputs2;
|
||||
for (size_t k = 0; k < add2->inputs.size(); k++) {
|
||||
const Arg& add_inp = add2->inputs[k];
|
||||
int dq_idx2 = producer_of.at(add_inp.idx);
|
||||
DequantizeLinearLayer* dq2 =
|
||||
getLayer<DequantizeLinearLayer>(newprog, dq_idx2);
|
||||
if (dq2 && dq2->inputs.size() >= 3 &&
|
||||
usecounts.at(add_inp.idx) == 1) {
|
||||
dq_ptrs2.push_back(dq2);
|
||||
dq_prog_indices2.push_back(dq_idx2);
|
||||
int8_inputs2.push_back(dq2->inputs[0]);
|
||||
} else {
|
||||
int arg_type = netimpl->argData(add_inp).type;
|
||||
if (arg_type == CV_8S || arg_type == CV_8U) {
|
||||
dq_ptrs2.push_back(nullptr);
|
||||
dq_prog_indices2.push_back(-1);
|
||||
int8_inputs2.push_back(add_inp);
|
||||
} else {
|
||||
break; // not int8, can't fuse
|
||||
}
|
||||
}
|
||||
}
|
||||
const int elt_out_type2 = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
|
||||
const bool elt_out_int82 = (elt_out_type2 == CV_8S || elt_out_type2 == CV_8U);
|
||||
if (int8_inputs2.size() == add2->inputs.size() && elt_out_int82) {
|
||||
vector<float> in_scales2(2);
|
||||
vector<int> in_zps2(2);
|
||||
bool elt_in_int82 = true;
|
||||
for (int k = 0; k < 2; k++) {
|
||||
if (dq_ptrs2[k]) {
|
||||
int it = netimpl->argData(dq_ptrs2[k]->inputs[0]).type;
|
||||
elt_in_int82 = elt_in_int82 && (it == CV_8S || it == CV_8U);
|
||||
in_scales2[k] = netimpl->argTensor(dq_ptrs2[k]->inputs[1]).at<float>(0);
|
||||
const Mat& zp_m2 = netimpl->argTensor(dq_ptrs2[k]->inputs[2]);
|
||||
in_zps2[k] = zp_m2.depth() == CV_8U
|
||||
? (int)zp_m2.at<uint8_t>(0)
|
||||
: (int)zp_m2.at<int8_t>(0);
|
||||
} else {
|
||||
int prod_idx = producer_of.at(add2->inputs[k].idx);
|
||||
Layer* prod = prod_idx >= 0 && !newprog[prod_idx].empty()
|
||||
? newprog[prod_idx].get() : nullptr;
|
||||
ConvolutionLayerInt8* ci = prod ? dynamic_cast<ConvolutionLayerInt8*>(prod) : nullptr;
|
||||
EltwiseLayerInt8* ei = prod ? dynamic_cast<EltwiseLayerInt8*>(prod) : nullptr;
|
||||
InnerProductLayerInt8* fi = prod ? dynamic_cast<InnerProductLayerInt8*>(prod) : nullptr;
|
||||
if (ci) { in_scales2[k] = ci->output_sc; in_zps2[k] = ci->output_zp; }
|
||||
else if (ei) { in_scales2[k] = ei->output_sc; in_zps2[k] = ei->output_zp; }
|
||||
else if (fi) { in_scales2[k] = fi->output_sc; in_zps2[k] = fi->output_zp; }
|
||||
else { elt_in_int82 = false; }
|
||||
}
|
||||
}
|
||||
if (elt_in_int82) {
|
||||
float out_sc2 = netimpl->argTensor(out_scale2).at<float>(0);
|
||||
const Mat& out_zp_m2 = netimpl->argTensor(out_zp2);
|
||||
int out_zp_val2 = out_zp_m2.depth() == CV_8U
|
||||
? (int)out_zp_m2.at<uint8_t>(0)
|
||||
: (int)out_zp_m2.at<int8_t>(0);
|
||||
if (out_sc2 > 0.f) {
|
||||
LayerParams eltParams = makeLayerParamsFromOriginal(add2, "EltwiseInt8");
|
||||
eltParams.blobs.clear();
|
||||
Ptr<Layer> eltInt8 = createFusedLayer(eltParams);
|
||||
if (!eltInt8.empty()) {
|
||||
auto* elt = dynamic_cast<EltwiseLayerInt8*>(eltInt8.get());
|
||||
CV_Assert(elt);
|
||||
elt->scales = in_scales2;
|
||||
elt->zeropoints = in_zps2;
|
||||
elt->output_sc = out_sc2;
|
||||
elt->output_zp = out_zp_val2;
|
||||
|
||||
const bool isU8 = (elt_out_type2 == CV_8U);
|
||||
Mat lut(1, 256, isU8 ? CV_8U : CV_8S);
|
||||
if (isU8) {
|
||||
uint8_t* tbl = lut.ptr<uint8_t>();
|
||||
for (int t = 0; t < 256; t++) {
|
||||
float x = out_sc2 * (t - out_zp_val2);
|
||||
float y = std::max(0.0f, x);
|
||||
tbl[t] = saturate_cast<uint8_t>(out_zp_val2 + cvRound(y / out_sc2));
|
||||
}
|
||||
} else {
|
||||
int8_t* tbl = lut.ptr<int8_t>();
|
||||
for (int t = -128; t < 128; t++) {
|
||||
float x = out_sc2 * (t - out_zp_val2);
|
||||
float y = std::max(0.0f, x);
|
||||
tbl[t + 128] = saturate_cast<int8_t>(out_zp_val2 + cvRound(y / out_sc2));
|
||||
}
|
||||
}
|
||||
LayerParams reluActParams;
|
||||
reluActParams.name = relu2->name;
|
||||
reluActParams.type = "ReLUInt8";
|
||||
Ptr<Layer> reluAct = createFusedLayer(reluActParams);
|
||||
if (!reluAct.empty()) {
|
||||
auto* reluActLayer = dynamic_cast<ActivationLayerInt8*>(reluAct.get());
|
||||
if (reluActLayer) {
|
||||
reluActLayer->input_sc = out_sc2;
|
||||
reluActLayer->input_zp = out_zp_val2;
|
||||
reluActLayer->output_sc = out_sc2;
|
||||
reluActLayer->output_zp = out_zp_val2;
|
||||
reluActLayer->activationLUT = lut;
|
||||
eltInt8->setActivation(reluAct.dynamicCast<ActivationLayer>());
|
||||
}
|
||||
}
|
||||
|
||||
fused_layer_idx = add_idx2;
|
||||
newprog[add_idx2] = eltInt8;
|
||||
newprog[relu_layer_idx2] = Ptr<Layer>();
|
||||
fused_inputs.swap(int8_inputs2);
|
||||
removed_args.push_back(q_data_in2);
|
||||
removed_args.push_back(relu_in2);
|
||||
for (const Arg& add_inp : add2->inputs)
|
||||
removed_args.push_back(add_inp);
|
||||
for (int dq_prog_idx : dq_prog_indices2) {
|
||||
if (dq_prog_idx >= 0)
|
||||
newprog[dq_prog_idx] = Ptr<Layer>();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Arg out_scale_arg, out_zp_arg;
|
||||
int conv_layer_idx = -1;
|
||||
Conv2Layer* conv = 0;
|
||||
if (getQdqPatternContext<Conv2Layer>(layer_ptr, ninputs, inputs, producer_of,
|
||||
newprog, q_data_in, out_scale_arg, out_zp_arg,
|
||||
conv_layer_idx, conv) &&
|
||||
(conv->inputs.size() == 2 || conv->inputs.size() == 3)) {
|
||||
const Arg conv_x = conv->inputs[0];
|
||||
const Arg conv_w = conv->inputs[1];
|
||||
const int dq_x_idx = producer_of.at(conv_x.idx);
|
||||
const int dq_w_idx = producer_of.at(conv_w.idx);
|
||||
DequantizeLinearLayer* dq_x = getLayer<DequantizeLinearLayer>(newprog, dq_x_idx);
|
||||
DequantizeLinearLayer* dq_w = getLayer<DequantizeLinearLayer>(newprog, dq_w_idx);
|
||||
|
||||
// Allow usecounts > 1 for conv input (shared DQ output at stage transitions)
|
||||
// The int8 data (DQ's input[0]) can be shared safely.
|
||||
if (dq_x && dq_w &&
|
||||
dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 &&
|
||||
usecounts.at(conv_w.idx) == 1) {
|
||||
float inp_sc = netimpl->argTensor(dq_x->inputs[1]).at<float>(0);
|
||||
float out_sc = netimpl->argTensor(out_scale_arg).at<float>(0);
|
||||
int inp_zp = 0, out_zp = 0;
|
||||
const Mat& x_zp_m_read = netimpl->argTensor(dq_x->inputs[2]);
|
||||
if (x_zp_m_read.depth() == CV_8S)
|
||||
inp_zp = (int)x_zp_m_read.at<int8_t>(0);
|
||||
else
|
||||
inp_zp = (int)x_zp_m_read.at<uint8_t>(0);
|
||||
const Mat& out_zp_m = netimpl->argTensor(out_zp_arg);
|
||||
if (out_zp_m.depth() == CV_8S)
|
||||
out_zp = (int)out_zp_m.at<int8_t>(0);
|
||||
else if (out_zp_m.depth() == CV_8U)
|
||||
out_zp = (int)out_zp_m.at<uint8_t>(0);
|
||||
else
|
||||
out_zp = out_zp_m.at<int>(0);
|
||||
if (!(inp_sc > 0.f && out_sc > 0.f))
|
||||
break;
|
||||
Mat w_q = netimpl->argTensor(dq_w->inputs[0]);
|
||||
Mat w_sc_m = netimpl->argTensor(dq_w->inputs[1]);
|
||||
Mat w_zp_m = netimpl->argTensor(dq_w->inputs[2]);
|
||||
const Mat& x_zp_m = netimpl->argTensor(dq_x->inputs[2]);
|
||||
if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims >= 3) {
|
||||
const int outCn = w_q.size[0];
|
||||
Mat wt_sc = (w_sc_m.total() == (size_t)outCn)
|
||||
? w_sc_m.reshape(1, 1)
|
||||
: Mat(1, outCn, CV_32F, Scalar(w_sc_m.at<float>(0))).clone();
|
||||
bool per_channel = w_sc_m.total() == (size_t)outCn;
|
||||
|
||||
bool all_wzp_zero = true;
|
||||
if (w_zp_m.total() > 1 && w_zp_m.total() != (size_t)outCn)
|
||||
all_wzp_zero = false;
|
||||
for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) {
|
||||
int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at<int8_t>((int)t)
|
||||
: (int)w_zp_m.at<uint8_t>((int)t);
|
||||
if (wz != 0) all_wzp_zero = false;
|
||||
}
|
||||
|
||||
bool symmetric_pads = true;
|
||||
size_t npads = conv->pads.size();
|
||||
size_t ndims_pad = npads / 2;
|
||||
for (size_t d = 0; d < ndims_pad && symmetric_pads; d++) {
|
||||
if (conv->pads[d] != conv->pads[d + ndims_pad])
|
||||
symmetric_pads = false;
|
||||
}
|
||||
|
||||
if (all_wzp_zero && symmetric_pads) {
|
||||
Mat bias = Mat::zeros(1, outCn, CV_32S);
|
||||
bool biasOk = true;
|
||||
int dq_bias_idx = -1;
|
||||
if (conv->inputs.size() == 3) {
|
||||
if (netimpl->isConstArg(conv->inputs[2])) {
|
||||
Mat b = netimpl->argTensor(conv->inputs[2]);
|
||||
if (b.empty() || b.total() != (size_t)outCn) {
|
||||
biasOk = false;
|
||||
} else if (b.depth() == CV_32S) {
|
||||
bias = b.reshape(1, 1);
|
||||
} else if (b.depth() == CV_32F || b.depth() == CV_64F) {
|
||||
Mat b1 = b.reshape(1, 1);
|
||||
for (int oc = 0; oc < outCn; oc++) {
|
||||
const float b_real = b1.depth() == CV_32F
|
||||
? b1.at<float>(oc)
|
||||
: (float)b1.at<double>(oc);
|
||||
const float denom = inp_sc * wt_sc.at<float>(oc);
|
||||
if (std::abs(denom) < 1e-12f) { biasOk = false; break; }
|
||||
bias.at<int>(oc) = cvRound(b_real / denom);
|
||||
}
|
||||
} else {
|
||||
biasOk = false;
|
||||
}
|
||||
} else {
|
||||
dq_bias_idx = producer_of.at(conv->inputs[2].idx);
|
||||
DequantizeLinearLayer* dq_b =
|
||||
getLayer<DequantizeLinearLayer>(newprog, dq_bias_idx);
|
||||
if (!dq_b || dq_b->inputs.size() < 2 ||
|
||||
usecounts.at(conv->inputs[2].idx) != 1 ||
|
||||
!netimpl->isConstArg(dq_b->inputs[0])) {
|
||||
biasOk = false;
|
||||
} else {
|
||||
Mat bq = netimpl->argTensor(dq_b->inputs[0]);
|
||||
if (bq.empty() || bq.total() != (size_t)outCn || bq.depth() != CV_32S)
|
||||
biasOk = false;
|
||||
else
|
||||
bias = bq.reshape(1, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!biasOk)
|
||||
break;
|
||||
|
||||
const bool inputIsU8 =
|
||||
(netimpl->argData(dq_x->inputs[0]).type == CV_8U) ||
|
||||
(x_zp_m.depth() == CV_8U);
|
||||
const int inp_zp_kernel = inputIsU8 ? (inp_zp - 128) : inp_zp;
|
||||
Mat weights_2d = w_q.reshape(1, outCn);
|
||||
Mat biasFused(1, outCn, CV_32S);
|
||||
Mat outputMultiplier(1, outCn, CV_32F);
|
||||
for (int oc = 0; oc < outCn; oc++) {
|
||||
biasFused.at<int>(oc) = bias.at<int>(oc) - inp_zp_kernel * (int)cv::sum(weights_2d.row(oc))[0];
|
||||
outputMultiplier.at<float>(oc) = (inp_sc * wt_sc.at<float>(oc)) / out_sc;
|
||||
}
|
||||
|
||||
LayerParams convInt8Params = makeLayerParamsFromOriginal(conv, "ConvolutionInt8");
|
||||
{
|
||||
int kndims = w_q.dims - 2;
|
||||
std::vector<int> ksize(kndims);
|
||||
for (int d = 0; d < kndims; d++)
|
||||
ksize[d] = w_q.size[d + 2];
|
||||
convInt8Params.set("kernel_size", DictValue::arrayInt(ksize.data(), kndims));
|
||||
if (!conv->strides.empty())
|
||||
convInt8Params.set("stride", DictValue::arrayInt(conv->strides.data(), (int)conv->strides.size()));
|
||||
if (!conv->dilations.empty())
|
||||
convInt8Params.set("dilation", DictValue::arrayInt(conv->dilations.data(), (int)conv->dilations.size()));
|
||||
if (!conv->pads.empty())
|
||||
convInt8Params.set("pad", DictValue::arrayInt(conv->pads.data(), (int)conv->pads.size()));
|
||||
}
|
||||
convInt8Params.set("num_output", outCn);
|
||||
convInt8Params.set("group", conv->ngroups);
|
||||
convInt8Params.blobs.resize(3);
|
||||
convInt8Params.blobs[0] = w_q;
|
||||
convInt8Params.blobs[1] = biasFused;
|
||||
convInt8Params.blobs[2] = outputMultiplier;
|
||||
Ptr<Layer> convInt8 = createFusedLayer(convInt8Params);
|
||||
if (!convInt8.empty()) {
|
||||
auto* convInt8Layer = dynamic_cast<ConvolutionLayerInt8*>(convInt8.get());
|
||||
CV_Assert(convInt8Layer);
|
||||
convInt8Layer->input_zp = inp_zp;
|
||||
convInt8Layer->input_sc = inp_sc;
|
||||
convInt8Layer->output_zp = out_zp;
|
||||
convInt8Layer->output_sc = out_sc;
|
||||
convInt8Layer->per_channel = per_channel;
|
||||
fused_layer_idx = conv_layer_idx;
|
||||
newprog[conv_layer_idx] = convInt8;
|
||||
fused_inputs.assign(1, dq_x->inputs[0]);
|
||||
removed_args.push_back(q_data_in);
|
||||
removed_args.push_back(conv_w);
|
||||
if (conv->inputs.size() == 3) {
|
||||
removed_args.push_back(conv->inputs[2]);
|
||||
if (dq_bias_idx >= 0)
|
||||
newprog[dq_bias_idx] = Ptr<Layer>();
|
||||
}
|
||||
if (usecounts.at(conv_x.idx) == 1) {
|
||||
removed_args.push_back(conv_x);
|
||||
newprog[dq_x_idx] = Ptr<Layer>();
|
||||
}
|
||||
newprog[dq_w_idx] = Ptr<Layer>();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int mm_layer_idx = -1;
|
||||
MatMulLayer* mm = 0;
|
||||
if (getQdqPatternContext<MatMulLayer>(layer_ptr, ninputs, inputs, producer_of,
|
||||
newprog, q_data_in, out_scale, out_zp,
|
||||
mm_layer_idx, mm) &&
|
||||
mm->inputs.size() == 2) {
|
||||
const Arg mm_x = mm->inputs[0];
|
||||
const Arg mm_w = mm->inputs[1];
|
||||
int dq_x_idx = producer_of.at(mm_x.idx);
|
||||
int dq_w_idx = producer_of.at(mm_w.idx);
|
||||
DequantizeLinearLayer* dq_x = getLayer<DequantizeLinearLayer>(newprog, dq_x_idx);
|
||||
DequantizeLinearLayer* dq_w = getLayer<DequantizeLinearLayer>(newprog, dq_w_idx);
|
||||
float inp_sc = 0.f, out_sc = 0.f;
|
||||
int inp_zp = 0, out_zp_i = 0;
|
||||
const int fc_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
|
||||
const bool fc_out_int8 = (fc_out_type == CV_8S || fc_out_type == CV_8U);
|
||||
const int fc_in_type = (dq_x && !dq_x->inputs.empty()) ? netimpl->argData(dq_x->inputs[0]).type : -1;
|
||||
const bool fc_in_int8 = (fc_in_type == CV_8S || fc_in_type == CV_8U);
|
||||
if (dq_x && dq_w &&
|
||||
dq_x->inputs.size() >= 3 && dq_w->inputs.size() >= 3 &&
|
||||
fc_in_int8 && fc_out_int8 &&
|
||||
usecounts.at(mm_x.idx) == 1 && usecounts.at(mm_w.idx) == 1) {
|
||||
inp_sc = netimpl->argTensor(dq_x->inputs[1]).at<float>(0);
|
||||
const Mat& fc_zp_m = netimpl->argTensor(dq_x->inputs[2]);
|
||||
inp_zp = fc_zp_m.depth() == CV_8U
|
||||
? (int)fc_zp_m.at<uint8_t>(0)
|
||||
: (int)fc_zp_m.at<int8_t>(0);
|
||||
out_sc = netimpl->argTensor(out_scale).at<float>(0);
|
||||
const Mat& fc_out_zp_m = netimpl->argTensor(out_zp);
|
||||
out_zp_i = fc_out_zp_m.depth() == CV_8U
|
||||
? (int)fc_out_zp_m.at<uint8_t>(0)
|
||||
: (int)fc_out_zp_m.at<int8_t>(0);
|
||||
if (!(inp_sc > 0.f && out_sc > 0.f))
|
||||
break;
|
||||
Mat w_q = netimpl->argTensor(dq_w->inputs[0]);
|
||||
Mat w_sc_m = netimpl->argTensor(dq_w->inputs[1]);
|
||||
Mat w_zp_m = netimpl->argTensor(dq_w->inputs[2]);
|
||||
if (!w_q.empty() && w_q.depth() == CV_8S && w_q.dims == 2) {
|
||||
bool all_wzp_zero = true;
|
||||
for (size_t t = 0; all_wzp_zero && t < w_zp_m.total(); t++) {
|
||||
int wz = w_zp_m.depth() == CV_8S ? (int)w_zp_m.at<int8_t>((int)t)
|
||||
: (int)w_zp_m.at<uint8_t>((int)t);
|
||||
if (wz != 0) all_wzp_zero = false;
|
||||
}
|
||||
if (all_wzp_zero) {
|
||||
Mat weights = w_q.t();
|
||||
int outCn = weights.size[0];
|
||||
Mat wt_sc = (w_sc_m.total() == (size_t)outCn)
|
||||
? w_sc_m.reshape(1, 1)
|
||||
: Mat(1, outCn, CV_32F, Scalar(w_sc_m.at<float>(0))).clone();
|
||||
bool per_channel = w_sc_m.total() == (size_t)outCn;
|
||||
Mat bias(1, outCn, CV_32S);
|
||||
Mat outputMultiplier(1, outCn, CV_32F);
|
||||
for (int ioc = 0; ioc < outCn; ioc++) {
|
||||
bias.at<int>(ioc) = -inp_zp * (int)cv::sum(weights.row(ioc))[0];
|
||||
outputMultiplier.at<float>(ioc) = (inp_sc * wt_sc.at<float>(ioc)) / out_sc;
|
||||
}
|
||||
int firstInpDims = (int)netimpl->argData(mm_x).shape.size();
|
||||
int axis = std::max(1, firstInpDims - w_q.dims + 1);
|
||||
|
||||
LayerParams fcInt8Params = makeLayerParamsFromOriginal(mm, "InnerProductInt8");
|
||||
fcInt8Params.set("num_output", outCn);
|
||||
fcInt8Params.set("axis", axis);
|
||||
fcInt8Params.blobs.resize(3);
|
||||
fcInt8Params.blobs[0] = weights;
|
||||
fcInt8Params.blobs[1] = bias;
|
||||
fcInt8Params.blobs[2] = outputMultiplier;
|
||||
Ptr<Layer> fcInt8 = createFusedLayer(fcInt8Params);
|
||||
if (!fcInt8.empty()) {
|
||||
auto* fcInt8Layer = dynamic_cast<InnerProductLayerInt8*>(fcInt8.get());
|
||||
CV_Assert(fcInt8Layer);
|
||||
fcInt8Layer->input_zp = inp_zp;
|
||||
fcInt8Layer->input_sc = inp_sc;
|
||||
fcInt8Layer->output_zp = out_zp_i;
|
||||
fcInt8Layer->output_sc = out_sc;
|
||||
fcInt8Layer->per_channel = per_channel;
|
||||
fused_layer_idx = mm_layer_idx;
|
||||
newprog[mm_layer_idx] = fcInt8;
|
||||
fused_inputs.assign(1, dq_x->inputs[0]);
|
||||
removed_args.push_back(q_data_in);
|
||||
removed_args.push_back(mm_x);
|
||||
removed_args.push_back(mm_w);
|
||||
newprog[dq_x_idx] = Ptr<Layer>();
|
||||
newprog[dq_w_idx] = Ptr<Layer>();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int pool_layer_idx = -1;
|
||||
PoolingLayer* pool = 0;
|
||||
if (getQdqPatternContext<PoolingLayer>(layer_ptr, ninputs, inputs, producer_of,
|
||||
newprog, q_data_in, out_scale, out_zp,
|
||||
pool_layer_idx, pool) &&
|
||||
pool->inputs.size() == 1) {
|
||||
Arg pool_in = pool->inputs[0];
|
||||
int dq_idx = producer_of.at(pool_in.idx);
|
||||
DequantizeLinearLayer* dq = getLayer<DequantizeLinearLayer>(newprog, dq_idx);
|
||||
float inp_sc = 0.f, out_sc = 0.f;
|
||||
int inp_zp = 0, out_zp_i = 0;
|
||||
const int pool_out_type = !outputs.empty() ? netimpl->argData(outputs[0]).type : -1;
|
||||
const bool pool_out_int8 = (pool_out_type == CV_8S || pool_out_type == CV_8U);
|
||||
const int pool_in_type = (dq && !dq->inputs.empty()) ? netimpl->argData(dq->inputs[0]).type : -1;
|
||||
const bool pool_in_int8 = (pool_in_type == CV_8S || pool_in_type == CV_8U);
|
||||
if (dq && dq->inputs.size() >= 3 &&
|
||||
pool_in_int8 && pool_out_int8 &&
|
||||
usecounts.at(pool_in.idx) == 1) {
|
||||
inp_sc = netimpl->argTensor(dq->inputs[1]).at<float>(0);
|
||||
const Mat& pool_zp_m = netimpl->argTensor(dq->inputs[2]);
|
||||
inp_zp = pool_zp_m.depth() == CV_8U
|
||||
? (int)pool_zp_m.at<uint8_t>(0)
|
||||
: (int)pool_zp_m.at<int8_t>(0);
|
||||
out_sc = netimpl->argTensor(out_scale).at<float>(0);
|
||||
const Mat& pool_out_zp_m = netimpl->argTensor(out_zp);
|
||||
out_zp_i = pool_out_zp_m.depth() == CV_8U
|
||||
? (int)pool_out_zp_m.at<uint8_t>(0)
|
||||
: (int)pool_out_zp_m.at<int8_t>(0);
|
||||
bool isGlobalAve = pool->globalPooling;
|
||||
bool isMax = !isGlobalAve;
|
||||
if ((isGlobalAve && inp_sc > 0.f && out_sc > 0.f) ||
|
||||
(isMax && std::abs(inp_sc - out_sc) < 1e-6f && inp_zp == out_zp_i)) {
|
||||
LayerParams poolInt8Params = makeLayerParamsFromOriginal(pool, "PoolingInt8");
|
||||
poolInt8Params.blobs.clear();
|
||||
Ptr<Layer> poolInt8 = createFusedLayer(poolInt8Params);
|
||||
if (!poolInt8.empty()) {
|
||||
auto* poolInt8Layer = dynamic_cast<PoolingLayerInt8*>(poolInt8.get());
|
||||
CV_Assert(poolInt8Layer);
|
||||
const String poolInt8Type = static_cast<Layer&>(*poolInt8Layer).type;
|
||||
std::vector<Mat> poolInt8Blobs = poolInt8Layer->blobs;
|
||||
static_cast<PoolingLayer&>(*poolInt8Layer) = *pool;
|
||||
static_cast<Layer&>(*poolInt8Layer).type = poolInt8Type;
|
||||
poolInt8Layer->blobs = poolInt8Blobs;
|
||||
poolInt8Layer->input_sc = inp_sc;
|
||||
poolInt8Layer->input_zp = inp_zp;
|
||||
poolInt8Layer->output_sc = out_sc;
|
||||
poolInt8Layer->output_zp = out_zp_i;
|
||||
fused_layer_idx = pool_layer_idx;
|
||||
newprog[pool_layer_idx] = poolInt8;
|
||||
fused_inputs.assign(1, dq->inputs[0]);
|
||||
removed_args.push_back(q_data_in);
|
||||
removed_args.push_back(pool_in);
|
||||
newprog[dq_idx] = Ptr<Layer>();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
ActivationLayerInt8* activ_int8 = dynamic_cast<ActivationLayerInt8*>(layer_ptr);
|
||||
if (activ_int8 && ninputs == 1 &&
|
||||
usecounts.at(inputs[0].idx) == 1) {
|
||||
Arg activ_inp = inputs[0];
|
||||
int producer_idx = producer_of.at(activ_inp.idx);
|
||||
if (producer_idx >= 0 && !newprog[producer_idx].empty()) {
|
||||
Layer* producer_layer = newprog[producer_idx].get();
|
||||
Ptr<ActivationLayer> activ_layer = layer.dynamicCast<ActivationLayer>();
|
||||
|
||||
ConvolutionLayerInt8* conv_int8 =
|
||||
dynamic_cast<ConvolutionLayerInt8*>(producer_layer);
|
||||
if (conv_int8 && conv_int8->output_sc == activ_int8->input_sc &&
|
||||
conv_int8->output_zp == activ_int8->input_zp) {
|
||||
if (newprog[producer_idx]->setActivation(activ_layer)) {
|
||||
conv_int8->output_sc = activ_int8->output_sc;
|
||||
conv_int8->output_zp = activ_int8->output_zp;
|
||||
fused_layer_idx = producer_idx;
|
||||
removed_args.push_back(activ_inp);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
InnerProductLayerInt8* fc_int8 =
|
||||
dynamic_cast<InnerProductLayerInt8*>(producer_layer);
|
||||
if (fc_int8 && fc_int8->output_sc == activ_int8->input_sc &&
|
||||
fc_int8->output_zp == activ_int8->input_zp) {
|
||||
if (newprog[producer_idx]->setActivation(activ_layer)) {
|
||||
fc_int8->output_sc = activ_int8->output_sc;
|
||||
fc_int8->output_zp = activ_int8->output_zp;
|
||||
fused_layer_idx = producer_idx;
|
||||
removed_args.push_back(activ_inp);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
EltwiseLayerInt8* elt_int8 =
|
||||
dynamic_cast<EltwiseLayerInt8*>(producer_layer);
|
||||
if (elt_int8 && elt_int8->output_sc == activ_int8->input_sc &&
|
||||
elt_int8->output_zp == activ_int8->input_zp) {
|
||||
if (newprog[producer_idx]->setActivation(activ_layer)) {
|
||||
elt_int8->output_sc = activ_int8->output_sc;
|
||||
elt_int8->output_zp = activ_int8->output_zp;
|
||||
fused_layer_idx = producer_idx;
|
||||
removed_args.push_back(activ_inp);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (fused_layer_idx >= 0) {
|
||||
modified = true;
|
||||
Layer* fused_layer = newprog[fused_layer_idx];
|
||||
fused_layer->outputs = outputs;
|
||||
if (!fused_inputs.empty())
|
||||
fused_layer->inputs = fused_inputs;
|
||||
for (Arg new_out: outputs)
|
||||
producer_of[new_out.idx] = fused_layer_idx;
|
||||
for (Arg old_out: removed_args) {
|
||||
usecounts.at(old_out.idx) = 0;
|
||||
producer_of.at(old_out.idx) = -1;
|
||||
}
|
||||
} else {
|
||||
for (auto out: outputs)
|
||||
producer_of[out.idx] = (int)newprog.size();
|
||||
newprog.push_back(layer);
|
||||
}
|
||||
}
|
||||
|
||||
if (modified) {
|
||||
size_t i, j = 0, newops = newprog.size();
|
||||
for (i = 0; i < newops; i++) {
|
||||
if (!newprog[i].empty()) {
|
||||
if (j < i)
|
||||
newprog[j] = newprog[i];
|
||||
j++;
|
||||
}
|
||||
}
|
||||
newprog.resize(j);
|
||||
graph->setProg(newprog);
|
||||
}
|
||||
|
||||
return modified;
|
||||
}
|
||||
|
||||
Net::Impl* netimpl;
|
||||
vector<int> usecounts;
|
||||
};
|
||||
|
||||
void Net::Impl::fuseQDQ()
|
||||
{
|
||||
ModelFusionQDQ qdqFusion(this);
|
||||
qdqFusion.fuse();
|
||||
}
|
||||
|
||||
CV__DNN_INLINE_NS_END
|
||||
}} // cv::dnn
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
|
||||
|
||||
namespace cv
|
||||
{
|
||||
namespace dnn
|
||||
@@ -42,16 +43,32 @@ public:
|
||||
BaseConvolutionLayerInt8Impl(const LayerParams ¶ms)
|
||||
{
|
||||
setParamsFrom(params);
|
||||
getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads, useWinograd);
|
||||
const bool hasKernelParams =
|
||||
params.has("kernel") || params.has("kernel_size") || params.has("kernel_h") || params.has("kernel_w");
|
||||
if (hasKernelParams)
|
||||
{
|
||||
getConvolutionKernelParams(params, kernel_size, pads_begin, pads_end, strides, dilations, padMode, adjust_pads, useWinograd);
|
||||
}
|
||||
else
|
||||
{
|
||||
kernel_size.clear();
|
||||
pads_begin.clear();
|
||||
pads_end.clear();
|
||||
strides.clear();
|
||||
dilations.clear();
|
||||
adjust_pads.clear();
|
||||
padMode = params.get<String>("pad_mode", "");
|
||||
}
|
||||
|
||||
numOutput = params.get<int>("num_output");
|
||||
numOutput = params.get<int>("num_output", blobs.empty() ? 0 : blobs[0].size[0]);
|
||||
int ngroups = params.get<int>("group", 1);
|
||||
CV_Assert(numOutput % ngroups == 0);
|
||||
if (numOutput > 0)
|
||||
CV_Assert(numOutput % ngroups == 0);
|
||||
|
||||
input_sc = params.get<float>("input_scale");
|
||||
input_zp = params.get<int>("input_zeropoint");
|
||||
output_zp = params.get<int>("zeropoints");
|
||||
output_sc = params.get<float>("scales");
|
||||
input_sc = params.get<float>("input_scale", 1.0f);
|
||||
input_zp = params.get<int>("input_zeropoint", 0);
|
||||
output_zp = params.get<int>("zeropoints", 0);
|
||||
output_sc = params.get<float>("scales", 1.0f);
|
||||
per_channel = params.get<bool>("per_channel", true);
|
||||
|
||||
if (kernel_size.size() == 2) {
|
||||
@@ -100,7 +117,8 @@ public:
|
||||
}
|
||||
|
||||
const Mat &input = inputs[0];
|
||||
CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || input.dims == 4 || input.dims == 5) && input.type() == CV_8S);
|
||||
CV_Assert(((input.dims == 3 && kernel_size.size() == 1) || input.dims == 4 || input.dims == 5) &&
|
||||
(input.type() == CV_8S || input.type() == CV_8U));
|
||||
for (size_t i = 0; i < outputs.size(); i++)
|
||||
{
|
||||
CV_Assert(inputs[i].type() == input.type());
|
||||
@@ -126,6 +144,19 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
void getTypes(const std::vector<MatType>& inputs,
|
||||
const int requiredOutputs,
|
||||
const int requiredInternals,
|
||||
std::vector<MatType>& outputs,
|
||||
std::vector<MatType>& internals) const CV_OVERRIDE
|
||||
{
|
||||
CV_Assert(!inputs.empty());
|
||||
for (auto t : inputs)
|
||||
CV_CheckType(t, t == CV_8S || t == CV_8U, "");
|
||||
outputs.assign(requiredOutputs, inputs[0]);
|
||||
internals.assign(requiredInternals, CV_32S);
|
||||
}
|
||||
|
||||
virtual MatShape computeColRowShape(const MatShape &inpShape, const MatShape &outShape) const = 0;
|
||||
bool is1x1() const
|
||||
{
|
||||
@@ -163,9 +194,11 @@ public:
|
||||
enum { VEC_ALIGN = 32, DFT_TYPE = CV_8S };
|
||||
Mat weightsMat;
|
||||
std::vector<int> biasvec;
|
||||
std::vector<int> biasvecVNNI; // Bias adjusted for VNNI uint8 direct path
|
||||
std::vector<float> outputMultiplier;
|
||||
Mat activationLUT;
|
||||
Ptr<ActivationLayerInt8> activ;
|
||||
Mat outputInt32Buf_; // pre-allocated scratch buffer for int32 accumulation
|
||||
|
||||
ConvolutionLayerInt8Impl(const LayerParams ¶ms) : BaseConvolutionLayerInt8Impl(params){}
|
||||
|
||||
@@ -247,13 +280,16 @@ public:
|
||||
|
||||
std::vector<Mat> inputs;
|
||||
inputs_arr.getMatVector(inputs);
|
||||
const int outCn = numOutput > 0 ? numOutput : blobs[0].size[0];
|
||||
CV_Assert(outCn > 0);
|
||||
numOutput = outCn;
|
||||
// prepare weightsMat where each row is aligned and has enough zero padding on the right to
|
||||
// use vectorized (i.e. with intrinsics) loops without tail processing
|
||||
Mat wm = blobs[0].reshape(1, numOutput);
|
||||
Mat wm = blobs[0].reshape(1, outCn);
|
||||
if( wm.step1() % VEC_ALIGN != 0 )
|
||||
{
|
||||
int newcols = (int)alignSize(wm.step1(), VEC_ALIGN);
|
||||
Mat wm_buffer = Mat(numOutput, newcols, wm.type());
|
||||
Mat wm_buffer = Mat(outCn, newcols, wm.type());
|
||||
Mat wm_padding = wm_buffer.colRange(wm.cols, newcols);
|
||||
wm_padding.setTo(Scalar::all(0));
|
||||
Mat wm_aligned = wm_buffer.colRange(0, wm.cols);
|
||||
@@ -263,11 +299,11 @@ public:
|
||||
weightsMat = wm;
|
||||
|
||||
Mat biasMat = blobs[1];
|
||||
biasvec.resize(numOutput+2);
|
||||
biasvec.resize(outCn + 2);
|
||||
|
||||
Mat outMult = blobs[2];
|
||||
outputMultiplier.resize(numOutput+2);
|
||||
for(int i = 0; i < numOutput; i++ )
|
||||
outputMultiplier.resize(outCn + 2);
|
||||
for(int i = 0; i < outCn; i++ )
|
||||
{
|
||||
biasvec[i] = biasMat.at<int>(i);
|
||||
outputMultiplier[i] = outMult.at<float>(i);
|
||||
@@ -687,7 +723,7 @@ public:
|
||||
class ParallelConv : public cv::ParallelLoopBody
|
||||
{
|
||||
public:
|
||||
enum { BLK_SIZE = 32, BLK_SIZE_CN = 64 };
|
||||
enum { BLK_SIZE = 64, BLK_SIZE_CN = 64 };
|
||||
|
||||
const Mat* input_;
|
||||
const Mat* weights_;
|
||||
@@ -704,14 +740,16 @@ public:
|
||||
bool useAVX512;
|
||||
bool useLASX;
|
||||
bool useRVV;
|
||||
bool useVNNI;
|
||||
int blk_size_cn;
|
||||
int inpZp, outZp;
|
||||
const std::vector<float>* multiplier;
|
||||
Mat* directU8Out_;
|
||||
|
||||
ParallelConv()
|
||||
: input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0),
|
||||
biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false), useLASX(false), useRVV(false)
|
||||
, blk_size_cn(0), inpZp(0), outZp(0), multiplier(0)
|
||||
biasvec_(0), activLUT_(0), activ_(0), is1x1_(false), useAVX2(false), useAVX512(false), useLASX(false), useRVV(false), useVNNI(false)
|
||||
, blk_size_cn(0), inpZp(0), outZp(0), multiplier(0), directU8Out_(0)
|
||||
{}
|
||||
|
||||
static void run( const Mat& input, Mat& output, const Mat& weights, const std::vector<float>& multipliers,
|
||||
@@ -719,7 +757,8 @@ public:
|
||||
const std::vector<size_t>& kernel_size, const std::vector<size_t>& strides,
|
||||
const std::vector<size_t>& pads_begin, const std::vector<size_t>& pads_end,
|
||||
const std::vector<size_t>& dilations,
|
||||
const ActivationLayerInt8* activ, int ngroups, int nstripes, int inp_Zp, int out_Zp)
|
||||
const ActivationLayerInt8* activ, int ngroups, int nstripes, int inp_Zp, int out_Zp,
|
||||
Mat* directU8Out = nullptr)
|
||||
{
|
||||
size_t karea = std::accumulate(kernel_size.begin(), kernel_size.end(),
|
||||
1, std::multiplies<size_t>());
|
||||
@@ -731,9 +770,9 @@ public:
|
||||
input.size[0] == output.size[0],
|
||||
weights.rows == output.size[1],
|
||||
weights.cols == (input.size[1]/ngroups)*karea,
|
||||
input.type() == CV_8SC1,
|
||||
(input.type() == CV_8SC1 || input.type() == CV_8UC1),
|
||||
output.type() == CV_32SC1,
|
||||
input.type() == weights.type(),
|
||||
weights.type() == CV_8SC1,
|
||||
input.isContinuous(),
|
||||
output.isContinuous(),
|
||||
biasvec.size() == (size_t)output.size[1]+2);
|
||||
@@ -765,7 +804,10 @@ public:
|
||||
|
||||
p.useAVX2 = checkHardwareSupport(CPU_AVX2) && isConv2D;
|
||||
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX512_SKX && isConv2D;
|
||||
|
||||
#if CV_AVXVNNI_AVAILABLE
|
||||
p.useVNNI = p.useAVX2 && checkHardwareSupport(CPU_AVX_VNNI) && isConv2D &&
|
||||
input.type() == CV_8UC1;
|
||||
#endif
|
||||
p.useLASX = checkHardwareSupport(CPU_LASX) && isConv2D;
|
||||
p.useRVV = checkHardwareSupport(CPU_RVV) && isConv2D;
|
||||
|
||||
@@ -818,6 +860,7 @@ public:
|
||||
p.biasvec_ = &biasvec;
|
||||
p.activLUT_ = &activLUT;
|
||||
p.activ_ = !activLUT.empty() ? activ : 0;
|
||||
p.directU8Out_ = directU8Out;
|
||||
|
||||
parallel_for_(Range(0, nstripes), p, nstripes);
|
||||
}
|
||||
@@ -898,27 +941,18 @@ public:
|
||||
const float* multptr_ = &multiplier->at(0);
|
||||
const int* lutptr_ = !activLUT_->empty() ? activLUT_->ptr<int>() : 0;
|
||||
int* data_out0_ = output_->ptr<int>();
|
||||
AutoBuffer<int8_t> rowbuf0_;
|
||||
int8_t* rowbuf0 = 0;
|
||||
bool use_rowbuf = !depthWiseConvolution;
|
||||
int blk_size = depthWiseConvolution ? outPlaneSize : min((int)BLK_SIZE, stripeSize);
|
||||
|
||||
// im2row buffer is not used for depth-wise convolution
|
||||
// Use thread_local to avoid repeated allocation across layers
|
||||
if(use_rowbuf)
|
||||
{
|
||||
size_t rowbufsz = alignSize(karea*blk_size_cn, valign)*min((int)BLK_SIZE, blk_size);
|
||||
//printf("karea=%d, blk_size_cn=%d, rowbufsz=%d, stripeSize=%d\n", karea, blk_size_cn, (int)rowbufsz, stripeSize);
|
||||
rowbuf0_.allocate(rowbufsz + valign);
|
||||
rowbuf0 = alignPtr(rowbuf0_.data(), (int)(valign*sizeof(int8_t)));
|
||||
// we clear the buffer once; ultimately, it lets us to avoid
|
||||
// tail processing after running the unrolled/vectorized loop.
|
||||
// the main idea is to make sure that the tail (a.k.a. padding) of each row
|
||||
// (i.e. the elements with indices between vsz=karea*ncn and vsz_a)
|
||||
// does not contain NaNs or Infs. Because the padding in the weights
|
||||
// matrix is explicitly initialized with 0's, we handle all other
|
||||
// cases nicely, i.e. we can skip expliciting re-initialization
|
||||
// of the padding - we just retain elements from the previous iteration
|
||||
// of the loop over channels (cn0).
|
||||
thread_local AutoBuffer<int8_t> rowbuf0_tls;
|
||||
rowbuf0_tls.allocate(rowbufsz + valign);
|
||||
rowbuf0 = alignPtr(rowbuf0_tls.data(), (int)(valign*sizeof(int8_t)));
|
||||
memset(rowbuf0, (int8_t)inpZp, rowbufsz*sizeof(rowbuf0[0]) );
|
||||
}
|
||||
|
||||
@@ -1346,8 +1380,12 @@ public:
|
||||
}
|
||||
}
|
||||
}
|
||||
// now compute dot product of the weights
|
||||
// and im2row-transformed part of the tensor
|
||||
#if CV_AVXVNNI_AVAILABLE
|
||||
if(useVNNI)
|
||||
opt_AVX2::fastConvVNNI(wptr, wstep, biasptr, (const uint8_t*)rowbuf0, data_out0 + ofs0,
|
||||
outShape, bsz, vsz, vsz_a, outZp, multptr, cn0 == 0, cn1 == inpCn);
|
||||
else
|
||||
#endif
|
||||
#if CV_TRY_AVX512_SKX
|
||||
if(useAVX512)
|
||||
opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
|
||||
@@ -1492,6 +1530,34 @@ public:
|
||||
activ_->forwardSlice(data_out0 + stripeStart, lutptr_,
|
||||
data_out0 + stripeStart, (int)(stripeEnd - stripeStart),
|
||||
outPlaneSize, startOutCn, startOutCn + outCn);
|
||||
|
||||
// Fused int32 → uint8 conversion (cache-hot, avoids separate convertTo pass)
|
||||
if( directU8Out_ )
|
||||
{
|
||||
uint8_t* u8base = directU8Out_->ptr<uint8_t>() + subsampleIdx*outPlaneSize*outCn;
|
||||
for( int c = 0; c < outCn; c++ )
|
||||
{
|
||||
const int* src = data_out0 + c*outPlaneSize + stripeStart;
|
||||
uint8_t* dst = u8base + c*outPlaneSize + stripeStart;
|
||||
int len = stripeEnd - stripeStart;
|
||||
int j = 0;
|
||||
#if CV_SSE2
|
||||
__m128i voffset = _mm_set1_epi32(128);
|
||||
for( ; j <= len - 8; j += 8 )
|
||||
{
|
||||
__m128i v0 = _mm_loadu_si128((const __m128i*)(src + j));
|
||||
__m128i v1 = _mm_loadu_si128((const __m128i*)(src + j + 4));
|
||||
v0 = _mm_add_epi32(v0, voffset);
|
||||
v1 = _mm_add_epi32(v1, voffset);
|
||||
__m128i p16 = _mm_packs_epi32(v0, v1);
|
||||
__m128i p8 = _mm_packus_epi16(p16, p16);
|
||||
_mm_storel_epi64((__m128i*)(dst + j), p8);
|
||||
}
|
||||
#endif
|
||||
for( ; j < len; j++ )
|
||||
dst[j] = (uint8_t)std::min(std::max(src[j] + 128, 0), 255);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -1512,19 +1578,6 @@ public:
|
||||
inputs_arr.getMatVector(inputs);
|
||||
outputs_arr.getMatVector(outputs);
|
||||
|
||||
/*if (inputs[0].dims > 3) {
|
||||
printf("conv %s: input (%d x %d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n",
|
||||
name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2], inputs[0].size[3],
|
||||
kernel.width, kernel.height, pad.width, pad.height,
|
||||
stride.width, stride.height, dilation.width, dilation.height);
|
||||
}
|
||||
else {
|
||||
printf("conv %s: input (%d x %d x %d), kernel (%d x %d), pad (%d x %d), stride (%d x %d), dilation (%d x %d)\n",
|
||||
name.c_str(), inputs[0].size[0], inputs[0].size[1], inputs[0].size[2],
|
||||
kernel.width, kernel.height, pad.width, pad.height,
|
||||
stride.width, stride.height, dilation.width, dilation.height);
|
||||
}*/
|
||||
|
||||
int inpGroupCn = blobs[0].size[1];
|
||||
CV_Assert_N(inputs.size() == (size_t)1, inputs[0].size[1] % inpGroupCn == 0,
|
||||
outputs.size() == 1, inputs[0].data != outputs[0].data);
|
||||
@@ -1533,12 +1586,53 @@ public:
|
||||
CV_Assert(outputs[0].size[1] % ngroups == 0);
|
||||
|
||||
int nstripes = std::max(getNumThreads(), 1);
|
||||
Mat outputInt32 = Mat(shape(outputs[0]), CV_32S);
|
||||
|
||||
ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides,
|
||||
pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, input_zp, output_zp);
|
||||
|
||||
outputInt32.convertTo(outputs[0], CV_8S);
|
||||
outputInt32Buf_.fit(shape(outputs[0]), CV_32S);
|
||||
Mat outputInt32 = outputInt32Buf_;
|
||||
if (inputs[0].type() == CV_8U)
|
||||
{
|
||||
#if CV_AVXVNNI_AVAILABLE
|
||||
// AVX-VNNI path: pass uint8 input directly, skip u8→s8 conversion.
|
||||
if (checkHardwareSupport(CPU_AVX_VNNI) && inputs[0].dims == 4) {
|
||||
if (biasvecVNNI.empty() && !biasvec.empty()) {
|
||||
int outCn = weightsMat.rows;
|
||||
biasvecVNNI.resize(biasvec.size());
|
||||
for (int oc = 0; oc < outCn; oc++) {
|
||||
const int8_t* wrow = weightsMat.ptr<int8_t>(oc);
|
||||
int colsum = 0;
|
||||
for (int j = 0; j < weightsMat.cols; j++)
|
||||
colsum += (int)wrow[j];
|
||||
biasvecVNNI[oc] = biasvec[oc] - 128 * colsum;
|
||||
}
|
||||
for (size_t oc = outCn; oc < biasvec.size(); oc++)
|
||||
biasvecVNNI[oc] = biasvecVNNI[outCn > 0 ? outCn - 1 : 0];
|
||||
}
|
||||
const int outZpS8 = output_zp - 128;
|
||||
ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier,
|
||||
biasvecVNNI, activationLUT, kernel_size, strides,
|
||||
pads_begin, pads_end, dilations, activ.get(),
|
||||
ngroups, nstripes, input_zp, outZpS8,
|
||||
&outputs[0]);
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
Mat inpS8;
|
||||
inputs[0].convertTo(inpS8, CV_8S, 1.0, -128.0);
|
||||
const int inpZpS8 = input_zp - 128;
|
||||
const int outZpS8 = output_zp - 128;
|
||||
ParallelConv::run(inpS8, outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides,
|
||||
pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, inpZpS8, outZpS8);
|
||||
Mat outS8;
|
||||
outputInt32.convertTo(outS8, CV_8S);
|
||||
outS8.convertTo(outputs[0], CV_8U, 1.0, 128.0);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ParallelConv::run(inputs[0], outputInt32, weightsMat, outputMultiplier, biasvec, activationLUT, kernel_size, strides,
|
||||
pads_begin, pads_end, dilations, activ.get(), ngroups, nstripes, input_zp, output_zp);
|
||||
outputInt32.convertTo(outputs[0], outputs[0].type());
|
||||
}
|
||||
|
||||
#if CV_SSE3
|
||||
_MM_SET_FLUSH_ZERO_MODE(ftzMode);
|
||||
|
||||
@@ -18,8 +18,6 @@ namespace dnn
|
||||
class ActivationLayerInt8Impl CV_FINAL : public ActivationLayerInt8
|
||||
{
|
||||
public:
|
||||
int input_zp, output_zp;
|
||||
float input_sc, output_sc;
|
||||
float slope = 0.0f;
|
||||
|
||||
#ifdef HAVE_TIMVX
|
||||
@@ -30,10 +28,10 @@ public:
|
||||
setParamsFrom(params);
|
||||
activationLUT = !blobs.empty() ? blobs[0] : Mat();
|
||||
|
||||
input_zp = params.get<int>("input_zeropoint");
|
||||
input_sc = params.get<float>("input_scale");
|
||||
output_zp = params.get<int>("zeropoints");
|
||||
output_sc = params.get<float>("scales");
|
||||
input_zp = params.get<int>("input_zeropoint", 0);
|
||||
input_sc = params.get<float>("input_scale", 1.0f);
|
||||
output_zp = params.get<int>("zeropoints", 0);
|
||||
output_sc = params.get<float>("scales", 1.0f);
|
||||
|
||||
if (params.has("slope"))
|
||||
{
|
||||
@@ -355,7 +353,6 @@ public:
|
||||
|
||||
}
|
||||
|
||||
Mat activationLUT;
|
||||
};
|
||||
|
||||
Ptr<ActivationLayerInt8> ActivationLayerInt8::create(const LayerParams& params)
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
#include "../op_timvx.hpp"
|
||||
#include "../ie_ngraph.hpp"
|
||||
#include <opencv2/dnn/shape_utils.hpp>
|
||||
#include "opencv2/core/hal/hal.hpp"
|
||||
#include "opencv2/core/hal/intrin.hpp"
|
||||
|
||||
namespace cv
|
||||
{
|
||||
@@ -22,12 +24,6 @@ public:
|
||||
SUM = 1,
|
||||
MAX = 2
|
||||
} op;
|
||||
std::vector<float> coeffs;
|
||||
std::vector<int> zeropoints;
|
||||
std::vector<float> scales;
|
||||
|
||||
int output_zp;
|
||||
float output_sc;
|
||||
|
||||
enum OutputChannelsMode
|
||||
{
|
||||
@@ -101,8 +97,8 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
output_zp = params.get<int>("zeropoints");
|
||||
output_sc = params.get<float>("scales");
|
||||
output_zp = params.get<int>("zeropoints", 0);
|
||||
output_sc = params.get<float>("scales", 1.0f);
|
||||
|
||||
channelsModeInput = ELTWISE_CHANNNELS_SAME;
|
||||
if (params.has("output_channels_mode"))
|
||||
@@ -142,6 +138,56 @@ public:
|
||||
return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH;
|
||||
}
|
||||
|
||||
void ensureQuantizationParams()
|
||||
{
|
||||
if (!coeffs.empty())
|
||||
return;
|
||||
|
||||
if (scales.empty())
|
||||
return;
|
||||
|
||||
CV_CheckEQ(scales.size(), zeropoints.size(), "EltwiseInt8: scales and zeropoints sizes must match");
|
||||
CV_Assert(output_sc > 0.0f);
|
||||
|
||||
if (op == SUM)
|
||||
{
|
||||
coeffs.resize(scales.size());
|
||||
offset = (float)output_zp;
|
||||
for (size_t i = 0; i < scales.size(); i++)
|
||||
{
|
||||
coeffs[i] = scales[i] / output_sc;
|
||||
offset -= coeffs[i] * zeropoints[i];
|
||||
}
|
||||
}
|
||||
else if (op == PROD)
|
||||
{
|
||||
coeffs.resize(scales.size());
|
||||
coeffs[0] = scales[0] / output_sc;
|
||||
for (size_t i = 1; i < scales.size(); i++)
|
||||
coeffs[i] = scales[i];
|
||||
offset = (float)output_zp;
|
||||
}
|
||||
else if (op == MAX)
|
||||
{
|
||||
for (size_t i = 0; i < scales.size(); i++)
|
||||
{
|
||||
const bool sameQuant =
|
||||
std::abs(scales[i] - output_sc) < 1e-6f &&
|
||||
zeropoints[i] == output_zp;
|
||||
if (!sameQuant)
|
||||
{
|
||||
CV_Error(Error::StsBadArg,
|
||||
"EltwiseInt8 'max' requires identical quantization "
|
||||
"(same scale and zero-point) for all inputs and output.");
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
CV_Error(Error::StsNotImplemented, "Unsupported eltwise operation");
|
||||
}
|
||||
}
|
||||
|
||||
bool getMemoryShapes(const std::vector<MatShape> &inputs,
|
||||
const int requiredOutputs,
|
||||
std::vector<MatShape> &outputs,
|
||||
@@ -248,6 +294,7 @@ public:
|
||||
bool isLast) CV_OVERRIDE
|
||||
{
|
||||
#ifdef HAVE_TIMVX
|
||||
ensureQuantizationParams();
|
||||
// tvGraph Initialization.
|
||||
if (inputsWrapper.size() != 2)
|
||||
return Ptr<BackendNode>();
|
||||
@@ -374,6 +421,7 @@ public:
|
||||
virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> > &inputs,
|
||||
const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
|
||||
{
|
||||
ensureQuantizationParams();
|
||||
CV_Assert(nodes.size() >= 2);
|
||||
std::vector<ov::Output<ov::Node>> ieInpNodes(nodes.size());
|
||||
for (size_t i = 0; i < nodes.size(); i++)
|
||||
@@ -544,6 +592,7 @@ public:
|
||||
size_t dstIdx = delta + (sampleIdx*channels + c)*planeSize;
|
||||
int8_t* dstptr = dstptr0 + dstIdx;
|
||||
float* bufptr = bufptr0 + dstIdx;
|
||||
bool directOutput = false;
|
||||
|
||||
// process first two inputs
|
||||
{
|
||||
@@ -589,7 +638,16 @@ public:
|
||||
}
|
||||
else if (op == MAX)
|
||||
{
|
||||
for (int j = 0; j < blockSize; j++)
|
||||
int j = 0;
|
||||
#if CV_SIMD128
|
||||
for (; j <= blockSize - 16; j += 16)
|
||||
{
|
||||
v_int8x16 va = v_load(srcptr0 + j);
|
||||
v_int8x16 vb = v_load(srcptrI + j);
|
||||
v_store(dstptr + j, v_max(va, vb));
|
||||
}
|
||||
#endif
|
||||
for (; j < blockSize; j++)
|
||||
{
|
||||
dstptr[j] = std::max(srcptr0[j], srcptrI[j]);
|
||||
}
|
||||
@@ -598,9 +656,86 @@ public:
|
||||
{
|
||||
float c0 = coeffsptr[0];
|
||||
float c1 = coeffsptr[1];
|
||||
for (int j = 0; j < blockSize; j++)
|
||||
if (nsrcs == 2)
|
||||
{
|
||||
bufptr[j] = c0*srcptr0[j] + c1*srcptrI[j];
|
||||
directOutput = true;
|
||||
int j = 0;
|
||||
#if CV_SIMD128
|
||||
{
|
||||
v_float32x4 vc0 = v_setall_f32(c0);
|
||||
v_float32x4 vc1 = v_setall_f32(c1);
|
||||
v_float32x4 voffs = v_setall_f32(offset);
|
||||
for (; j <= blockSize - 16; j += 16)
|
||||
{
|
||||
v_int8x16 va = v_load(srcptr0 + j);
|
||||
v_int8x16 vb = v_load(srcptrI + j);
|
||||
|
||||
v_int16x8 va_lo, va_hi, vb_lo, vb_hi;
|
||||
v_expand(va, va_lo, va_hi);
|
||||
v_expand(vb, vb_lo, vb_hi);
|
||||
|
||||
v_int32x4 va0, va1, va2, va3, vb0, vb1, vb2, vb3;
|
||||
v_expand(va_lo, va0, va1);
|
||||
v_expand(va_hi, va2, va3);
|
||||
v_expand(vb_lo, vb0, vb1);
|
||||
v_expand(vb_hi, vb2, vb3);
|
||||
|
||||
v_float32x4 r0 = v_add(v_add(v_mul(v_cvt_f32(va0), vc0),
|
||||
v_mul(v_cvt_f32(vb0), vc1)), voffs);
|
||||
v_float32x4 r1 = v_add(v_add(v_mul(v_cvt_f32(va1), vc0),
|
||||
v_mul(v_cvt_f32(vb1), vc1)), voffs);
|
||||
v_float32x4 r2 = v_add(v_add(v_mul(v_cvt_f32(va2), vc0),
|
||||
v_mul(v_cvt_f32(vb2), vc1)), voffs);
|
||||
v_float32x4 r3 = v_add(v_add(v_mul(v_cvt_f32(va3), vc0),
|
||||
v_mul(v_cvt_f32(vb3), vc1)), voffs);
|
||||
|
||||
v_store(dstptr + j, v_pack(v_pack(v_round(r0), v_round(r1)),
|
||||
v_pack(v_round(r2), v_round(r3))));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
for (; j < blockSize; j++)
|
||||
{
|
||||
dstptr[j] = saturate_cast<int8_t>(std::round(c0*srcptr0[j] + c1*srcptrI[j] + offset));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
int j = 0;
|
||||
#if CV_SIMD128
|
||||
{
|
||||
v_float32x4 vc0 = v_setall_f32(c0);
|
||||
v_float32x4 vc1 = v_setall_f32(c1);
|
||||
for (; j <= blockSize - 16; j += 16)
|
||||
{
|
||||
v_int8x16 va = v_load(srcptr0 + j);
|
||||
v_int8x16 vb = v_load(srcptrI + j);
|
||||
|
||||
v_int16x8 va_lo, va_hi, vb_lo, vb_hi;
|
||||
v_expand(va, va_lo, va_hi);
|
||||
v_expand(vb, vb_lo, vb_hi);
|
||||
|
||||
v_int32x4 va0, va1, va2, va3, vb0, vb1, vb2, vb3;
|
||||
v_expand(va_lo, va0, va1);
|
||||
v_expand(va_hi, va2, va3);
|
||||
v_expand(vb_lo, vb0, vb1);
|
||||
v_expand(vb_hi, vb2, vb3);
|
||||
|
||||
v_store(bufptr + j, v_add(v_mul(v_cvt_f32(va0), vc0),
|
||||
v_mul(v_cvt_f32(vb0), vc1)));
|
||||
v_store(bufptr + j + 4, v_add(v_mul(v_cvt_f32(va1), vc0),
|
||||
v_mul(v_cvt_f32(vb1), vc1)));
|
||||
v_store(bufptr + j + 8, v_add(v_mul(v_cvt_f32(va2), vc0),
|
||||
v_mul(v_cvt_f32(vb2), vc1)));
|
||||
v_store(bufptr + j + 12, v_add(v_mul(v_cvt_f32(va3), vc0),
|
||||
v_mul(v_cvt_f32(vb3), vc1)));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
for (; j < blockSize; j++)
|
||||
{
|
||||
bufptr[j] = c0*srcptr0[j] + c1*srcptrI[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -628,7 +763,14 @@ public:
|
||||
}
|
||||
else if (op == MAX)
|
||||
{
|
||||
for (int j = 0; j < blockSize; j++)
|
||||
int j = 0;
|
||||
#if CV_SIMD128
|
||||
for (; j <= blockSize - 16; j += 16)
|
||||
{
|
||||
v_store(dstptr + j, v_max(v_load(dstptr + j), v_load(srcptrI + j)));
|
||||
}
|
||||
#endif
|
||||
for (; j < blockSize; j++)
|
||||
{
|
||||
dstptr[j] = std::max(dstptr[j], srcptrI[j]);
|
||||
}
|
||||
@@ -636,7 +778,31 @@ public:
|
||||
else if (op == SUM)
|
||||
{
|
||||
float cI = coeffsptr[inputIdx];
|
||||
for (int j = 0; j < blockSize; j++)
|
||||
int j = 0;
|
||||
#if CV_SIMD128
|
||||
{
|
||||
v_float32x4 vcI = v_setall_f32(cI);
|
||||
for (; j <= blockSize - 16; j += 16)
|
||||
{
|
||||
v_int8x16 vi = v_load(srcptrI + j);
|
||||
v_int16x8 vi_lo, vi_hi;
|
||||
v_expand(vi, vi_lo, vi_hi);
|
||||
v_int32x4 vi0, vi1, vi2, vi3;
|
||||
v_expand(vi_lo, vi0, vi1);
|
||||
v_expand(vi_hi, vi2, vi3);
|
||||
|
||||
v_store(bufptr + j, v_add(v_load(bufptr + j),
|
||||
v_mul(v_cvt_f32(vi0), vcI)));
|
||||
v_store(bufptr + j + 4, v_add(v_load(bufptr + j + 4),
|
||||
v_mul(v_cvt_f32(vi1), vcI)));
|
||||
v_store(bufptr + j + 8, v_add(v_load(bufptr + j + 8),
|
||||
v_mul(v_cvt_f32(vi2), vcI)));
|
||||
v_store(bufptr + j + 12, v_add(v_load(bufptr + j + 12),
|
||||
v_mul(v_cvt_f32(vi3), vcI)));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
for (; j < blockSize; j++)
|
||||
{
|
||||
bufptr[j] += cI * srcptrI[j];
|
||||
}
|
||||
@@ -646,9 +812,24 @@ public:
|
||||
}
|
||||
|
||||
// add offset and saturate cast to int8
|
||||
if (op == SUM || op == PROD)
|
||||
if ((op == SUM || op == PROD) && !directOutput)
|
||||
{
|
||||
for (int j = 0; j < blockSize; j++)
|
||||
int j = 0;
|
||||
#if CV_SIMD128
|
||||
{
|
||||
v_float32x4 voffs = v_setall_f32(offset);
|
||||
for (; j <= blockSize - 16; j += 16)
|
||||
{
|
||||
v_float32x4 r0 = v_add(v_load(bufptr + j), voffs);
|
||||
v_float32x4 r1 = v_add(v_load(bufptr + j + 4), voffs);
|
||||
v_float32x4 r2 = v_add(v_load(bufptr + j + 8), voffs);
|
||||
v_float32x4 r3 = v_add(v_load(bufptr + j + 12), voffs);
|
||||
v_store(dstptr + j, v_pack(v_pack(v_round(r0), v_round(r1)),
|
||||
v_pack(v_round(r2), v_round(r3))));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
for (; j < blockSize; j++)
|
||||
{
|
||||
dstptr[j] = saturate_cast<int8_t>(std::round(bufptr[j] + offset));
|
||||
}
|
||||
@@ -667,6 +848,7 @@ public:
|
||||
{
|
||||
CV_TRACE_FUNCTION();
|
||||
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
|
||||
ensureQuantizationParams();
|
||||
|
||||
std::vector<Mat> inputs, outputs;
|
||||
inputs_arr.getMatVector(inputs);
|
||||
@@ -749,7 +931,6 @@ public:
|
||||
|
||||
private:
|
||||
bool hasVecInput;
|
||||
float offset;
|
||||
};
|
||||
|
||||
Ptr<EltwiseLayerInt8> EltwiseLayerInt8::create(const LayerParams& params)
|
||||
|
||||
@@ -22,10 +22,10 @@ public:
|
||||
{
|
||||
setParamsFrom(params);
|
||||
|
||||
input_sc = params.get<float>("input_scale");
|
||||
input_zp = params.get<int>("input_zeropoint");
|
||||
output_zp = params.get<int>("zeropoints");
|
||||
output_sc = params.get<float>("scales");
|
||||
input_sc = params.get<float>("input_scale", 1.0f);
|
||||
input_zp = params.get<int>("input_zeropoint", 0);
|
||||
output_zp = params.get<int>("zeropoints", 0);
|
||||
output_sc = params.get<float>("scales", 1.0f);
|
||||
axis = params.get<int>("axis", 1);
|
||||
per_channel = params.get<bool>("per_channel", true);
|
||||
|
||||
|
||||
@@ -4,6 +4,16 @@
|
||||
|
||||
#include "opencv2/core/hal/intrin.hpp"
|
||||
|
||||
#if !defined(CV_AVXVNNI_AVAILABLE)
|
||||
#if (CV_TRY_AVX2 || CV_AVX2) && \
|
||||
((defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 11) || \
|
||||
(defined(__clang__) && !defined(__apple_build_version__) && __clang_major__ >= 12))
|
||||
#define CV_AVXVNNI_AVAILABLE 1
|
||||
#else
|
||||
#define CV_AVXVNNI_AVAILABLE 0
|
||||
#endif
|
||||
#endif
|
||||
|
||||
namespace cv {
|
||||
namespace dnn {
|
||||
CV_CPU_OPTIMIZATION_NAMESPACE_BEGIN
|
||||
@@ -27,6 +37,17 @@ void fastGEMM1T( const int8_t* vec, const int8_t* weights,
|
||||
size_t wstep, const int* bias, const float* multiplier,
|
||||
int* dst, int nvecs, int vecsize, int outZp );
|
||||
|
||||
#if CV_AVXVNNI_AVAILABLE
|
||||
// AVX-VNNI variants: take uint8 input directly, use _mm256_dpbusd_epi32
|
||||
void fastConvVNNI( const int8_t* weights, size_t wstep, const int* bias,
|
||||
const uint8_t* rowbuf, int* output, const int* outShape,
|
||||
int blockSize, int vecsize, int vecsize_aligned, int outZp,
|
||||
const float* multiplier, bool initOutput, bool finalOutput );
|
||||
void fastGEMM1TVNNI( const uint8_t* vec, const int8_t* weights,
|
||||
size_t wstep, const int* bias, const float* multiplier,
|
||||
int* dst, int nvecs, int vecsize, int outZp );
|
||||
#endif
|
||||
|
||||
#if !defined(CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY) && CV_AVX2
|
||||
#define OPENCV_FMADD_EPI8(_Tpvec, func) \
|
||||
inline _Tpvec _##func##_fmaddepi8_epi32(const _Tpvec& a, const _Tpvec& b, const _Tpvec& c) \
|
||||
@@ -328,6 +349,231 @@ void fastConv( const int8_t* weights, size_t wstep, const int* bias,
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
|
||||
#if CV_AVXVNNI_AVAILABLE
|
||||
// AVX-VNNI optimized convolution: uses _mm256_dpbusd_epi32 (unsigned input × signed weights)
|
||||
__attribute__((target("avxvnni")))
|
||||
void fastConvVNNI( const int8_t* weights, size_t wstep, const int* bias,
|
||||
const uint8_t* rowbuf, int* output, const int* outShape,
|
||||
int blockSize, int vecsize, int vecsize_aligned, int outZp,
|
||||
const float* multiplier, bool initOutput, bool finalOutput )
|
||||
{
|
||||
int outCn = outShape[1];
|
||||
size_t outPlaneSize = outShape[2]*outShape[3];
|
||||
int CV_DECL_ALIGNED(16) maskbuf[FASCONV_BASE_VECSZ] = {0};
|
||||
int rsz = blockSize % FASCONV_BASE_VECSZ;
|
||||
for( int i = 0; i < rsz; i++ )
|
||||
maskbuf[FASCONV_BASE_VECSZ - i - 1] = -1;
|
||||
__m128 mask = _mm_loadu_ps((const float*)maskbuf);
|
||||
|
||||
for( int i = 0; i < outCn; i += 3 )
|
||||
{
|
||||
const int8_t* wptr0 = weights + i*wstep;
|
||||
const int8_t* wptr1 = wptr0 + wstep;
|
||||
const int8_t* wptr2 = wptr1 + wstep;
|
||||
int* outptr0 = output + i*outPlaneSize;
|
||||
int* outptr1 = outptr0 + outPlaneSize;
|
||||
int* outptr2 = outptr1 + outPlaneSize;
|
||||
int bias0 = bias[i], bias1 = bias[i+1], bias2 = bias[i+2];
|
||||
float mult0 = multiplier[i], mult1 = multiplier[i+1], mult2 = multiplier[i+2];
|
||||
|
||||
if( i+2 >= outCn )
|
||||
{
|
||||
wptr2 = wptr1;
|
||||
outptr2 = outptr1;
|
||||
bias2 = bias1;
|
||||
mult2 = mult1;
|
||||
|
||||
if( i+1 >= outCn )
|
||||
{
|
||||
wptr2 = wptr1 = wptr0;
|
||||
outptr2 = outptr1 = outptr0;
|
||||
bias2 = bias1 = bias0;
|
||||
mult2 = mult1 = mult0;
|
||||
}
|
||||
}
|
||||
int j = 0;
|
||||
for( ; j < blockSize; j += FASCONV_BASE_VECSZ )
|
||||
{
|
||||
bool tail = false;
|
||||
if (j + FASCONV_BASE_VECSZ > blockSize)
|
||||
{
|
||||
if (j == 0)
|
||||
break;
|
||||
j = blockSize - FASCONV_BASE_VECSZ;
|
||||
tail = true;
|
||||
}
|
||||
int k = 0;
|
||||
const uint8_t* rptr = rowbuf + j*vecsize_aligned;
|
||||
|
||||
__m256i vs00 = _mm256_setzero_si256(), vs01 = _mm256_setzero_si256(),
|
||||
vs02 = _mm256_setzero_si256(), vs03 = _mm256_setzero_si256(),
|
||||
vs10 = _mm256_setzero_si256(), vs11 = _mm256_setzero_si256(),
|
||||
vs12 = _mm256_setzero_si256(), vs13 = _mm256_setzero_si256(),
|
||||
vs20 = _mm256_setzero_si256(), vs21 = _mm256_setzero_si256(),
|
||||
vs22 = _mm256_setzero_si256(), vs23 = _mm256_setzero_si256();
|
||||
|
||||
for (; k < vecsize; k += 32, rptr += 32 )
|
||||
{
|
||||
__m256i w0 = _mm256_load_si256((const __m256i*)(wptr0 + k));
|
||||
__m256i w1 = _mm256_load_si256((const __m256i*)(wptr1 + k));
|
||||
__m256i w2 = _mm256_load_si256((const __m256i*)(wptr2 + k));
|
||||
__m256i r0 = _mm256_load_si256((const __m256i*)rptr);
|
||||
|
||||
vs00 = _mm256_dpbusd_epi32(vs00, r0, w0);
|
||||
vs10 = _mm256_dpbusd_epi32(vs10, r0, w1);
|
||||
vs20 = _mm256_dpbusd_epi32(vs20, r0, w2);
|
||||
|
||||
r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned));
|
||||
vs01 = _mm256_dpbusd_epi32(vs01, r0, w0);
|
||||
vs11 = _mm256_dpbusd_epi32(vs11, r0, w1);
|
||||
vs21 = _mm256_dpbusd_epi32(vs21, r0, w2);
|
||||
|
||||
r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*2));
|
||||
vs02 = _mm256_dpbusd_epi32(vs02, r0, w0);
|
||||
vs12 = _mm256_dpbusd_epi32(vs12, r0, w1);
|
||||
vs22 = _mm256_dpbusd_epi32(vs22, r0, w2);
|
||||
|
||||
r0 = _mm256_load_si256((const __m256i*)(rptr + vecsize_aligned*3));
|
||||
vs03 = _mm256_dpbusd_epi32(vs03, r0, w0);
|
||||
vs13 = _mm256_dpbusd_epi32(vs13, r0, w1);
|
||||
vs23 = _mm256_dpbusd_epi32(vs23, r0, w2);
|
||||
}
|
||||
|
||||
__m256i t0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs00, vs01), _mm256_hadd_epi32(vs02, vs03));
|
||||
__m256i t1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs10, vs11), _mm256_hadd_epi32(vs12, vs13));
|
||||
__m256i t2 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs20, vs21), _mm256_hadd_epi32(vs22, vs23));
|
||||
|
||||
t0 = _mm256_add_epi32(t0, _mm256_permute2x128_si256(t0, t0, 1));
|
||||
t1 = _mm256_add_epi32(t1, _mm256_permute2x128_si256(t1, t1, 1));
|
||||
t2 = _mm256_add_epi32(t2, _mm256_permute2x128_si256(t2, t2, 1));
|
||||
|
||||
__m128i s0, s1, s2;
|
||||
|
||||
if( initOutput )
|
||||
{
|
||||
s0 = _mm_set1_epi32(bias0);
|
||||
s1 = _mm_set1_epi32(bias1);
|
||||
s2 = _mm_set1_epi32(bias2);
|
||||
}
|
||||
else
|
||||
{
|
||||
s0 = _mm_loadu_si128((__m128i*)(outptr0 + j));
|
||||
s1 = _mm_loadu_si128((__m128i*)(outptr1 + j));
|
||||
s2 = _mm_loadu_si128((__m128i*)(outptr2 + j));
|
||||
}
|
||||
|
||||
s0 = _mm_add_epi32(s0, _mm256_castsi256_si128(t0));
|
||||
s1 = _mm_add_epi32(s1, _mm256_castsi256_si128(t1));
|
||||
s2 = _mm_add_epi32(s2, _mm256_castsi256_si128(t2));
|
||||
|
||||
if( finalOutput )
|
||||
{
|
||||
__m128i voutzp = _mm_set1_epi32(outZp);
|
||||
__m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127);
|
||||
s0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s0), _mm_set1_ps(mult0))));
|
||||
s1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s1), _mm_set1_ps(mult1))));
|
||||
s2 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(s2), _mm_set1_ps(mult2))));
|
||||
|
||||
s0 = _mm_min_epi32(_mm_max_epi32(s0, outmin), outmax);
|
||||
s1 = _mm_min_epi32(_mm_max_epi32(s1, outmin), outmax);
|
||||
s2 = _mm_min_epi32(_mm_max_epi32(s2, outmin), outmax);
|
||||
}
|
||||
if( tail )
|
||||
{
|
||||
s0 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr0 + j), _mm_castsi128_ps(s0), mask));
|
||||
s1 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr1 + j), _mm_castsi128_ps(s1), mask));
|
||||
s2 = _mm_castps_si128(_mm_blendv_ps(_mm_loadu_ps((const float*)outptr2 + j), _mm_castsi128_ps(s2), mask));
|
||||
}
|
||||
_mm_storeu_si128((__m128i*)(outptr0 + j), s0);
|
||||
_mm_storeu_si128((__m128i*)(outptr1 + j), s1);
|
||||
_mm_storeu_si128((__m128i*)(outptr2 + j), s2);
|
||||
}
|
||||
|
||||
for( ; j <= blockSize - 2; j += 2 )
|
||||
{
|
||||
const uint8_t* rptr0 = rowbuf + j*vecsize_aligned;
|
||||
const uint8_t* rptr1 = rowbuf + (j+1)*vecsize_aligned;
|
||||
int s00, s01, s10, s11, s20, s21;
|
||||
|
||||
if( initOutput )
|
||||
{
|
||||
s00 = s01 = bias0;
|
||||
s10 = s11 = bias1;
|
||||
s20 = s21 = bias2;
|
||||
}
|
||||
else
|
||||
{
|
||||
s00 = outptr0[j]; s01 = outptr0[j+1];
|
||||
s10 = outptr1[j]; s11 = outptr1[j+1];
|
||||
s20 = outptr2[j]; s21 = outptr2[j+1];
|
||||
}
|
||||
|
||||
for( int k = 0; k < vecsize; k++ )
|
||||
{
|
||||
int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k];
|
||||
int r = (int)rptr0[k];
|
||||
s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r;
|
||||
r = (int)rptr1[k];
|
||||
s01 += (int)w0*r; s11 += (int)w1*r; s21 += (int)w2*r;
|
||||
}
|
||||
|
||||
if( finalOutput )
|
||||
{
|
||||
s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127);
|
||||
s01 = std::min(std::max(outZp + (int)std::round(s01*mult0), -128), 127);
|
||||
s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127);
|
||||
s11 = std::min(std::max(outZp + (int)std::round(s11*mult1), -128), 127);
|
||||
s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127);
|
||||
s21 = std::min(std::max(outZp + (int)std::round(s21*mult2), -128), 127);
|
||||
}
|
||||
outptr0[j] = s00;
|
||||
outptr0[j+1] = s01;
|
||||
outptr1[j] = s10;
|
||||
outptr1[j+1] = s11;
|
||||
outptr2[j] = s20;
|
||||
outptr2[j+1] = s21;
|
||||
}
|
||||
|
||||
for( ; j < blockSize; j++ )
|
||||
{
|
||||
const uint8_t* rptr0 = rowbuf + j*vecsize_aligned;
|
||||
int s00, s10, s20;
|
||||
|
||||
if( initOutput )
|
||||
{
|
||||
s00 = bias0;
|
||||
s10 = bias1;
|
||||
s20 = bias2;
|
||||
}
|
||||
else
|
||||
{
|
||||
s00 = outptr0[j];
|
||||
s10 = outptr1[j];
|
||||
s20 = outptr2[j];
|
||||
}
|
||||
|
||||
for( int k = 0; k < vecsize; k++ )
|
||||
{
|
||||
int8_t w0 = wptr0[k], w1 = wptr1[k], w2 = wptr2[k];
|
||||
int r = (int)rptr0[k];
|
||||
s00 += (int)w0*r; s10 += (int)w1*r; s20 += (int)w2*r;
|
||||
}
|
||||
|
||||
if( finalOutput )
|
||||
{
|
||||
s00 = std::min(std::max(outZp + (int)std::round(s00*mult0), -128), 127);
|
||||
s10 = std::min(std::max(outZp + (int)std::round(s10*mult1), -128), 127);
|
||||
s20 = std::min(std::max(outZp + (int)std::round(s20*mult2), -128), 127);
|
||||
}
|
||||
outptr0[j] = s00;
|
||||
outptr1[j] = s10;
|
||||
outptr2[j] = s20;
|
||||
}
|
||||
}
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // CV_AVXVNNI_AVAILABLE (fastConvVNNI)
|
||||
|
||||
static inline void _mm256_expand_mul_add(const __m256i& a, const __m256i& b,
|
||||
__m256i& out0, __m256i& out1, __m256i& out2, __m256i& out3)
|
||||
{
|
||||
@@ -631,6 +877,81 @@ void fastGEMM1T( const int8_t* vec, const int8_t* weights,
|
||||
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
|
||||
#if CV_AVXVNNI_AVAILABLE
|
||||
// AVX-VNNI variant of fastGEMM1T: uses _mm256_dpbusd_epi32 (unsigned input × signed weights)
|
||||
__attribute__((target("avxvnni")))
|
||||
void fastGEMM1TVNNI( const uint8_t* vec, const int8_t* weights,
|
||||
size_t wstep, const int* bias, const float* multiplier,
|
||||
int* dst, int nvecs, int vecsize, int outZp )
|
||||
{
|
||||
int i = 0;
|
||||
|
||||
for( ; i <= nvecs - 8; i += 8 )
|
||||
{
|
||||
const int8_t* wptr = weights + i*wstep;
|
||||
__m256i vs0 = _mm256_setzero_si256(), vs1 = _mm256_setzero_si256(),
|
||||
vs2 = _mm256_setzero_si256(), vs3 = _mm256_setzero_si256(),
|
||||
vs4 = _mm256_setzero_si256(), vs5 = _mm256_setzero_si256(),
|
||||
vs6 = _mm256_setzero_si256(), vs7 = _mm256_setzero_si256();
|
||||
|
||||
__m128i voutzp = _mm_set1_epi32(outZp);
|
||||
__m128i outmin = _mm_set1_epi32(-128), outmax = _mm_set1_epi32(127);
|
||||
|
||||
for( int k = 0; k < vecsize; k += 32, wptr += 32 )
|
||||
{
|
||||
__m256i v = _mm256_load_si256((const __m256i*)(vec + k));
|
||||
|
||||
vs0 = _mm256_dpbusd_epi32(vs0, v, _mm256_load_si256((const __m256i*)wptr));
|
||||
vs1 = _mm256_dpbusd_epi32(vs1, v, _mm256_load_si256((const __m256i*)(wptr + wstep)));
|
||||
vs2 = _mm256_dpbusd_epi32(vs2, v, _mm256_load_si256((const __m256i*)(wptr + wstep*2)));
|
||||
vs3 = _mm256_dpbusd_epi32(vs3, v, _mm256_load_si256((const __m256i*)(wptr + wstep*3)));
|
||||
vs4 = _mm256_dpbusd_epi32(vs4, v, _mm256_load_si256((const __m256i*)(wptr + wstep*4)));
|
||||
vs5 = _mm256_dpbusd_epi32(vs5, v, _mm256_load_si256((const __m256i*)(wptr + wstep*5)));
|
||||
vs6 = _mm256_dpbusd_epi32(vs6, v, _mm256_load_si256((const __m256i*)(wptr + wstep*6)));
|
||||
vs7 = _mm256_dpbusd_epi32(vs7, v, _mm256_load_si256((const __m256i*)(wptr + wstep*7)));
|
||||
}
|
||||
|
||||
__m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs1), _mm256_hadd_epi32(vs2, vs3));
|
||||
__m256i s1 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs4, vs5), _mm256_hadd_epi32(vs6, vs7));
|
||||
|
||||
s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1));
|
||||
s1 = _mm256_add_epi32(s1, _mm256_permute2x128_si256(s1, s1, 1));
|
||||
|
||||
__m128i t0 = _mm_add_epi32(_mm256_castsi256_si128(s0), _mm_loadu_si128((__m128i*)(bias + i)));
|
||||
__m128i t1 = _mm_add_epi32(_mm256_castsi256_si128(s1), _mm_loadu_si128((__m128i*)(bias + i + 4)));
|
||||
|
||||
t0 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t0), _mm_loadu_ps(multiplier + i))));
|
||||
t1 = _mm_add_epi32(voutzp, _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(t1), _mm_loadu_ps(multiplier + i + 4))));
|
||||
|
||||
t0 = _mm_min_epi32(_mm_max_epi32(t0, outmin), outmax);
|
||||
t1 = _mm_min_epi32(_mm_max_epi32(t1, outmin), outmax);
|
||||
|
||||
_mm_storeu_si128((__m128i*)(dst + i), t0);
|
||||
_mm_storeu_si128((__m128i*)(dst + i + 4), t1);
|
||||
}
|
||||
|
||||
for( ; i < nvecs; i++ )
|
||||
{
|
||||
const int8_t* wptr = weights + i*wstep;
|
||||
__m256i vs0 = _mm256_setzero_si256();
|
||||
|
||||
for( int k = 0; k < vecsize; k += 32, wptr += 32 )
|
||||
{
|
||||
__m256i v = _mm256_load_si256((const __m256i*)(vec + k));
|
||||
vs0 = _mm256_dpbusd_epi32(vs0, v, _mm256_load_si256((const __m256i*)wptr));
|
||||
}
|
||||
|
||||
__m256i s0 = _mm256_hadd_epi32(_mm256_hadd_epi32(vs0, vs0), vs0);
|
||||
s0 = _mm256_add_epi32(s0, _mm256_permute2x128_si256(s0, s0, 1));
|
||||
int temp = _mm_extract_epi32(_mm256_castsi256_si128(s0), 0);
|
||||
dst[i] = outZp + (int)std::round((temp + bias[i]) * multiplier[i]);
|
||||
}
|
||||
|
||||
_mm256_zeroupper();
|
||||
}
|
||||
#endif // CV_AVXVNNI_AVAILABLE (fastGEMM1TVNNI)
|
||||
|
||||
#endif // CV_CPU_OPTIMIZATION_DECLARATIONS_ONLY
|
||||
|
||||
|
||||
|
||||
@@ -30,16 +30,13 @@ public:
|
||||
isGlobalPooling = std::vector<bool>(3, false);
|
||||
output_zp = params.get<int>("zeropoints", 0);
|
||||
input_zp = params.get<int>("input_zeropoint", output_zp);
|
||||
multiplier = params.get<float>("multiplier", 1.f);
|
||||
|
||||
output_sc = params.get<float>("scales", 1.f);
|
||||
input_sc = multiplier * output_sc;
|
||||
input_sc = params.get<float>("input_scale", params.get<float>("multiplier", 1.f) * output_sc);
|
||||
|
||||
hasDynamicShapes = params.get<bool>("has_dynamic_shapes", false);
|
||||
shapesInitialized = !hasDynamicShapes;
|
||||
|
||||
if (params.has("pool") || params.has("kernel_size") ||
|
||||
params.has("kernel_w") || params.has("kernel_h"))
|
||||
if (params.has("pool"))
|
||||
{
|
||||
String pool = toLowerCase(params.get<String>("pool", "max"));
|
||||
if (pool == "max")
|
||||
@@ -50,12 +47,21 @@ public:
|
||||
type = SUM;
|
||||
else
|
||||
CV_Error(Error::StsBadArg, "Unknown pooling type \"" + pool + "\"");
|
||||
}
|
||||
else
|
||||
{
|
||||
type = MAX;
|
||||
}
|
||||
|
||||
const bool hasKernelOrGlobalSpec =
|
||||
params.has("kernel_size") || params.has("kernel_w") || params.has("kernel_h") || params.has("kernel_d") ||
|
||||
params.has("global_pooling") || params.has("global_pooling_d") ||
|
||||
params.has("global_pooling_h") || params.has("global_pooling_w");
|
||||
if (hasKernelOrGlobalSpec)
|
||||
{
|
||||
getPoolingKernelParams(params, kernel_size, isGlobalPooling, pads_begin, pads_end, strides, padMode);
|
||||
globalPooling = isGlobalPooling[0] || isGlobalPooling[1] || isGlobalPooling[2];
|
||||
}
|
||||
else
|
||||
CV_Error(Error::StsBadArg, "Cannot determine pooling type");
|
||||
setParamsFrom(params);
|
||||
ceilMode = params.get<bool>("ceil_mode", true);
|
||||
spatialScale = params.get<float>("spatial_scale", 1);
|
||||
@@ -357,7 +363,7 @@ public:
|
||||
int nstripes, inpZp, outZp;
|
||||
std::vector<int> ofsbuf;
|
||||
int poolingType;
|
||||
float multiplier;
|
||||
float inputScaleRatio;
|
||||
float spatialScale;
|
||||
|
||||
std::vector<size_t> pads_begin, pads_end;
|
||||
@@ -366,13 +372,13 @@ public:
|
||||
|
||||
PoolingInvoker() : src(0), rois(0), dst(0), pad_l(0), pad_t(0), pad_r(0), pad_b(0),
|
||||
avePoolPaddedArea(false), nstripes(0), inpZp(0), outZp(0),
|
||||
poolingType(MAX), multiplier(1), spatialScale(0){}
|
||||
poolingType(MAX), inputScaleRatio(1.f), spatialScale(0){}
|
||||
|
||||
static void run(const Mat& src, const Mat& rois, Mat& dst,
|
||||
std::vector<size_t> kernel_size, std::vector<size_t> strides,
|
||||
std::vector<size_t> pads_begin, std::vector<size_t> pads_end,
|
||||
bool avePoolPaddedArea, int poolingType, float spatialScale,
|
||||
float multiplier, int inpZp, int outZp, int nstripes)
|
||||
float inputScaleRatio, int inpZp, int outZp, int nstripes)
|
||||
{
|
||||
CV_Assert_N(
|
||||
src.isContinuous(), dst.isContinuous(),
|
||||
@@ -405,7 +411,7 @@ public:
|
||||
p.outZp = outZp;
|
||||
p.poolingType = poolingType;
|
||||
p.spatialScale = spatialScale;
|
||||
p.multiplier = multiplier;
|
||||
p.inputScaleRatio = inputScaleRatio;
|
||||
|
||||
int height = isPool1D ? 1 : src.size[src.dims - 2];
|
||||
int width = src.size[src.dims - 1];
|
||||
@@ -610,7 +616,7 @@ public:
|
||||
|
||||
int bias = (avePoolPaddedArea ? (padded_kernel_area - real_kernel_area) * inpZp : 0)
|
||||
- (inpZp * kernel_area);
|
||||
float inv_kernel_area = poolingType == AVE ? multiplier / kernel_area : multiplier;
|
||||
float inv_kernel_area = poolingType == AVE ? inputScaleRatio / kernel_area : inputScaleRatio;
|
||||
#if CV_SIMD128
|
||||
if( isPool2D && xstart > 0 && x0 + 15 < x1 && (x0 + 15) * stride_w - pad_l + kernel_w < inp_width )
|
||||
{
|
||||
@@ -680,16 +686,18 @@ public:
|
||||
{
|
||||
const int nstripes = getNumThreads();
|
||||
Mat rois;
|
||||
const float inputScaleRatio = input_sc / std::max(output_sc, 1e-12f);
|
||||
PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type,
|
||||
spatialScale, multiplier, input_zp, output_zp, nstripes);
|
||||
spatialScale, inputScaleRatio, input_zp, output_zp, nstripes);
|
||||
}
|
||||
|
||||
void avePooling(Mat &src, Mat &dst)
|
||||
{
|
||||
const int nstripes = getNumThreads();
|
||||
Mat rois;
|
||||
const float inputScaleRatio = input_sc / std::max(output_sc, 1e-12f);
|
||||
PoolingInvoker::run(src, rois, dst, kernel_size, strides, pads_begin, pads_end, avePoolPaddedArea, type,
|
||||
spatialScale, multiplier, input_zp, output_zp, nstripes);
|
||||
spatialScale, inputScaleRatio, input_zp, output_zp, nstripes);
|
||||
}
|
||||
|
||||
bool getMemoryShapes(const std::vector<MatShape> &inputs,
|
||||
@@ -787,7 +795,6 @@ private:
|
||||
};
|
||||
bool hasDynamicShapes;
|
||||
bool shapesInitialized;
|
||||
float multiplier;
|
||||
};
|
||||
|
||||
Ptr<PoolingLayerInt8> PoolingLayerInt8::create(const LayerParams& params)
|
||||
|
||||
@@ -11,6 +11,42 @@ namespace cv
|
||||
namespace dnn
|
||||
{
|
||||
|
||||
#if CV_SIMD || CV_SIMD_SCALABLE
|
||||
static void dequantizeLinearChunk_u8_f32(const uint8_t* src, float* dst,
|
||||
float scale, int zp, int64_t len)
|
||||
{
|
||||
const int vlanes = VTraits<v_float32>::vlanes();
|
||||
v_float32 vscale = vx_setall_f32(scale);
|
||||
v_int32 vzp = vx_setall_s32(zp);
|
||||
int64_t j = 0;
|
||||
for (; j <= len - vlanes; j += vlanes) {
|
||||
v_int32 vi = v_reinterpret_as_s32(vx_load_expand_q(src + j));
|
||||
vi = v_sub(vi, vzp);
|
||||
v_float32 vf = v_mul(v_cvt_f32(vi), vscale);
|
||||
v_store(dst + j, vf);
|
||||
}
|
||||
for (; j < len; j++)
|
||||
dst[j] = (float)(src[j] - zp) * scale;
|
||||
}
|
||||
|
||||
static void dequantizeLinearFast_u8_f32(const uint8_t* inp, float* out,
|
||||
float scale, int zp,
|
||||
int64_t total)
|
||||
{
|
||||
const int64_t block = 1024;
|
||||
int64_t nblocks = (total + block - 1) / block;
|
||||
|
||||
parallel_for_(Range(0, (int)nblocks), [&](const Range& r) {
|
||||
for (int i = r.start; i < r.end; i++) {
|
||||
int64_t ofs = i * block;
|
||||
int64_t len = std::min(block, total - ofs);
|
||||
dequantizeLinearChunk_u8_f32(inp + ofs, out + ofs, scale, zp, len);
|
||||
}
|
||||
});
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
DequantizeLinear layer, as defined in ONNX specification:
|
||||
https://onnx.ai/onnx/operators/onnx__DequantizeLinear.html
|
||||
@@ -165,6 +201,19 @@ static void dequantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp,
|
||||
}
|
||||
}
|
||||
|
||||
// Fast path: per-tensor dequantization uint8→float with universal intrinsics
|
||||
#if CV_SIMD || CV_SIMD_SCALABLE
|
||||
if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F) {
|
||||
float sc = reinterpret_cast<const float*>(scale.data)[0];
|
||||
int zpval = zp.empty() ? 0 : (int)reinterpret_cast<const uint8_t*>(zp.data)[0];
|
||||
int64_t total = nslices * slice_size;
|
||||
dequantizeLinearFast_u8_f32(reinterpret_cast<const uint8_t*>(inp.data),
|
||||
reinterpret_cast<float*>(out.data),
|
||||
sc, zpval, total);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (inptype == CV_8U && sctype == CV_32F && outtype == CV_32F)
|
||||
dequantizeLinear(reinterpret_cast<const uint8_t*>(inp.data),
|
||||
reinterpret_cast<const float*>(scale.data),
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
#include "../op_vkcom.hpp"
|
||||
|
||||
#include <opencv2/dnn/shape_utils.hpp>
|
||||
#include "opencv2/core/hal/intrin.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
@@ -468,13 +469,70 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
#if CV_SIMD
|
||||
// Fast path: fully contiguous float Add → flatten + SIMD + parallel_for_
|
||||
bool is_add = (this->op == OPERATION::SUM || this->op == OPERATION::ADD);
|
||||
if (is_add && std::is_same<T, float>::value && std::is_same<RESULT_T, float>::value &&
|
||||
dp1 == 1 && dp2 == 1 && dp == 1 && ndims >= 1) {
|
||||
bool contiguous = true;
|
||||
for (int k = ndims - 2; k >= 0; k--) {
|
||||
if (shape[k] <= 1) continue; // size-1 dims have stride 0, skip
|
||||
size_t expected = (size_t)shape[k + 1] * step1[k + 1];
|
||||
if (step1[k] != expected || step2[k] != expected || step[k] != expected) {
|
||||
contiguous = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (contiguous) {
|
||||
int64_t total = (int64_t)nplanes * plane_size;
|
||||
const float* p1 = (const float*)data1;
|
||||
const float* p2 = (const float*)data2;
|
||||
float* po = (float*)data;
|
||||
const int64_t chunk = 1024;
|
||||
int64_t nchunks = (total + chunk - 1) / chunk;
|
||||
parallel_for_(Range(0, (int)nchunks), [&](const Range& r) {
|
||||
for (int c = r.start; c < r.end; c++) {
|
||||
int64_t start = c * chunk;
|
||||
int64_t end = std::min(start + chunk, total);
|
||||
int64_t i = start;
|
||||
for (; i <= end - (int64_t)VTraits<v_float32>::nlanes * 4; i += VTraits<v_float32>::nlanes * 4) {
|
||||
v_store(po + i, v_add(vx_load(p1 + i), vx_load(p2 + i)));
|
||||
v_store(po + i + VTraits<v_float32>::nlanes, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes), vx_load(p2 + i + VTraits<v_float32>::nlanes)));
|
||||
v_store(po + i + VTraits<v_float32>::nlanes*2, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes*2), vx_load(p2 + i + VTraits<v_float32>::nlanes*2)));
|
||||
v_store(po + i + VTraits<v_float32>::nlanes*3, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes*3), vx_load(p2 + i + VTraits<v_float32>::nlanes*3)));
|
||||
}
|
||||
for (; i < end; i++)
|
||||
po[i] = p1[i] + p2[i];
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (nplanes == 1) { // parallelize within the plane
|
||||
const T* ptr1 = (const T*)data1;
|
||||
const T* ptr2 = (const T*)data2;
|
||||
RESULT_T* ptr = (RESULT_T*)data;
|
||||
auto worker = [&](const Range &r) {
|
||||
if (dp1 == 1 && dp2 == 1 && dp == 1) {
|
||||
for(int i = r.start; i < r.end; i++) {
|
||||
int i = r.start;
|
||||
#if CV_SIMD
|
||||
if (is_add && std::is_same<T, float>::value && std::is_same<RESULT_T, float>::value) {
|
||||
const float* p1 = (const float*)(const void*)&ptr1[r.start];
|
||||
const float* p2 = (const float*)(const void*)&ptr2[r.start];
|
||||
float* po = (float*)(void*)&ptr[r.start];
|
||||
int len = r.end - r.start, j = 0;
|
||||
for (; j <= len - VTraits<v_float32>::nlanes * 4; j += VTraits<v_float32>::nlanes * 4) {
|
||||
v_store(po + j, v_add(vx_load(p1 + j), vx_load(p2 + j)));
|
||||
v_store(po + j + VTraits<v_float32>::nlanes, v_add(vx_load(p1 + j + VTraits<v_float32>::nlanes), vx_load(p2 + j + VTraits<v_float32>::nlanes)));
|
||||
v_store(po + j + VTraits<v_float32>::nlanes*2, v_add(vx_load(p1 + j + VTraits<v_float32>::nlanes*2), vx_load(p2 + j + VTraits<v_float32>::nlanes*2)));
|
||||
v_store(po + j + VTraits<v_float32>::nlanes*3, v_add(vx_load(p1 + j + VTraits<v_float32>::nlanes*3), vx_load(p2 + j + VTraits<v_float32>::nlanes*3)));
|
||||
}
|
||||
i = r.start + j;
|
||||
}
|
||||
#endif
|
||||
for(; i < r.end; i++) {
|
||||
ptr[i] = op(ptr1[i], ptr2[i]);
|
||||
}
|
||||
} else if (dp1 == 1 && dp2 == 0 && dp == 1){
|
||||
@@ -516,7 +574,21 @@ public:
|
||||
const T* ptr2 = (const T*)ptr2_;
|
||||
RESULT_T* ptr = (RESULT_T*)ptr_;
|
||||
if (dp1 == 1 && dp2 == 1 && dp == 1) {
|
||||
for(int i = 0; i < plane_size; i++) {
|
||||
int i = 0;
|
||||
#if CV_SIMD
|
||||
if (is_add && std::is_same<T, float>::value && std::is_same<RESULT_T, float>::value) {
|
||||
const float* p1 = (const float*)(const void*)ptr1;
|
||||
const float* p2 = (const float*)(const void*)ptr2;
|
||||
float* po = (float*)(void*)ptr;
|
||||
for (; i <= plane_size - VTraits<v_float32>::nlanes * 4; i += VTraits<v_float32>::nlanes * 4) {
|
||||
v_store(po + i, v_add(vx_load(p1 + i), vx_load(p2 + i)));
|
||||
v_store(po + i + VTraits<v_float32>::nlanes, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes), vx_load(p2 + i + VTraits<v_float32>::nlanes)));
|
||||
v_store(po + i + VTraits<v_float32>::nlanes*2, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes*2), vx_load(p2 + i + VTraits<v_float32>::nlanes*2)));
|
||||
v_store(po + i + VTraits<v_float32>::nlanes*3, v_add(vx_load(p1 + i + VTraits<v_float32>::nlanes*3), vx_load(p2 + i + VTraits<v_float32>::nlanes*3)));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
for(; i < plane_size; i++) {
|
||||
ptr[i] = op(ptr1[i], ptr2[i]);
|
||||
}
|
||||
} else if (dp1 == 1 && dp2 == 0 && dp == 1){
|
||||
|
||||
@@ -7,11 +7,63 @@
|
||||
#include "layers_common.hpp"
|
||||
#include "../net_impl.hpp"
|
||||
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
|
||||
namespace cv
|
||||
{
|
||||
namespace dnn
|
||||
{
|
||||
|
||||
// Fast path for per-tensor quantization: float → uint8 with AVX2
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
__attribute__((target("avx2")))
|
||||
#endif
|
||||
static void quantizeLinearChunk_f32_u8_avx2(const float* src, uint8_t* dst,
|
||||
float inv_scale, float zp_f,
|
||||
int64_t len)
|
||||
{
|
||||
__m256 vscale = _mm256_set1_ps(inv_scale);
|
||||
__m256 vzp = _mm256_set1_ps(zp_f);
|
||||
__m256 vmin = _mm256_setzero_ps();
|
||||
__m256 vmax = _mm256_set1_ps(255.f);
|
||||
|
||||
int64_t j = 0;
|
||||
for (; j <= len - 8; j += 8) {
|
||||
__m256 v = _mm256_loadu_ps(src + j);
|
||||
v = _mm256_add_ps(_mm256_mul_ps(v, vscale), vzp);
|
||||
v = _mm256_min_ps(_mm256_max_ps(v, vmin), vmax);
|
||||
__m256i vi = _mm256_cvtps_epi32(v);
|
||||
__m128i lo = _mm256_castsi256_si128(vi);
|
||||
__m128i hi = _mm256_extracti128_si256(vi, 1);
|
||||
__m128i packed16 = _mm_packs_epi32(lo, hi);
|
||||
__m128i packed8 = _mm_packus_epi16(packed16, packed16);
|
||||
_mm_storel_epi64((__m128i*)(dst + j), packed8);
|
||||
}
|
||||
for (; j < len; j++)
|
||||
dst[j] = saturate_cast<uint8_t>(src[j] * inv_scale + zp_f);
|
||||
}
|
||||
|
||||
static void quantizeLinearFast_f32_u8_avx2(const float* inp, uint8_t* out,
|
||||
float inv_scale, float zp_f,
|
||||
int64_t total)
|
||||
{
|
||||
const int64_t block = 1024;
|
||||
int64_t nblocks = (total + block - 1) / block;
|
||||
|
||||
parallel_for_(Range(0, (int)nblocks), [&](const Range& r) {
|
||||
for (int i = r.start; i < r.end; i++) {
|
||||
int64_t ofs = i * block;
|
||||
int64_t len = std::min(block, total - ofs);
|
||||
quantizeLinearChunk_f32_u8_avx2(inp + ofs, out + ofs, inv_scale, zp_f, len);
|
||||
}
|
||||
});
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
/*
|
||||
QuantizeLinear layer, as defined in ONNX specification:
|
||||
https://onnx.ai/onnx/operators/onnx__QuantizeLinear.html
|
||||
@@ -172,6 +224,20 @@ static void quantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp,
|
||||
}
|
||||
}
|
||||
|
||||
// Fast path: per-tensor quantization float→uint8 with AVX2 + proper parallelism
|
||||
#if defined(__x86_64__) || defined(_M_X64)
|
||||
if (block_size == 0 && sz_a == 1 && inptype == CV_32F && outtype == CV_8U && sctype == CV_32F
|
||||
&& checkHardwareSupport(CV_CPU_AVX2)) {
|
||||
float inv_scale = 1.f / reinterpret_cast<const float*>(scale.data)[0];
|
||||
float zp_f = zp.empty() ? 0.f : (float)reinterpret_cast<const uint8_t*>(zp.data)[0];
|
||||
int64_t total = nslices * slice_size;
|
||||
quantizeLinearFast_f32_u8_avx2(reinterpret_cast<const float*>(inp.data),
|
||||
reinterpret_cast<uint8_t*>(out.data),
|
||||
inv_scale, zp_f, total);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (outtype == CV_8U && sctype == CV_32F && inptype == CV_32F)
|
||||
quantizeLinear(reinterpret_cast<const float*>(inp.data),
|
||||
reinterpret_cast<const float*>(scale.data),
|
||||
|
||||
@@ -450,6 +450,8 @@ struct Net::Impl : public detail::NetImplBase
|
||||
// fuse batch norm, add bias and activation to convolution
|
||||
void fuseBasic();
|
||||
// replace constant sub-expressions with their results
|
||||
|
||||
void fuseQDQ();
|
||||
void constFold();
|
||||
// make some operations (activation, batch norm, convolution) unary if
|
||||
// all their arguments except for the 1st one are constant.
|
||||
|
||||
@@ -497,6 +497,7 @@ void Net::Impl::prepareForInference()
|
||||
#endif
|
||||
|
||||
if (!prepared) {
|
||||
fuseQDQ();
|
||||
constFold();
|
||||
constArgs();
|
||||
useBlockLayout();
|
||||
|
||||
Reference in New Issue
Block a user