potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-21 19:33:03 +04:00
Code Packages Releases Wiki Activity

Merge pull request #28741 from abhishek-gola:int8_block_layout

Browse Source 
Int8 block layout support #28741

After this patch we got the following speed ups on **resnet50-qdq.onnx** model.

- Inference time now: **_~6.6ms_** (inference time using onnxruntime is ~5.7ms).
- Inference time before: _**~11.5ms**_ [after QDQ PR #28595]
- Speed up: _**~42.6% or 1.74x**_
- Device details:
- Model name: Intel(R) Core(TM) i9-14900KS, x86, 32 Cores, ubuntu 24.04, 

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV
- [x] The PR is proposed to the proper branch
- [x] There is a reference to the original bug report and related work
- [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Abhishek Gola
2026-04-12 22:10:03 +05:30
committed by GitHub
parent 1a6f669763
commit 53d9a67cf3
11 changed files with 3092 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/dnn/CMakeLists.txt
+1
View File
@@ -12,6 +12,7 @@ ocv_add_dispatched_file("layers/cpu_kernels/conv_winograd_f63" AVX AVX2 NEON NEO
ocv_add_dispatched_file_force_all("layers/cpu_kernels/fast_gemm_kernels" AVX AVX2 NEON LASX)
ocv_add_dispatched_file("layers/cpu_kernels/conv2_depthwise" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file("layers/cpu_kernels/conv2_kernels" AVX AVX2 NEON NEON_FP16)
ocv_add_dispatched_file_force_all("int8layers/conv2_int8_kernels" AVX2)
ocv_add_module(dnn opencv_core opencv_imgproc WRAP python java objc js)
modules/dnn/include/opencv2/dnn/all_layers.hpp
+49
View File
@@ -382,6 +382,21 @@ CV__DNN_INLINE_NS_BEGIN
bool ceil_mode;
};
class CV_EXPORTS Conv2Int8Layer : public Layer
{
public:
static Ptr<Conv2Int8Layer> create(const LayerParams& params);
int input_zp, output_zp;
float input_sc, output_sc;
bool per_channel;
std::vector<int> strides, dilations, pads;
int ngroups;
AutoPadding auto_pad;
bool ceil_mode;
};
class CV_EXPORTS LRNLayer : public Layer
{
public:
@@ -461,6 +476,9 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<PoolingLayer> create(const LayerParams& params);
};
// Old-engine int8 pooling. Created directly by the ONNX importer for
// QLinearAveragePool / QLinearGlobalAveragePool / int8 MaxPool ops.
// Inherits PoolingLayer so it can delegate to TIMVX / NGRAPH backends.
class CV_EXPORTS PoolingLayerInt8 : public PoolingLayer
{
public:
@@ -469,6 +487,25 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<PoolingLayerInt8> create(const LayerParams& params);
};
// New-engine int8 pooling with block memory layout (DATA_LAYOUT_BLOCK).
// Created by the QDQ graph fusion pass (graph_fusion_qdq.cpp) when it
// detects a DequantizeLinear -> Pooling -> QuantizeLinear pattern.
// Uses optimised SIMD kernels; OPENCV (CPU) backend only.
class CV_EXPORTS Pool2Int8Layer : public Layer
{
public:
static Ptr<Pool2Int8Layer> create(const LayerParams& params);
int input_zp, output_zp;
float input_sc, output_sc;
std::vector<int> kernel_shape, strides, dilations, pads;
AutoPadding auto_pad;
bool ceil_mode;
bool is_global_pooling;
bool is_max_pool;
};
class CV_EXPORTS AveragePoolLayer : public Layer
{
public:
@@ -559,6 +596,7 @@ CV__DNN_INLINE_NS_BEGIN
public:
int input_zp, output_zp;
float input_sc, output_sc;
int output_type; // CV_8S or CV_8U
// quantization type flag. The perChannel default is true, that means it contains the parameters
// of per-Channel quantization. Otherwise, that means this layer contains per-Tensor quantized parameters.
@@ -1214,6 +1252,17 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<EltwiseLayerInt8> create(const LayerParams &params);
};
class CV_EXPORTS Eltwise2Int8Layer : public Layer
{
public:
static Ptr<Eltwise2Int8Layer> create(const LayerParams& params);
std::vector<float> scales;
std::vector<int> zeropoints;
float output_sc;
int output_zp;
};
class CV_EXPORTS NaryEltwiseLayer : public Layer
{
public:
modules/dnn/src/graph_fusion_qdq.cpp
+535 -373
View File
File diff suppressed because it is too large Load Diff
modules/dnn/src/init.cpp
+3
View File
@@ -263,6 +263,9 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(InnerProductInt8, InnerProductLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(PoolingInt8, PoolingLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(EltwiseInt8, EltwiseLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(Conv2Int8, Conv2Int8Layer);
CV_DNN_REGISTER_LAYER_CLASS(Pool2Int8, Pool2Int8Layer);
CV_DNN_REGISTER_LAYER_CLASS(Eltwise2Int8, Eltwise2Int8Layer);
CV_DNN_REGISTER_LAYER_CLASS(BatchNormInt8, BatchNormLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(ScaleInt8, ScaleLayerInt8);
CV_DNN_REGISTER_LAYER_CLASS(ShiftInt8, ShiftLayerInt8);
modules/dnn/src/int8layers/conv2_int8_kernels.simd.hpp
+1221
View File
File diff suppressed because it is too large Load Diff
modules/dnn/src/int8layers/conv2_int8_layer.cpp
+371
View File
@@ -0,0 +1,371 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "../net_impl.hpp"
#include "../layers/conv2_common.hpp"
#include "opencv2/core/hal/intrin.hpp"
#include "conv2_int8_kernels.simd.hpp"
#include "int8layers/conv2_int8_kernels.simd_declarations.hpp"
namespace cv {
namespace dnn {
static MatShape getWpackShapeInt8(const MatShape& wshape, int ngroups, int C0)
{
CV_Assert(wshape.dims >= 3);
int K = wshape[0], Cg = wshape[1];
int ksize = (int)(wshape.total()) / (K * Cg);
CV_Assert(K % ngroups == 0);
int Kg = K / ngroups, K0 = C0;
int Kblk = (Kg + K0 - 1) / K0;
int C1Max = 0;
for (int g = 0; g < ngroups; ++g) {
int c_start = g * Cg;
int c00 = c_start & (C0 - 1);
int cblocks = (c00 + Cg + C0 - 1) / C0;
C1Max = std::max(C1Max, cblocks);
}
return MatShape({ngroups, Kblk, ksize, C1Max, C0 * K0}, DATA_LAYOUT_UNKNOWN);
}
static void repackConvWeightsInt8(const Mat& weights, Mat& Wpack, int ngroups, int C0_)
{
CV_Assert(weights.isContinuous());
CV_Assert(weights.type() == CV_8SC1);
CV_Assert(ngroups > 0);
CV_Assert((C0_ & (C0_ - 1)) == 0 && C0_ >= 4);
MatShape wshape = weights.shape();
CV_Assert(wshape.dims >= 3);
int K = wshape[0];
CV_Assert(K % ngroups == 0);
if (!Wpack.isContinuous())
Wpack.release();
MatShape wpackShape = getWpackShapeInt8(wshape, ngroups, C0_);
Wpack.create(wpackShape, CV_8SC1);
Wpack.setZero();
parallel_for_(Range(0, K), [&](const Range& range) {
int Cg = wshape[1], Kg = K / ngroups;
int ksize = wpackShape[2], C1Max = wpackShape[3];
int C0 = C0_, K0 = C0;
const int8_t* wdata = weights.ptr<int8_t>();
int8_t* Wpackdata = Wpack.ptr<int8_t>();
for (int k = range.start; k < range.end; ++k) {
int g = k / Kg;
int kin = k - g * Kg;
int kblk = kin / K0;
int k0 = kin & (K0 - 1);
int c_start = g * Cg;
int c00 = c_start & (C0 - 1);
for (int c = 0; c < Cg; ++c) {
int ch = c00 + c;
int c1 = ch / C0;
int c0 = ch & (C0 - 1);
const int8_t* wptr = wdata + ((k * Cg + c) * ksize);
int8_t* wpackptr = Wpackdata +
(((g * (int64_t)wpackShape[1] + kblk) * ksize * C1Max + c1) * C0 + c0) * K0 + k0;
for (int i = 0; i < ksize; ++i) {
wpackptr[i * (C1Max * C0 * K0)] = wptr[i];
}
}
}
});
}
static void repackWeightsForVNNI(const Mat& wpack, int ngroups, int Kg, int Cg,
const Mat& biasInt32,
Mat& wpackVNNI, Mat& biasVNNI)
{
constexpr int C0 = 8, K0 = 8;
MatShape ws = wpack.shape();
int Kblk = ws[1], ksize = ws[2], C1Max = ws[3];
wpackVNNI.create(ws, CV_8SC1);
const int8_t* src = wpack.ptr<int8_t>();
int8_t* dst = wpackVNNI.ptr<int8_t>();
int blockSize = C0 * K0;
int totalBlocks = (int)(ws.total() / blockSize);
parallel_for_(Range(0, totalBlocks), [&](const Range& range) {
for (int b = range.start; b < range.end; b++) {
const int8_t* s = src + (size_t)b * blockSize;
int8_t* d = dst + (size_t)b * blockSize;
for (int k = 0; k < K0; k++) {
d[k*4 + 0] = s[0*K0 + k];
d[k*4 + 1] = s[1*K0 + k];
d[k*4 + 2] = s[2*K0 + k];
d[k*4 + 3] = s[3*K0 + k];
d[32 + k*4 + 0] = s[4*K0 + k];
d[32 + k*4 + 1] = s[5*K0 + k];
d[32 + k*4 + 2] = s[6*K0 + k];
d[32 + k*4 + 3] = s[7*K0 + k];
}
}
});
int K = ngroups * Kg;
biasVNNI.create({K}, CV_32SC1);
const int32_t* bsrc = biasInt32.ptr<int32_t>();
int32_t* bdst = biasVNNI.ptr<int32_t>();
parallel_for_(Range(0, K), [&](const Range& range) {
for (int k = range.start; k < range.end; k++) {
int g = k / Kg;
int kin = k - g * Kg;
int kblk = kin / K0;
int k0 = kin & (K0 - 1);
int c_start = g * Cg;
int c00 = c_start & (C0 - 1);
int cblocks = (c00 + Cg + C0 - 1) / C0;
const int8_t* wbase = src + (size_t)((g * Kblk + kblk) * ksize * C1Max) * C0 * K0;
int32_t wsum = 0;
for (int ki = 0; ki < ksize; ki++)
for (int c1 = 0; c1 < cblocks; c1++)
for (int c0 = 0; c0 < C0; c0++)
wsum += (int)wbase[((size_t)ki * C1Max + c1) * C0 * K0 + c0 * K0 + k0];
bdst[k] = bsrc[k] - 128 * wsum;
}
});
}
static void convInt8Block(const void* inp_, const void* residual_,
void* out_, const ConvState& cs,
const void* weights_,
const void* weightsVNNI_,
const int* bias, const int* biasVNNI_,
const float* multiplier,
int inp_zp, int out_zp,
const int8_t* activLUT,
bool inputIsU8)
{
#if CV_TRY_AVX2
if (cv::checkHardwareSupport(CV_CPU_AVX2)) {
opt_AVX2::convInt8Block(inp_, residual_, out_, cs, weights_,
weightsVNNI_, bias, biasVNNI_,
multiplier, inp_zp, out_zp, activLUT, inputIsU8);
return;
}
#endif
CV_CPU_CALL_BASELINE(convInt8Block, (inp_, residual_, out_, cs, weights_,
weightsVNNI_, bias, biasVNNI_,
multiplier, inp_zp, out_zp, activLUT, inputIsU8));
}
class Conv2Int8LayerImpl CV_FINAL : public Conv2Int8Layer
{
public:
Mat weights; // repacked int8 weights in block format
Mat weightsVNNI; // VNNI-transposed weights (pre-computed)
Mat biasInt32; // int32 fused bias
Mat biasVNNI; // int32 VNNI-adjusted bias (pre-computed)
Mat outMultiplier; // float32 per-channel output multiplier
MatShape wshape0; // original weight shape (K x Cg x kH x kW)
MatShape prevInpshape;
ConvState cs;
Mat activationLUT;
Ptr<ActivationLayerInt8> activ;
bool addResidual;
bool inputIsU8;
Conv2Int8LayerImpl(const LayerParams& params)
{
setParamsFrom(params);
auto_pad = getAutoPadding(params);
ceil_mode = params.get<bool>("ceil_mode", false);
strides = params.getVector<int>("stride");
dilations = params.getVector<int>("dilation");
pads = params.getVector<int>("pad");
ngroups = params.get<int>("group", 1);
input_sc = params.get<float>("input_scale", 1.f);
input_zp = params.get<int>("input_zeropoint", 0);
output_sc = params.get<float>("scales", 1.f);
output_zp = params.get<int>("zeropoints", 0);
per_channel = params.get<bool>("per_channel", true);
inputIsU8 = params.get<bool>("input_is_u8", false);
addResidual = false;
if (!blobs.empty()) {
CV_Assert(blobs.size() >= 3);
wshape0 = blobs[0].shape();
biasInt32 = blobs[1];
outMultiplier = blobs[2];
}
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs, const int,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
// Input can be CV_8SC1 or CV_8UC1; output matches input type
int outtype = !inputs.empty() ? inputs[0] : CV_8SC1;
outputs.assign(requiredOutputs, outtype);
internals.clear();
}
bool getMemoryShapes(const std::vector<MatShape>& inpshapes,
const int,
std::vector<MatShape>& outshapes,
std::vector<MatShape>& tempshapes) const CV_OVERRIDE
{
size_t ninputs = inpshapes.size();
CV_Assert(ninputs >= 1);
std::vector<int> emptyKernelShape;
outshapes.assign(1, convInferShape(inpshapes[0], wshape0, emptyKernelShape,
ngroups, strides, dilations,
pads, auto_pad, ceil_mode));
tempshapes.clear();
return true;
}
int getLayouts(const std::vector<DataLayout>& actualInputs,
std::vector<DataLayout>& desiredInputs,
const int requiredOutputs,
std::vector<DataLayout>& outputs) const CV_OVERRIDE
{
size_t ninputs = actualInputs.size();
CV_Assert(ninputs >= 1u && requiredOutputs == 1u);
desiredInputs = actualInputs;
desiredInputs[0] = DATA_LAYOUT_BLOCK;
for (size_t i = 1; i < ninputs; i++)
desiredInputs[i] = DATA_LAYOUT_UNKNOWN;
outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK);
return getNetImpl(this)->defaultC0;
}
void setWeightsInt8(const Mat& w_q, const Mat& bias, const Mat& multiplier, int C0)
{
CV_Assert(w_q.type() == CV_8SC1);
wshape0 = w_q.shape();
biasInt32 = bias;
outMultiplier = multiplier;
repackConvWeightsInt8(w_q, weights, ngroups, C0);
}
bool fuseAddResidual(Arg /*residual*/)
{
addResidual = true;
return true;
}
bool setActivation(const Ptr<ActivationLayer>& layer) CV_OVERRIDE
{
Ptr<ActivationLayerInt8> activ_int8 = layer.dynamicCast<ActivationLayerInt8>();
if (!activ_int8.empty()) {
activ = activ_int8;
if (!activ_int8->blobs.empty())
activ_int8->blobs[0].convertTo(activationLUT, CV_8S);
return true;
}
return false;
}
void forward(InputArrayOfArrays input_arrs,
OutputArrayOfArrays output_arrs,
OutputArrayOfArrays) CV_OVERRIDE
{
int ninputs = (int)input_arrs.total();
CV_Assert(ninputs >= 1);
const Mat& inp = input_arrs.getMat(0);
MatShape inpshape = inp.shape();
CV_Assert(inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(inp.type() == CV_8SC1 || inp.type() == CV_8UC1);
int C0 = inpshape.back();
if (weights.empty() && !blobs.empty()) {
repackConvWeightsInt8(blobs[0], weights, ngroups, C0);
}
if (weightsVNNI.empty() && !weights.empty()) {
int K = wshape0[0];
int Cg = wshape0[1];
int Kg = K / ngroups;
repackWeightsForVNNI(weights, ngroups, Kg, Cg,
biasInt32, weightsVNNI, biasVNNI);
}
Mat residual;
const void* resptr = nullptr;
if (addResidual) {
residual = input_arrs.getMat(ninputs - 1);
resptr = residual.data;
ninputs--;
}
std::vector<int> emptyKernelShape;
MatShape outshape = convInferShape(inpshape, wshape0, emptyKernelShape,
ngroups, strides, dilations,
pads, auto_pad, ceil_mode);
int outtype = inp.type();
int outkind = output_arrs.kind();
Mat out;
if (outkind == _InputArray::STD_VECTOR_MAT) {
std::vector<Mat>& outs = output_arrs.getMatVecRef();
outs.resize(1);
outs[0].fit(outshape, outtype);
out = outs[0];
} else {
std::vector<UMat>& outs = output_arrs.getUMatVecRef();
outs.resize(1);
outs[0].fit(outshape, outtype);
out.fit(outshape, outtype);
}
if (inpshape != prevInpshape) {
cs.initConv(inpshape, wshape0, outshape, ngroups,
strides, dilations, pads, auto_pad, ceil_mode,
FAST_ACTIV_NONE, {});
prevInpshape = inpshape;
}
const int8_t* lutptr = !activationLUT.empty() ? activationLUT.ptr<int8_t>() : nullptr;
convInt8Block(inp.data, resptr, out.data, cs,
weights.data,
weightsVNNI.empty() ? nullptr : weightsVNNI.data,
biasInt32.ptr<int>(),
biasVNNI.empty() ? nullptr : biasVNNI.ptr<int>(),
outMultiplier.ptr<float>(),
input_zp, output_zp,
lutptr, inputIsU8);
if (outkind == _InputArray::STD_VECTOR_UMAT) {
std::vector<UMat>& outs = output_arrs.getUMatVecRef();
out.copyTo(outs[0]);
}
}
};
Ptr<Conv2Int8Layer> Conv2Int8Layer::create(const LayerParams& params)
{
return Ptr<Conv2Int8Layer>(new Conv2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/int8layers/eltwise2_int8_layer.cpp
+413
View File
@@ -0,0 +1,413 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "../net_impl.hpp"
#include "opencv2/core/hal/intrin.hpp"
#if CV_AVX2
#include <immintrin.h>
#endif
#if CV_NEON
#include <arm_neon.h>
#endif
namespace cv {
namespace dnn {
class Eltwise2Int8LayerImpl CV_FINAL : public Eltwise2Int8Layer
{
public:
std::vector<float> coeffs;
float offset;
bool withRelu;
Mat activationLUT;
Ptr<ActivationLayerInt8> activ;
Eltwise2Int8LayerImpl(const LayerParams& params)
{
setParamsFrom(params);
output_zp = params.get<int>("zeropoints", 0);
output_sc = params.get<float>("scales", 1.0f);
withRelu = params.get<bool>("with_relu", false);
if (params.has("input_scales"))
{
DictValue sc = params.get("input_scales");
int n = sc.size();
scales.resize(n);
for (int i = 0; i < n; i++)
scales[i] = sc.get<float>(i);
}
if (params.has("input_zeropoints"))
{
DictValue zp = params.get("input_zeropoints");
int n = zp.size();
zeropoints.resize(n);
for (int i = 0; i < n; i++)
zeropoints[i] = zp.get<int>(i);
}
offset = 0.f;
}
void ensureCoeffs()
{
if (!coeffs.empty() || scales.empty())
return;
CV_CheckEQ(scales.size(), zeropoints.size(),
"Eltwise2Int8: scales and zeropoints sizes must match");
CV_Assert(output_sc > 0.f);
coeffs.resize(scales.size());
offset = (float)output_zp;
for (size_t i = 0; i < scales.size(); i++)
{
coeffs[i] = scales[i] / output_sc;
offset -= coeffs[i] * zeropoints[i];
}
}
bool getMemoryShapes(const std::vector<MatShape>& inputs,
const int requiredOutputs,
std::vector<MatShape>& outputs,
std::vector<MatShape>& internals) const CV_OVERRIDE
{
CV_Assert(inputs.size() >= 2);
outputs.assign(1, inputs[0]);
internals.clear();
return true;
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs,
const int requiredInternals,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
CV_Assert(!inputs.empty());
outputs.assign(requiredOutputs, inputs[0]);
internals.clear();
}
int getLayouts(const std::vector<DataLayout>& actualInputs,
std::vector<DataLayout>& desiredInputs,
const int requiredOutputs,
std::vector<DataLayout>& outputs) const CV_OVERRIDE
{
auto* netimpl_ = getNetImpl(this);
DataLayout defaultLayout = netimpl_->originalLayout;
size_t ninputs = actualInputs.size(), nblockInputs = 0;
CV_Assert(ninputs >= 1u);
for (size_t i = 0; i < ninputs; i++)
nblockInputs += actualInputs[i] == DATA_LAYOUT_BLOCK;
desiredInputs = actualInputs;
if (nblockInputs == ninputs) {
outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK);
} else {
if (nblockInputs < ninputs) {
for (size_t i = 0; i < ninputs; i++) {
DataLayout layout = actualInputs[i];
desiredInputs[i] = layout == DATA_LAYOUT_BLOCK ? defaultLayout : layout;
}
}
outputs.assign(requiredOutputs, DATA_LAYOUT_UNKNOWN);
}
return outputs[0] == DATA_LAYOUT_BLOCK ? netimpl_->defaultC0 : 0;
}
void forward(InputArrayOfArrays input_arrs,
OutputArrayOfArrays output_arrs,
OutputArrayOfArrays) CV_OVERRIDE
{
ensureCoeffs();
int ninputs = (int)input_arrs.total();
CV_Assert(ninputs >= 2 && coeffs.size() == (size_t)ninputs);
const Mat& inp0 = input_arrs.getMat(0);
MatShape outshape = inp0.shape();
int outtype = inp0.type();
CV_Assert(outtype == CV_8SC1 || outtype == CV_8UC1);
const bool isU8 = (outtype == CV_8UC1);
int outkind = output_arrs.kind();
std::vector<Mat>* outs = nullptr;
std::vector<UMat>* uouts = nullptr;
Mat out;
if (outkind == _InputArray::STD_VECTOR_MAT) {
outs = &output_arrs.getMatVecRef();
outs->resize(1);
outs->at(0).fit(outshape, outtype);
out = outs->at(0);
} else {
uouts = &output_arrs.getUMatVecRef();
uouts->resize(1);
uouts->at(0).fit(outshape, outtype);
out.fit(outshape, outtype);
}
const size_t total_elems = inp0.total();
const float* cptr = coeffs.data();
const float off = offset;
const int8_t* lutptr = !activationLUT.empty() ? activationLUT.ptr<int8_t>() : nullptr;
const float c0 = cptr[0];
const float c1 = cptr[1];
const int out_min = isU8 ? (withRelu ? output_zp : 0) : (withRelu ? output_zp : -128);
const int out_max = isU8 ? 255 : 127;
const double nstripes = (double)std::max(1, getNumThreads() * 4);
if (isU8) {
std::vector<const uint8_t*> inptrs(ninputs);
for (int k = 0; k < ninputs; k++)
inptrs[k] = input_arrs.getMat(k).ptr<uint8_t>();
const uint8_t* p0 = inptrs[0];
const uint8_t* p1 = inptrs[1];
uint8_t* outptr = out.ptr<uint8_t>();
parallel_for_(Range(0, (int)total_elems), [&](const Range& r) {
int i = r.start;
#if CV_AVX2
if (!lutptr && ninputs == 2) {
__m256 vc0 = _mm256_set1_ps(c0);
__m256 vc1 = _mm256_set1_ps(c1);
__m256 voff = _mm256_set1_ps(off);
__m256i vmin = _mm256_set1_epi32(out_min);
__m256i vmax = _mm256_set1_epi32(out_max);
for (; i <= r.end - 8; i += 8) {
__m128i a8 = _mm_loadl_epi64((const __m128i*)(p0 + i));
__m128i b8 = _mm_loadl_epi64((const __m128i*)(p1 + i));
__m256i a32 = _mm256_cvtepu8_epi32(a8);
__m256i b32 = _mm256_cvtepu8_epi32(b8);
__m256 af = _mm256_cvtepi32_ps(a32);
__m256 bf = _mm256_cvtepi32_ps(b32);
__m256 val = _mm256_fmadd_ps(vc0, af, _mm256_fmadd_ps(vc1, bf, voff));
__m256i ival = _mm256_cvtps_epi32(val);
ival = _mm256_max_epi32(ival, vmin);
ival = _mm256_min_epi32(ival, vmax);
__m128i lo = _mm256_castsi256_si128(ival);
__m128i hi = _mm256_extracti128_si256(ival, 1);
__m128i packed16 = _mm_packs_epi32(lo, hi);
__m128i packed8 = _mm_packus_epi16(packed16, packed16);
_mm_storel_epi64((__m128i*)(outptr + i), packed8);
}
}
#elif CV_NEON
if (ninputs == 2) {
float32x4_t vc0 = vdupq_n_f32(c0);
float32x4_t vc1 = vdupq_n_f32(c1);
float32x4_t voff = vdupq_n_f32(off);
int32x4_t vmin = vdupq_n_s32(out_min);
int32x4_t vmax = vdupq_n_s32(out_max);
for (; i <= r.end - 8; i += 8) {
uint8x8_t a8 = vld1_u8(p0 + i);
uint8x8_t b8 = vld1_u8(p1 + i);
uint16x8_t a16 = vmovl_u8(a8);
uint16x8_t b16 = vmovl_u8(b8);
int32x4_t a32_lo = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(a16)));
int32x4_t b32_lo = vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(b16)));
int32x4_t a32_hi = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(a16)));
int32x4_t b32_hi = vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(b16)));
float32x4_t val_lo = vaddq_f32(vaddq_f32(voff,
vmulq_f32(vc0, vcvtq_f32_s32(a32_lo))),
vmulq_f32(vc1, vcvtq_f32_s32(b32_lo)));
float32x4_t val_hi = vaddq_f32(vaddq_f32(voff,
vmulq_f32(vc0, vcvtq_f32_s32(a32_hi))),
vmulq_f32(vc1, vcvtq_f32_s32(b32_hi)));
#if CV_NEON_AARCH64
int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_lo), vmin), vmax);
int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_hi), vmin), vmax);
#else
float32x4_t half = vdupq_n_f32(0.5f);
int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_lo,
vbslq_f32(vcgeq_f32(val_lo, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax);
int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_hi,
vbslq_f32(vcgeq_f32(val_hi, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax);
#endif
int16x8_t p16 = vcombine_s16(vqmovn_s32(ival_lo), vqmovn_s32(ival_hi));
uint8x8_t p8u = vqmovun_s16(p16);
if (lutptr) {
uint8_t tmp[8];
vst1_u8(tmp, p8u);
for (int k = 0; k < 8; k++)
outptr[i + k] = (uint8_t)lutptr[tmp[k]];
} else {
vst1_u8(outptr + i, p8u);
}
}
}
#endif
for (; i < r.end; i++)
{
float val = c0 * (float)p0[i] + c1 * (float)p1[i] + off;
for (int k = 2; k < ninputs; k++)
val += cptr[k] * (float)inptrs[k][i];
int ival = cvRound(val);
ival = std::max(out_min, std::min(out_max, ival));
if (lutptr)
ival = (int)(uint8_t)lutptr[ival];
outptr[i] = (uint8_t)ival;
}
}, nstripes);
} else {
std::vector<const int8_t*> inptrs(ninputs);
for (int k = 0; k < ninputs; k++)
inptrs[k] = input_arrs.getMat(k).ptr<int8_t>();
const int8_t* p0 = inptrs[0];
const int8_t* p1 = inptrs[1];
int8_t* outptr = out.ptr<int8_t>();
parallel_for_(Range(0, (int)total_elems), [&](const Range& r) {
int i = r.start;
#if CV_AVX2
if (!lutptr && ninputs == 2) {
__m256 vc0 = _mm256_set1_ps(c0);
__m256 vc1 = _mm256_set1_ps(c1);
__m256 voff = _mm256_set1_ps(off);
__m256i vmin = _mm256_set1_epi32(out_min);
__m256i vmax = _mm256_set1_epi32(out_max);
for (; i <= r.end - 8; i += 8) {
__m128i a8 = _mm_loadl_epi64((const __m128i*)(p0 + i));
__m128i b8 = _mm_loadl_epi64((const __m128i*)(p1 + i));
__m256i a32 = _mm256_cvtepi8_epi32(a8);
__m256i b32 = _mm256_cvtepi8_epi32(b8);
__m256 af = _mm256_cvtepi32_ps(a32);
__m256 bf = _mm256_cvtepi32_ps(b32);
__m256 val = _mm256_fmadd_ps(vc0, af, _mm256_fmadd_ps(vc1, bf, voff));
__m256i ival = _mm256_cvtps_epi32(val);
ival = _mm256_max_epi32(ival, vmin);
ival = _mm256_min_epi32(ival, vmax);
__m128i lo = _mm256_castsi256_si128(ival);
__m128i hi = _mm256_extracti128_si256(ival, 1);
__m128i packed16 = _mm_packs_epi32(lo, hi);
__m128i packed8 = _mm_packs_epi16(packed16, packed16);
_mm_storel_epi64((__m128i*)(outptr + i), packed8);
}
}
#elif CV_NEON
if (ninputs == 2) {
float32x4_t vc0 = vdupq_n_f32(c0);
float32x4_t vc1 = vdupq_n_f32(c1);
float32x4_t voff = vdupq_n_f32(off);
int32x4_t vmin = vdupq_n_s32(out_min);
int32x4_t vmax = vdupq_n_s32(out_max);
for (; i <= r.end - 8; i += 8) {
int8x8_t a8 = vld1_s8(p0 + i);
int8x8_t b8 = vld1_s8(p1 + i);
int16x8_t a16 = vmovl_s8(a8);
int16x8_t b16 = vmovl_s8(b8);
int32x4_t a32_lo = vmovl_s16(vget_low_s16(a16));
int32x4_t b32_lo = vmovl_s16(vget_low_s16(b16));
int32x4_t a32_hi = vmovl_s16(vget_high_s16(a16));
int32x4_t b32_hi = vmovl_s16(vget_high_s16(b16));
float32x4_t val_lo = vaddq_f32(vaddq_f32(voff,
vmulq_f32(vc0, vcvtq_f32_s32(a32_lo))),
vmulq_f32(vc1, vcvtq_f32_s32(b32_lo)));
float32x4_t val_hi = vaddq_f32(vaddq_f32(voff,
vmulq_f32(vc0, vcvtq_f32_s32(a32_hi))),
vmulq_f32(vc1, vcvtq_f32_s32(b32_hi)));
#if CV_NEON_AARCH64
int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_lo), vmin), vmax);
int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtnq_s32_f32(val_hi), vmin), vmax);
#else
float32x4_t half = vdupq_n_f32(0.5f);
int32x4_t ival_lo = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_lo,
vbslq_f32(vcgeq_f32(val_lo, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax);
int32x4_t ival_hi = vminq_s32(vmaxq_s32(vcvtq_s32_f32(vaddq_f32(val_hi,
vbslq_f32(vcgeq_f32(val_hi, vdupq_n_f32(0.f)), half, vnegq_f32(half)))), vmin), vmax);
#endif
int16x8_t p16 = vcombine_s16(vqmovn_s32(ival_lo), vqmovn_s32(ival_hi));
int8x8_t p8 = vqmovn_s16(p16);
if (lutptr) {
int8_t tmp[8];
vst1_s8(tmp, p8);
for (int k = 0; k < 8; k++)
outptr[i + k] = (int8_t)lutptr[(int)tmp[k] + 128];
} else {
vst1_s8(outptr + i, p8);
}
}
}
#endif
for (; i < r.end; i++)
{
float val = c0 * (float)p0[i] + c1 * (float)p1[i] + off;
for (int k = 2; k < ninputs; k++)
val += cptr[k] * (float)inptrs[k][i];
int ival = cvRound(val);
ival = std::max(out_min, std::min(out_max, ival));
if (lutptr)
ival = (int)lutptr[ival + 128];
outptr[i] = (int8_t)ival;
}
}, nstripes);
}
if (uouts) {
out.copyTo(uouts->at(0));
}
}
bool setActivation(const Ptr<ActivationLayer>& layer) CV_OVERRIDE
{
Ptr<ActivationLayerInt8> activ_int8 = layer.dynamicCast<ActivationLayerInt8>();
if (!activ_int8.empty())
{
activ = activ_int8;
if (!activ_int8->blobs.empty())
activationLUT = activ_int8->blobs[0];
return true;
}
return false;
}
};
Ptr<Eltwise2Int8Layer> Eltwise2Int8Layer::create(const LayerParams& params)
{
return Ptr<Eltwise2Int8Layer>(new Eltwise2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/int8layers/fully_connected_layer.cpp
+22 -3
View File
@@ -28,6 +28,7 @@ public:
output_sc = params.get<float>("scales", 1.0f);
axis = params.get<int>("axis", 1);
per_channel = params.get<bool>("per_channel", true);
output_type = CV_8S; // default, may be overridden by fusion code
if (blobs.size() == 3)
{
@@ -77,6 +78,16 @@ public:
return false;
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs,
const int requiredInternals,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
outputs.assign(requiredOutputs, output_type);
internals.clear();
}
virtual bool supportBackend(int backendId) CV_OVERRIDE
{
if (backendId == DNN_BACKEND_TIMVX && haveTimVX())
@@ -383,14 +394,22 @@ public:
int axisCan = normalize_axis(axis, input[0].dims);
int outerSize = input[0].total(0, axisCan);
Mat srcMat = input[0].reshape(1, outerSize);
Mat srcMat0 = input[0].reshape(1, outerSize);
Mat srcMat;
if (srcMat0.type() == CV_8U) {
// Convert uint8 to int8 by subtracting 128.
// Zero-point was adjusted at fusion time to compensate.
srcMat0.convertTo(srcMat, CV_8S, 1, -128);
} else {
srcMat = srcMat0;
}
Mat dstMat = output[0].reshape(1, outerSize);
Mat dstMatInt32= Mat(shape(dstMat), CV_32S);
const int nstripes = getNumThreads();
const int nstripes = outerSize <= 4 ? 1 : getNumThreads();
FullyConnected::run(srcMat, weightsMat, biasMat, outputMultiplier, activationLUT, dstMatInt32, activ.get(), nstripes, output_zp);
dstMatInt32.convertTo(dstMat, CV_8S);
dstMatInt32.convertTo(dstMat, output_type);
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
modules/dnn/src/int8layers/pool2_int8_layer.cpp
+381
View File
@@ -0,0 +1,381 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners
#include "../precomp.hpp"
#include "../net_impl.hpp"
#include "../layers/conv2_common.hpp"
#include "opencv2/core/hal/intrin.hpp"
namespace cv {
namespace dnn {
template <typename T>
static void maxPoolImpl(const void* inp_, void* out_, const ConvState& cs)
{
constexpr int MAX_POOL_DIMS = ConvState::MAX_CONV_DIMS;
int NC1 = cs.inpshape[0] * cs.inpshape[1];
CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.inpshape.dims == cs.outshape.dims);
parallel_for_(Range(0, NC1), [&](const Range& r) {
CV_Assert(cs.nspatialdims <= MAX_POOL_DIMS && MAX_POOL_DIMS == 3);
int sdims = cs.nspatialdims;
int nc0 = r.start, nc1 = r.end;
int C0 = cs.inpshape.back();
int Di = sdims > 2 ? cs.inpshape[sdims - 1] : 1;
int Hi = sdims > 1 ? cs.inpshape[sdims] : 1;
int Wi = cs.inpshape[sdims + 1];
int D = sdims > 2 ? cs.outshape[sdims - 1] : 1;
int H = sdims > 1 ? cs.outshape[sdims] : 1;
int W = cs.outshape[sdims + 1];
int iplanesize = Di * Hi * Wi * C0;
int planesize = D * H * W * C0;
int SZ = cs.strides[0], SY = cs.strides[1], SX = cs.strides[2];
int padZ0 = cs.pads[0], padY0 = cs.pads[1], padX0 = cs.pads[2];
int inner_z0 = cs.inner[0], inner_z1 = cs.inner[MAX_POOL_DIMS];
int inner_y0 = cs.inner[1], inner_y1 = cs.inner[MAX_POOL_DIMS + 1];
int inner_x0 = cs.inner[2], inner_x1 = cs.inner[MAX_POOL_DIMS + 2];
int ksize = (int)cs.ofstab.size();
const int* zyxtab = cs.coordtab.data();
const int* ofstab = cs.ofstab.data();
const T* inp = (const T*)inp_ + nc0 * iplanesize;
T* out = (T*)out_ + nc0 * planesize;
const T INITVAL = std::numeric_limits<T>::min();
for (int nc = nc0; nc < nc1; nc++, inp += iplanesize) {
for (int z0 = 0; z0 < D; z0++) {
int zi_ = z0 * SZ - padZ0;
for (int y0 = 0; y0 < H; y0++, out += W * C0) {
int x0 = 0;
int x1 = z0 >= inner_z0 && z0 < inner_z1 &&
y0 >= inner_y0 && y0 < inner_y1 ? inner_x0 : W;
int yi_ = y0 * SY - padY0;
for (;;) {
for (; x0 < x1; x0++) {
int xi_ = x0 * SX - padX0;
for (int c = 0; c < C0; c++)
out[x0 * C0 + c] = INITVAL;
for (int k = 0; k < ksize; k++) {
int zi = zi_ + zyxtab[k * MAX_POOL_DIMS];
int yi = yi_ + zyxtab[k * MAX_POOL_DIMS + 1];
int xi = xi_ + zyxtab[k * MAX_POOL_DIMS + 2];
if ((unsigned)zi >= (unsigned)Di ||
(unsigned)yi >= (unsigned)Hi ||
(unsigned)xi >= (unsigned)Wi)
continue;
const T* inptr = inp + ((zi * Hi + yi) * Wi + xi) * C0;
for (int c = 0; c < C0; c++)
out[x0 * C0 + c] = std::max(out[x0 * C0 + c], inptr[c]);
}
}
if (x0 == W)
break;
x1 = inner_x1;
for (; x0 < x1; x0++) {
int xi_ = x0 * SX - padX0;
const T* inp_xi = inp + ((Hi * zi_ + yi_) * Wi + xi_) * C0;
for (int c = 0; c < C0; c++) {
T s = inp_xi[ofstab[0] + c];
for (int k = 1; k < ksize; k++)
s = std::max(s, inp_xi[ofstab[k] + c]);
out[x0 * C0 + c] = s;
}
}
x1 = W;
}
}
}
}
});
}
static void maxPoolInt8(const void* inp_, void* out_, const ConvState& cs, bool isU8)
{
if (isU8)
maxPoolImpl<uint8_t>(inp_, out_, cs);
else
maxPoolImpl<int8_t>(inp_, out_, cs);
}
static void avgPoolInt8(const void* inp_, void* out_, const ConvState& cs,
float inp_sc, int inp_zp, float out_sc, int out_zp,
bool count_include_pad_, bool isU8)
{
constexpr int MAX_POOL_DIMS = ConvState::MAX_CONV_DIMS;
int NC1 = cs.inpshape[0] * cs.inpshape[1];
CV_Assert(cs.inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.outshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(cs.inpshape.dims == cs.outshape.dims);
parallel_for_(Range(0, NC1), [&](const Range& r) {
CV_Assert(cs.nspatialdims <= MAX_POOL_DIMS && MAX_POOL_DIMS == 3);
bool count_include_pad = count_include_pad_;
int sdims = cs.nspatialdims;
int nc0 = r.start, nc1 = r.end;
int C0 = cs.inpshape.back();
int Di = sdims > 2 ? cs.inpshape[sdims - 1] : 1;
int Hi = sdims > 1 ? cs.inpshape[sdims] : 1;
int Wi = cs.inpshape[sdims + 1];
int D = sdims > 2 ? cs.outshape[sdims - 1] : 1;
int H = sdims > 1 ? cs.outshape[sdims] : 1;
int W = cs.outshape[sdims + 1];
int iplanesize = Di * Hi * Wi * C0;
int planesize = D * H * W * C0;
int SZ = cs.strides[0], SY = cs.strides[1], SX = cs.strides[2];
int padZ0 = cs.pads[0], padY0 = cs.pads[1], padX0 = cs.pads[2];
int ksize_total = (int)cs.ofstab.size();
const int* zyxtab = cs.coordtab.data();
const uint8_t* inp = (const uint8_t*)inp_ + nc0 * iplanesize;
uint8_t* out = (uint8_t*)out_ + nc0 * planesize;
float scale_ratio = inp_sc / out_sc;
std::vector<int> accum(C0);
for (int nc = nc0; nc < nc1; nc++, inp += iplanesize) {
for (int z0 = 0; z0 < D; z0++) {
int zi_ = z0 * SZ - padZ0;
for (int y0 = 0; y0 < H; y0++, out += W * C0) {
int yi_ = y0 * SY - padY0;
for (int x0 = 0; x0 < W; x0++) {
int xi_ = x0 * SX - padX0;
for (int c = 0; c < C0; c++)
accum[c] = 0;
int count = 0;
for (int k = 0; k < ksize_total; k++) {
int zi = zi_ + zyxtab[k * MAX_POOL_DIMS];
int yi = yi_ + zyxtab[k * MAX_POOL_DIMS + 1];
int xi = xi_ + zyxtab[k * MAX_POOL_DIMS + 2];
if ((unsigned)zi >= (unsigned)Di ||
(unsigned)yi >= (unsigned)Hi ||
(unsigned)xi >= (unsigned)Wi) {
if (count_include_pad)
count++;
continue;
}
count++;
const uint8_t* inptr = inp + ((zi * Hi + yi) * Wi + xi) * C0;
for (int c = 0; c < C0; c++) {
int v = isU8 ? (int)inptr[c]
: (int)(int8_t)inptr[c];
accum[c] += v - inp_zp;
}
}
if (count == 0) count = 1;
float inv_count = 1.f / count;
for (int c = 0; c < C0; c++) {
float val = (float)accum[c] * inv_count * scale_ratio + (float)out_zp;
int ival = cvRound(val);
if (isU8)
out[x0 * C0 + c] = (uint8_t)std::max(0, std::min(255, ival));
else
out[x0 * C0 + c] = (uint8_t)(int8_t)std::max(-128, std::min(127, ival));
}
}
}
}
}
});
}
static void globalAvgPoolInt8(const void* inp_, void* out_,
int N, int C1, int spatialSize, int C0,
float inp_sc, int inp_zp, float out_sc, int out_zp,
bool isU8)
{
int NC1 = N * C1;
float scale_ratio = inp_sc / out_sc;
parallel_for_(Range(0, NC1), [&](const Range& r) {
for (int nc = r.start; nc < r.end; nc++) {
const uint8_t* inptr = (const uint8_t*)inp_ + (size_t)nc * spatialSize * C0;
uint8_t* outptr = (uint8_t*)out_ + (size_t)nc * C0;
for (int c = 0; c < C0; c++) {
int sum = 0;
for (int s = 0; s < spatialSize; s++) {
int v = isU8 ? (int)inptr[s * C0 + c]
: (int)(int8_t)inptr[s * C0 + c];
sum += v - inp_zp;
}
float val = (float)sum / spatialSize * scale_ratio + (float)out_zp;
int ival = cvRound(val);
if (isU8)
outptr[c] = (uint8_t)std::max(0, std::min(255, ival));
else
outptr[c] = (uint8_t)(int8_t)std::max(-128, std::min(127, ival));
}
}
});
}
class Pool2Int8LayerImpl CV_FINAL : public Pool2Int8Layer
{
public:
bool count_include_pad;
ConvState cs;
MatShape prevInpshape;
Pool2Int8LayerImpl(const LayerParams& params)
{
setParamsFrom(params);
auto_pad = getAutoPadding(params);
kernel_shape = params.getVector<int>("kernel_size");
strides = params.getVector<int>("stride");
dilations = params.getVector<int>("dilation");
pads = params.getVector<int>("pad");
ceil_mode = params.get<bool>("ceil_mode", false);
is_global_pooling = params.get<bool>("global_pooling", false);
is_max_pool = params.get<bool>("is_max_pool", true);
count_include_pad = params.get<bool>("count_include_pad", false);
input_sc = params.get<float>("input_scale", 1.f);
input_zp = params.get<int>("input_zeropoint", 0);
output_sc = params.get<float>("scales", 1.f);
output_zp = params.get<int>("zeropoints", 0);
}
void getTypes(const std::vector<MatType>& inputs,
const int requiredOutputs, const int,
std::vector<MatType>& outputs,
std::vector<MatType>& internals) const CV_OVERRIDE
{
CV_Assert(!inputs.empty());
outputs.assign(requiredOutputs, inputs[0]);
internals.clear();
}
bool getMemoryShapes(const std::vector<MatShape>& inpshapes,
const int,
std::vector<MatShape>& outshapes,
std::vector<MatShape>& tempshapes) const CV_OVERRIDE
{
CV_Assert(inpshapes.size() == 1);
if (is_global_pooling) {
MatShape outshape = inpshapes[0];
for (int d = 2; d < outshape.dims - 1; d++)
outshape[d] = 1;
outshapes.assign(1, outshape);
} else {
outshapes.assign(1, convInferShape(inpshapes[0], MatShape(),
kernel_shape, 0, strides, dilations,
pads, auto_pad, ceil_mode));
}
tempshapes.clear();
return true;
}
int getLayouts(const std::vector<DataLayout>& actualInputs,
std::vector<DataLayout>& desiredInputs,
const int requiredOutputs,
std::vector<DataLayout>& outputs) const CV_OVERRIDE
{
CV_Assert(actualInputs.size() == 1u);
desiredInputs.assign(1, DATA_LAYOUT_BLOCK);
outputs.assign(requiredOutputs, DATA_LAYOUT_BLOCK);
return getNetImpl(this)->defaultC0;
}
void forward(InputArrayOfArrays inputs_arr,
OutputArrayOfArrays outputs_arr,
OutputArrayOfArrays) CV_OVERRIDE
{
size_t ninputs = inputs_arr.total();
CV_Assert(ninputs == 1);
const Mat& inp = inputs_arr.getMat(0);
int inptype = inp.type();
MatShape inpshape = inp.shape();
CV_Assert(inpshape.layout == DATA_LAYOUT_BLOCK);
CV_Assert(inptype == CV_8SC1 || inptype == CV_8UC1);
MatShape outshape;
if (is_global_pooling) {
outshape = inpshape;
for (int d = 2; d < outshape.dims - 1; d++)
outshape[d] = 1;
} else {
outshape = convInferShape(inpshape, MatShape(),
kernel_shape, 0, strides, dilations,
pads, auto_pad, ceil_mode);
}
int outkind = outputs_arr.kind();
Mat out;
if (outkind == _InputArray::STD_VECTOR_MAT) {
std::vector<Mat>& outs = outputs_arr.getMatVecRef();
outs.resize(1);
outs[0].fit(outshape, inptype);
out = outs[0];
} else {
std::vector<UMat>& outs = outputs_arr.getUMatVecRef();
outs.resize(1);
outs[0].fit(outshape, inptype);
out.fit(outshape, inptype);
}
if (is_global_pooling) {
int N = inpshape[0];
int C1 = inpshape[1];
int C0 = inpshape.back();
int spatialSize = 1;
for (int d = 2; d < inpshape.dims - 1; d++)
spatialSize *= inpshape[d];
globalAvgPoolInt8(inp.data, out.data, N, C1, spatialSize, C0,
input_sc, input_zp, output_sc, output_zp,
inptype == CV_8UC1);
} else {
ConvState cs_local;
cs_local.initPooling(inpshape, outshape, kernel_shape, strides,
dilations, pads, auto_pad, ceil_mode);
if (is_max_pool) {
maxPoolInt8(inp.data, out.data, cs_local, inptype == CV_8UC1);
} else {
avgPoolInt8(inp.data, out.data, cs_local,
input_sc, input_zp, output_sc, output_zp,
count_include_pad, inptype == CV_8UC1);
}
}
if (outkind == _InputArray::STD_VECTOR_UMAT) {
std::vector<UMat>& outs = outputs_arr.getUMatVecRef();
out.copyTo(outs[0]);
}
}
};
Ptr<Pool2Int8Layer> Pool2Int8Layer::create(const LayerParams& params)
{
return Ptr<Pool2Int8Layer>(new Pool2Int8LayerImpl(params));
}
} // namespace dnn
} // namespace cv
modules/dnn/src/layers/dequantizelinear_layer.cpp
+39 -30
View File
@@ -1,37 +1,54 @@
// This file is part of OpenCV project.
// It is subject to the license terms in the LICENSE file found in the top-level directory
// of this distribution and at http://opencv.org/license.html.
// Copyright (C) 2026, BigVision LLC, all rights reserved.
// Third party copyrights are property of their respective owners.
#include "../precomp.hpp"
#include "layers_common.hpp"
#include "../net_impl.hpp"
#if defined(__x86_64__) || defined(_M_X64)
#include <immintrin.h>
#endif
namespace cv
{
namespace dnn
{
#if CV_SIMD || CV_SIMD_SCALABLE
static void dequantizeLinearChunk_u8_f32(const uint8_t* src, float* dst,
float scale, int zp, int64_t len)
/*
DequantizeLinear layer, as defined in ONNX specification:
https://onnx.ai/onnx/operators/onnx__DequantizeLinear.html
Opset's 10 to 25 are covered.
*/
#if defined(__x86_64__) || defined(_M_X64)
#if defined(__GNUC__) || defined(__clang__)
__attribute__((target("avx2")))
#endif
static void dequantizeLinearChunk_u8_f32_avx2(const uint8_t* src, float* dst,
float scale, int zp, int64_t len)
{
const int vlanes = VTraits<v_float32>::vlanes();
v_float32 vscale = vx_setall_f32(scale);
v_int32 vzp = vx_setall_s32(zp);
__m256 vscale = _mm256_set1_ps(scale);
__m256i vzp = _mm256_set1_epi32(zp);
int64_t j = 0;
for (; j <= len - vlanes; j += vlanes) {
v_int32 vi = v_reinterpret_as_s32(vx_load_expand_q(src + j));
vi = v_sub(vi, vzp);
v_float32 vf = v_mul(v_cvt_f32(vi), vscale);
v_store(dst + j, vf);
for (; j <= len - 8; j += 8) {
__m128i raw = _mm_loadl_epi64((__m128i*)(src + j));
__m256i vi = _mm256_cvtepu8_epi32(raw);
vi = _mm256_sub_epi32(vi, vzp);
__m256 vf = _mm256_cvtepi32_ps(vi);
vf = _mm256_mul_ps(vf, vscale);
_mm256_storeu_ps(dst + j, vf);
}
for (; j < len; j++)
dst[j] = (float)(src[j] - zp) * scale;
}
static void dequantizeLinearFast_u8_f32(const uint8_t* inp, float* out,
float scale, int zp,
int64_t total)
static void dequantizeLinearFast_u8_f32_avx2(const uint8_t* inp, float* out,
float scale, int zp,
int64_t total)
{
const int64_t block = 1024;
int64_t nblocks = (total + block - 1) / block;
@@ -40,20 +57,12 @@ static void dequantizeLinearFast_u8_f32(const uint8_t* inp, float* out,
for (int i = r.start; i < r.end; i++) {
int64_t ofs = i * block;
int64_t len = std::min(block, total - ofs);
dequantizeLinearChunk_u8_f32(inp + ofs, out + ofs, scale, zp, len);
dequantizeLinearChunk_u8_f32_avx2(inp + ofs, out + ofs, scale, zp, len);
}
});
}
#endif
/*
DequantizeLinear layer, as defined in ONNX specification:
https://onnx.ai/onnx/operators/onnx__DequantizeLinear.html
Opset's 10 to 23 are covered.
*/
template <typename _InpTp, typename _ScaleTp, typename _OutTp>
static void dequantizeLinear(const _InpTp* inp_, const _ScaleTp* scale_,
const _InpTp* zp_, _OutTp* out_,
@@ -91,7 +100,6 @@ static void dequantizeLinear(const _InpTp* inp_, const _ScaleTp* scale_,
const _ScaleTp* sc = scale_ + scale_ofs;
_OutTp* out = out_ + block_ofs;
// [TODO] vectorize using intrinsics
if (slice_size > 1) {
for (int k = 0; k < delta; k++, inp += slice_size, out += slice_size,
sc += scale_step, zp += zp_step) {
@@ -201,15 +209,16 @@ static void dequantizeLinear(const Mat& inp, const Mat& scale_, const Mat& zp,
}
}
// Fast path: per-tensor dequantization uint8→float with universal intrinsics
#if CV_SIMD || CV_SIMD_SCALABLE
if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F) {
// Fast path: per-tensor dequantization uint8→float with AVX2 + proper parallelism
#if defined(__x86_64__) || defined(_M_X64)
if (block_size == 0 && sz_a == 1 && inptype == CV_8U && outtype == CV_32F && sctype == CV_32F
&& checkHardwareSupport(CV_CPU_AVX2)) {
float sc = reinterpret_cast<const float*>(scale.data)[0];
int zpval = zp.empty() ? 0 : (int)reinterpret_cast<const uint8_t*>(zp.data)[0];
int64_t total = nslices * slice_size;
dequantizeLinearFast_u8_f32(reinterpret_cast<const uint8_t*>(inp.data),
reinterpret_cast<float*>(out.data),
sc, zpval, total);
dequantizeLinearFast_u8_f32_avx2(reinterpret_cast<const uint8_t*>(inp.data),
reinterpret_cast<float*>(out.data),
sc, zpval, total);
return;
}
#endif
modules/dnn/test/test_model.cpp
+57
View File
@@ -921,4 +921,61 @@ TEST_P(Reproducibility_ResNet50_ONNX, Accuracy)
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_ONNX,
testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)));
typedef testing::TestWithParam<Target> Reproducibility_ResNet50_QDQ_ONNX;
TEST_P(Reproducibility_ResNet50_QDQ_ONNX, Accuracy)
{
Target targetId = GetParam();
applyTestTag(targetId == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
ASSERT_TRUE(ocl::useOpenCL() || targetId == DNN_TARGET_CPU || targetId == DNN_TARGET_CPU_FP16);
std::string modelname = _tf("onnx/models/resnet50-v1-12-qdq.onnx", false);
Net net = readNetFromONNX(modelname);
net.setPreferableBackend(DNN_BACKEND_OPENCV);
net.setPreferableTarget(targetId);
if (targetId == DNN_TARGET_CPU_FP16)
net.enableWinograd(false);
std::string imgname = _tf("sqcat.png");
Mat image = imread(imgname);
Mat input = blobFromImage(image, 0.017, Size(224,224),
Scalar(103.939, 116.779, 123.68),
false, true, CV_32F);
ASSERT_TRUE(!input.empty());
Mat out;
double min_t = 0;
const int niters =
#ifdef _DEBUG
1;
#else
30;
#endif
for (int i = 0; i < niters; i++) {
double t = (double)getTickCount();
net.setInput(input);
out = net.forward();
t = (double)getTickCount() - t;
min_t = i == 0 ? t : std::min(min_t, t);
}
printf("run time = %.2fms\n", min_t*1000./getTickFrequency());
const int K = 5;
std::vector<std::pair<int, float> > res;
topK(out, res, K);
ASSERT_EQ(int(res.size()), K);
std::vector<std::pair<int, float> > ref = {{285, 10.44}, {287, 10.13}, {283, 8.89}, {278, 8.43}};
const float eps = 0.5f;
for (int i = 0; i < (int)ref.size(); i++) {
EXPECT_EQ(ref[i].first, res[i].first);
EXPECT_NEAR(ref[i].second, res[i].second, eps);
}
}
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50_QDQ_ONNX,
testing::ValuesIn(getAvailableTargets(DNN_BACKEND_OPENCV)));
}} // namespace