mirror of
https://github.com/opencv/opencv.git
synced 2026-07-21 19:33:03 +04:00
Merge pull request #29468 from aoguntayo:fix-mlas-s390x-missing-headers
mlas: add missing s390x ZVECTOR kernel headers to fix build
This commit is contained in:
Alexander Smorkalov
GitHub
+333
@@ -0,0 +1,333 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
FgemmKernelZVECTOR.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single/double precision matrix/matrix
|
||||
multiply operation (DGEMM/SGEMM).
|
||||
|
||||
--*/
|
||||
|
||||
#include "mlasi.h"
|
||||
#if defined(SINGLE)
|
||||
#define MLAS_FLOATTYPE MLAS_FLOAT32X4
|
||||
#define MLAS_GEMMTYPE float
|
||||
#define MLAS_LOAD_FLOAT MlasLoadFloat32x4
|
||||
#define MLAS_ZERO_FLOAT MlasZeroFloat32x4
|
||||
#define MLAS_STORE_FLOAT MlasStoreFloat32x4
|
||||
#define MLAS_EXTRACT_FLOAT MlasExtractLaneFloat32x4
|
||||
#define MLAS_MUL_FLOAT MlasMultiplyFloat32x4
|
||||
#define MLAS_MULADD_FLOAT MlasMultiplyAddFloat32x4
|
||||
#define MLAS_BROADCAST_FLOAT MlasBroadcastFloat32x4
|
||||
#else
|
||||
#define MLAS_FLOATTYPE MLAS_FLOAT64X2
|
||||
#define MLAS_GEMMTYPE double
|
||||
#define MLAS_LOAD_FLOAT MlasLoadFloat64x2
|
||||
#define MLAS_ZERO_FLOAT MlasZeroFloat64x2
|
||||
#define MLAS_STORE_FLOAT MlasStoreFloat64x2
|
||||
#define MLAS_EXTRACT_FLOAT MlasExtractLaneFloat64x2
|
||||
#define MLAS_MUL_FLOAT MlasMultiplyFloat64x2
|
||||
#define MLAS_MULADD_FLOAT MlasMultiplyAddFloat64x2
|
||||
#define MLAS_BROADCAST_FLOAT MlasBroadcastFloat64x2
|
||||
#endif
|
||||
//
|
||||
// Templates to ensure that a loop is unrolled.
|
||||
//
|
||||
|
||||
template<size_t Count, size_t Index>
|
||||
struct MlasLoopUnrollStep
|
||||
{
|
||||
template<typename IterationType, typename... IterationArgs>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Step(
|
||||
IterationArgs&&... Arguments
|
||||
)
|
||||
{
|
||||
IterationType::template Iteration<Count, Index>(Arguments...);
|
||||
MlasLoopUnrollStep<Count, Index + 1>::template Step<IterationType>(Arguments...);
|
||||
}
|
||||
};
|
||||
|
||||
template<size_t Count>
|
||||
struct MlasLoopUnrollStep<Count, Count>
|
||||
{
|
||||
template<typename IterationType, typename... IterationArgs>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Step(
|
||||
IterationArgs&&...
|
||||
)
|
||||
{
|
||||
// Terminate the loop.
|
||||
}
|
||||
};
|
||||
|
||||
template<size_t Count, typename IteratorType>
|
||||
struct MlasLoopUnroll
|
||||
{
|
||||
template<typename... IterationArgs>
|
||||
MLAS_FORCEINLINE
|
||||
void
|
||||
operator()(
|
||||
IterationArgs&&... Arguments
|
||||
)
|
||||
{
|
||||
MlasLoopUnrollStep<Count, 0>::template Step<IteratorType>(Arguments...);
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// Templates used with loop unrolling to perform an action on one row of the
|
||||
// output.
|
||||
//
|
||||
|
||||
struct MlasFgemmZeroAccumulators
|
||||
{
|
||||
template<size_t RowCount, size_t Row>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE Accumulators[RowCount][4]
|
||||
)
|
||||
{
|
||||
Accumulators[Row][0] = MLAS_ZERO_FLOAT();
|
||||
Accumulators[Row][1] = MLAS_ZERO_FLOAT();
|
||||
Accumulators[Row][2] = MLAS_ZERO_FLOAT();
|
||||
Accumulators[Row][3] = MLAS_ZERO_FLOAT();
|
||||
}
|
||||
};
|
||||
|
||||
struct MlasFgemmLoadAElements
|
||||
{
|
||||
template<size_t RowCount, size_t Row>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE AElements[RowCount],
|
||||
const MLAS_GEMMTYPE* A,
|
||||
size_t lda
|
||||
)
|
||||
{
|
||||
AElements[Row] = MLAS_LOAD_FLOAT(A + Row * lda);
|
||||
}
|
||||
};
|
||||
|
||||
struct MlasFgemmBroadcastAElements
|
||||
{
|
||||
template<size_t RowCount, size_t Row>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE ABroadcast[RowCount],
|
||||
const MLAS_GEMMTYPE* A,
|
||||
size_t lda
|
||||
)
|
||||
{
|
||||
ABroadcast[Row] = MLAS_BROADCAST_FLOAT(A + Row * lda);
|
||||
}
|
||||
};
|
||||
|
||||
template<unsigned Lane>
|
||||
struct MlasFgemmSplatAElements
|
||||
{
|
||||
template<size_t RowCount, size_t Row>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE AElements[RowCount],
|
||||
MLAS_FLOATTYPE ABroadcast[RowCount]
|
||||
)
|
||||
{
|
||||
ABroadcast[Row] = vec_splat(AElements[Row], Lane);
|
||||
}
|
||||
};
|
||||
|
||||
struct MlasFgemmMultiplyAddRow
|
||||
{
|
||||
template<size_t RowCount, size_t Row>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE Accumulators[RowCount][4],
|
||||
MLAS_FLOATTYPE ABroadcast[RowCount],
|
||||
MLAS_FLOATTYPE BElements[4]
|
||||
)
|
||||
{
|
||||
Accumulators[Row][0] = MLAS_MULADD_FLOAT(ABroadcast[Row], BElements[0], Accumulators[Row][0]);
|
||||
Accumulators[Row][1] = MLAS_MULADD_FLOAT(ABroadcast[Row], BElements[1], Accumulators[Row][1]);
|
||||
Accumulators[Row][2] = MLAS_MULADD_FLOAT(ABroadcast[Row], BElements[2], Accumulators[Row][2]);
|
||||
Accumulators[Row][3] = MLAS_MULADD_FLOAT(ABroadcast[Row], BElements[3], Accumulators[Row][3]);
|
||||
}
|
||||
};
|
||||
|
||||
template<size_t RowCount>
|
||||
MLAS_FORCEINLINE
|
||||
void
|
||||
MlasFgemmComputeBlock(
|
||||
MLAS_FLOATTYPE Accumulators[RowCount][4],
|
||||
MLAS_FLOATTYPE ABroadcast[RowCount],
|
||||
const MLAS_GEMMTYPE* B
|
||||
)
|
||||
{
|
||||
MLAS_FLOATTYPE BElements[4];
|
||||
#if defined(SINGLE)
|
||||
BElements[0] = MLAS_LOAD_FLOAT(B);
|
||||
BElements[1] = MLAS_LOAD_FLOAT(B + 4);
|
||||
BElements[2] = MLAS_LOAD_FLOAT(B + 8);
|
||||
BElements[3] = MLAS_LOAD_FLOAT(B + 12);
|
||||
#else
|
||||
BElements[0] = MLAS_LOAD_FLOAT(B);
|
||||
BElements[1] = MLAS_LOAD_FLOAT(B + 2);
|
||||
BElements[2] = MLAS_LOAD_FLOAT(B + 4);
|
||||
BElements[3] = MLAS_LOAD_FLOAT(B + 6);
|
||||
#endif
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmMultiplyAddRow>()(Accumulators, ABroadcast, BElements);
|
||||
}
|
||||
|
||||
struct MlasFgemmMultiplyAlphaRow
|
||||
{
|
||||
template<size_t Count, size_t Index>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE Accumulators[4],
|
||||
MLAS_FLOATTYPE AlphaBroadcast
|
||||
)
|
||||
{
|
||||
Accumulators[Index] = MLAS_MUL_FLOAT(Accumulators[Index], AlphaBroadcast);
|
||||
}
|
||||
};
|
||||
|
||||
struct MlasFgemmMultiplyAlphaAddRow
|
||||
{
|
||||
template<size_t Count, size_t Index>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE Accumulators[4],
|
||||
MLAS_FLOATTYPE AlphaBroadcast,
|
||||
const MLAS_GEMMTYPE* C
|
||||
)
|
||||
{
|
||||
#if defined(SINGLE)
|
||||
Accumulators[Index] = MLAS_MULADD_FLOAT(Accumulators[Index],
|
||||
AlphaBroadcast, MLAS_LOAD_FLOAT(C + Index * 4));
|
||||
#else
|
||||
Accumulators[Index] = MLAS_MULADD_FLOAT(Accumulators[Index],
|
||||
AlphaBroadcast, MLAS_LOAD_FLOAT(C + Index * 2));
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
struct MlasFgemmStoreRow
|
||||
{
|
||||
template<size_t Count, size_t Index>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE Accumulators[4],
|
||||
MLAS_GEMMTYPE* C
|
||||
)
|
||||
{
|
||||
#if defined(SINGLE)
|
||||
MLAS_STORE_FLOAT(C + Index * 4, Accumulators[Index]);
|
||||
#else
|
||||
MLAS_STORE_FLOAT(C + Index * 2, Accumulators[Index]);
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
template<size_t VectorCount>
|
||||
struct MlasFgemmStoreVector
|
||||
{
|
||||
template<size_t RowCount, size_t Row>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE Accumulators[RowCount][4],
|
||||
MLAS_GEMMTYPE* C,
|
||||
size_t ldc,
|
||||
MLAS_FLOATTYPE AlphaBroadcast,
|
||||
bool ZeroMode
|
||||
)
|
||||
{
|
||||
MLAS_GEMMTYPE* c = C + Row * ldc;
|
||||
|
||||
if (ZeroMode) {
|
||||
MlasLoopUnroll<VectorCount, MlasFgemmMultiplyAlphaRow>()(Accumulators[Row], AlphaBroadcast);
|
||||
} else {
|
||||
MlasLoopUnroll<VectorCount, MlasFgemmMultiplyAlphaAddRow>()(Accumulators[Row], AlphaBroadcast, c);
|
||||
}
|
||||
|
||||
MlasLoopUnroll<VectorCount, MlasFgemmStoreRow>()(Accumulators[Row], c);
|
||||
|
||||
//
|
||||
// Shift down any unaligned elements to the bottom for further processing.
|
||||
//
|
||||
|
||||
if (VectorCount < 4) {
|
||||
Accumulators[Row][0] = Accumulators[Row][VectorCount];
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct MlasFgemmMultiplyAlphaTrailing
|
||||
{
|
||||
template<size_t RowCount, size_t Row>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE Accumulators[RowCount][4],
|
||||
MLAS_FLOATTYPE AlphaBroadcast
|
||||
)
|
||||
{
|
||||
Accumulators[Row][0] = MLAS_MUL_FLOAT(Accumulators[Row][0], AlphaBroadcast);
|
||||
}
|
||||
};
|
||||
|
||||
template<unsigned Lane>
|
||||
struct MlasFgemmStoreScalar
|
||||
{
|
||||
template<size_t RowCount, size_t Row>
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
Iteration(
|
||||
MLAS_FLOATTYPE Accumulators[RowCount][4],
|
||||
MLAS_GEMMTYPE* C,
|
||||
size_t ldc,
|
||||
bool ZeroMode
|
||||
)
|
||||
{
|
||||
MLAS_GEMMTYPE* c = C + Row * ldc + Lane;
|
||||
MLAS_GEMMTYPE Value = MLAS_EXTRACT_FLOAT<Lane>(Accumulators[Row][0]);
|
||||
|
||||
if (!ZeroMode) {
|
||||
Value += *c;
|
||||
}
|
||||
|
||||
*c = Value;
|
||||
}
|
||||
};
|
||||
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelZVECTOR.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
--*/
|
||||
|
||||
#include "FgemmKernelZVECTOR.h"
|
||||
|
||||
template<size_t RowCount>
|
||||
MLAS_FORCEINLINE
|
||||
size_t
|
||||
MlasSgemmProcessCount(
|
||||
const float* A,
|
||||
const float* B,
|
||||
float* C,
|
||||
size_t CountK,
|
||||
size_t CountN,
|
||||
size_t lda,
|
||||
size_t ldc,
|
||||
MLAS_FLOAT32X4 AlphaBroadcast,
|
||||
bool ZeroMode
|
||||
)
|
||||
{
|
||||
do {
|
||||
|
||||
const float* a = A;
|
||||
size_t k = CountK;
|
||||
|
||||
MLAS_FLOAT32X4 Accumulators[RowCount][4];
|
||||
MLAS_FLOAT32X4 AElements[RowCount];
|
||||
MLAS_FLOAT32X4 ABroadcast[RowCount];
|
||||
|
||||
//
|
||||
// Clear the block accumulators.
|
||||
//
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmZeroAccumulators>()(Accumulators);
|
||||
|
||||
//
|
||||
// Compute the output block.
|
||||
//
|
||||
|
||||
while (k >= 4) {
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmLoadAElements>()(AElements, a, lda);
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmSplatAElements<0>>()(AElements, ABroadcast);
|
||||
MlasFgemmComputeBlock<RowCount>(Accumulators, ABroadcast, B);
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmSplatAElements<1>>()(AElements, ABroadcast);
|
||||
MlasFgemmComputeBlock<RowCount>(Accumulators, ABroadcast, B + 16);
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmSplatAElements<2>>()(AElements, ABroadcast);
|
||||
MlasFgemmComputeBlock<RowCount>(Accumulators, ABroadcast, B + 32);
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmSplatAElements<3>>()(AElements, ABroadcast);
|
||||
MlasFgemmComputeBlock<RowCount>(Accumulators, ABroadcast, B + 48);
|
||||
|
||||
a += 4;
|
||||
B += 16 * 4;
|
||||
k -= 4;
|
||||
}
|
||||
|
||||
while (k > 0) {
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmBroadcastAElements>()(ABroadcast, a, lda);
|
||||
MlasFgemmComputeBlock<RowCount>(Accumulators, ABroadcast, B);
|
||||
|
||||
a += 1;
|
||||
B += 16;
|
||||
k -= 1;
|
||||
}
|
||||
|
||||
if (CountN >= 16) {
|
||||
|
||||
//
|
||||
// Store the entire output block.
|
||||
//
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmStoreVector<4>>()(Accumulators, C, ldc, AlphaBroadcast, ZeroMode);
|
||||
|
||||
} else {
|
||||
|
||||
//
|
||||
// Store the partial output block.
|
||||
//
|
||||
|
||||
if (CountN >= 12) {
|
||||
MlasLoopUnroll<RowCount, MlasFgemmStoreVector<3>>()(Accumulators, C, ldc, AlphaBroadcast, ZeroMode);
|
||||
} else if (CountN >= 8) {
|
||||
MlasLoopUnroll<RowCount, MlasFgemmStoreVector<2>>()(Accumulators, C, ldc, AlphaBroadcast, ZeroMode);
|
||||
} else if (CountN >= 4) {
|
||||
MlasLoopUnroll<RowCount, MlasFgemmStoreVector<1>>()(Accumulators, C, ldc, AlphaBroadcast, ZeroMode);
|
||||
}
|
||||
|
||||
//
|
||||
// Store the remaining unaligned columns.
|
||||
//
|
||||
|
||||
C += (CountN & ~3);
|
||||
CountN &= 3;
|
||||
|
||||
if (CountN > 0) {
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmMultiplyAlphaTrailing>()(Accumulators, AlphaBroadcast);
|
||||
|
||||
MlasLoopUnroll<RowCount, MlasFgemmStoreScalar<0>>()(Accumulators, C, ldc, ZeroMode);
|
||||
|
||||
if (CountN >= 2) {
|
||||
MlasLoopUnroll<RowCount, MlasFgemmStoreScalar<1>>()(Accumulators, C, ldc, ZeroMode);
|
||||
}
|
||||
|
||||
if (CountN >= 3) {
|
||||
MlasLoopUnroll<RowCount, MlasFgemmStoreScalar<2>>()(Accumulators, C, ldc, ZeroMode);
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
C += 16;
|
||||
CountN -= 16;
|
||||
|
||||
} while (CountN > 0);
|
||||
|
||||
return RowCount;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user