mirror of
https://github.com/opencv/opencv.git
synced 2026-07-21 19:33:03 +04:00
Merge pull request #29226 from omrope79:fix_mlas_x86
Fix MLAS 32-bit x86 build by integrating missing upstream assembly files #29226 ### Pull Request Readiness Checklist This resolves the 32-bit x86 build failure for MLAS. Related to: https://github.com/opencv/opencv/pull/29218 * Added the required `x86` assembly files and headers from upstream. * Added `__x86_64__` guards around the AMX `syscall` and the FMA3/AVX512F kernel assignments to prevent 32-bit compilation crashes. See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
omrope79
GitHub
Vendored
+6
-15
@@ -83,15 +83,7 @@ set(_MLAS_REQUIRES_ASM FALSE)
|
||||
if(MLAS_X86_64 OR MLAS_X86 OR MLAS_ARM64 OR MLAS_LOONGARCH64)
|
||||
set(_MLAS_REQUIRES_ASM TRUE)
|
||||
endif()
|
||||
if(ANDROID AND _MLAS_REQUIRES_ASM)
|
||||
set(OPENCV_DNN_MLAS_SKIP_REASON
|
||||
"skipped on Android ${CMAKE_ANDROID_ARCH_ABI}; enable_language(ASM) from subdir is unreliable"
|
||||
CACHE INTERNAL "" FORCE)
|
||||
message(STATUS "MLAS: skipping on Android ${CMAKE_ANDROID_ARCH_ABI} "
|
||||
"(enable_language(ASM) from subdir is unreliable here); "
|
||||
"DNN will use its built-in SGEMM")
|
||||
return()
|
||||
endif()
|
||||
|
||||
include(CheckLanguage)
|
||||
set(MLAS_HAS_ASM FALSE CACHE INTERNAL "" FORCE)
|
||||
check_language(ASM)
|
||||
@@ -261,12 +253,11 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang"
|
||||
"$<$<COMPILE_LANGUAGE:ASM>:-Wa,--noexecstack>"
|
||||
)
|
||||
endif()
|
||||
# platform.cpp's __linux__ branch calls syscall() but upstream only includes
|
||||
# <sys/syscall.h>, not <unistd.h> (where syscall() is declared on glibc).
|
||||
# Force-include unistd.h on non-Windows x86_64 builds. MSVC doesn't ship
|
||||
# unistd.h and doesn't recognize the -include flag; its _WIN32 branch in
|
||||
# platform.cpp doesn't need syscall() anyway.
|
||||
if(MLAS_X86_64 AND NOT WIN32)
|
||||
# platform.cpp's MlasInitAMX() calls syscall() but only includes <sys/syscall.h>,
|
||||
# not <unistd.h> where glibc declares it. It's compiled on both x86 and x86_64
|
||||
# (shared MLAS_TARGET_AMD64_IX86 block), so force-include unistd.h on both.
|
||||
# MSVC has no unistd.h and its _WIN32 path doesn't use syscall() anyway.
|
||||
if((MLAS_X86_64 OR MLAS_X86) AND NOT WIN32)
|
||||
set_source_files_properties(${MLAS_SRC_DIR}/platform.cpp
|
||||
PROPERTIES COMPILE_FLAGS "-include unistd.h")
|
||||
endif()
|
||||
|
||||
Vendored
+2
-1
@@ -363,7 +363,7 @@ MLAS_PLATFORM::MLAS_PLATFORM(void)
|
||||
this->KernelM1Routine = MlasSgemmKernelM1Avx;
|
||||
this->KernelM1TransposeBRoutine = MlasSgemmKernelM1TransposeBAvx;
|
||||
this->TransposePackB16x4Routine = MlasSgemmTransposePackB16x4Avx;
|
||||
#endif
|
||||
// FMA3/AVX512F kernels exist for x86_64 only; 32-bit x86 stops at AVX.
|
||||
unsigned Cpuid7[4];
|
||||
#if defined(_WIN32)
|
||||
__cpuidex((int*)Cpuid7, 7, 0);
|
||||
@@ -378,6 +378,7 @@ MLAS_PLATFORM::MLAS_PLATFORM(void)
|
||||
this->GemmFloatKernel = MlasGemmFloatKernelAvx512F;
|
||||
}
|
||||
}
|
||||
#endif // MLAS_TARGET_AMD64
|
||||
}
|
||||
}
|
||||
#endif // MLAS_TARGET_AMD64_IX86
|
||||
|
||||
+435
@@ -0,0 +1,435 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelAvx.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
This implementation uses AVX instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
//
|
||||
// Stack frame layout for the SGEMM kernel.
|
||||
//
|
||||
|
||||
.equ .LSgemmKernelFrame_SavedEdi, 0
|
||||
.equ .LSgemmKernelFrame_SavedEsi, 4
|
||||
.equ .LSgemmKernelFrame_SavedEbx, 8
|
||||
.equ .LSgemmKernelFrame_SavedEbp, 12
|
||||
.equ .LSgemmKernelFrame_ReturnAddress, 16
|
||||
.equ .LSgemmKernelFrame_MatrixA, 20
|
||||
.equ .LSgemmKernelFrame_MatrixB, 24
|
||||
.equ .LSgemmKernelFrame_MatrixC, 28
|
||||
.equ .LSgemmKernelFrame_CountK, 32
|
||||
.equ .LSgemmKernelFrame_CountM, 36
|
||||
.equ .LSgemmKernelFrame_CountN, 40
|
||||
.equ .LSgemmKernelFrame_lda, 44
|
||||
.equ .LSgemmKernelFrame_ldc, 48
|
||||
.equ .LSgemmKernelFrame_alpha, 52
|
||||
.equ .LSgemmKernelFrame_ZeroMode, 56
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for a 16xN block of the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
ymm4-ymm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxBy16 RowCount, VectorOffset, BroadcastOffset
|
||||
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()]
|
||||
vmulps ymm1,ymm3,YMMWORD PTR [edx+\VectorOffset\()]
|
||||
vaddps ymm4,ymm1,ymm4
|
||||
vmulps ymm3,ymm3,YMMWORD PTR [edx+\VectorOffset\()+32]
|
||||
vaddps ymm5,ymm3,ymm5
|
||||
.else
|
||||
vmovaps ymm0,YMMWORD PTR [edx+\VectorOffset\()]
|
||||
vmovaps ymm1,YMMWORD PTR [edx+\VectorOffset\()+32]
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()]
|
||||
vmulps ymm2,ymm3,ymm0
|
||||
vaddps ymm4,ymm2,ymm4
|
||||
vmulps ymm2,ymm3,ymm1
|
||||
vaddps ymm5,ymm2,ymm5
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+ebx+\BroadcastOffset\()]
|
||||
vmulps ymm2,ymm3,ymm0
|
||||
vaddps ymm6,ymm2,ymm6
|
||||
vmulps ymm2,ymm3,ymm1
|
||||
vaddps ymm7,ymm2,ymm7
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for a 8xN block of the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
BroadcastOffset - Supplies the byte offset from matrix A to fetch elements.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
ymm4-ymm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxBy8 RowCount, VectorOffset, BroadcastOffset
|
||||
|
||||
.if \RowCount\() == 1
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()]
|
||||
vmulps ymm3,ymm3,YMMWORD PTR [edx+\VectorOffset\()]
|
||||
vaddps ymm5,ymm3,ymm5
|
||||
.else
|
||||
vmovaps ymm0,YMMWORD PTR [edx+\VectorOffset\()]
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()]
|
||||
vmulps ymm3,ymm3,ymm0
|
||||
vaddps ymm5,ymm3,ymm5
|
||||
vbroadcastss ymm3,DWORD PTR [ecx+ebx+\BroadcastOffset\()]
|
||||
vmulps ymm3,ymm3,ymm0
|
||||
vaddps ymm7,ymm3,ymm7
|
||||
.endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to execute the block compute macro multiple
|
||||
times and advancing the matrix A and matrix B data pointers.
|
||||
|
||||
Arguments:
|
||||
|
||||
ComputeBlock - Supplies the macro to compute a single block.
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the number of bytes to the next row of matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
edi - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
ymm4-ymm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockAvxLoop ComputeBlock, RowCount
|
||||
|
||||
sub edi,4
|
||||
jb .LProcessRemainingBlocks\@
|
||||
|
||||
.LComputeBlockBy4Loop\@:
|
||||
\ComputeBlock\() \RowCount\(), 0, 0
|
||||
\ComputeBlock\() \RowCount\(), 16*4, 4
|
||||
sub edx,-32*4 # advance matrix B by 32 columns
|
||||
\ComputeBlock\() \RowCount\(), 0, 8
|
||||
\ComputeBlock\() \RowCount\(), 16*4, 12
|
||||
sub edx,-32*4 # advance matrix B by 32 columns
|
||||
add ecx,4*4 # advance matrix A by 4 columns
|
||||
sub edi,4
|
||||
jae .LComputeBlockBy4Loop\@
|
||||
|
||||
.LProcessRemainingBlocks\@:
|
||||
add edi,4 # correct for over-subtract above
|
||||
jz .LOutputBlock\@
|
||||
|
||||
.LComputeBlockBy1Loop\@:
|
||||
\ComputeBlock\() \RowCount\(), 0, 0
|
||||
add edx,16*4 # advance matrix B by 16 columns
|
||||
add ecx,4 # advance matrix A by 1 column
|
||||
dec edi
|
||||
jne .LComputeBlockBy1Loop\@
|
||||
|
||||
.LOutputBlock\@:
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B. The matrix data has been packed using
|
||||
MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
CountM - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to iterate
|
||||
over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
Alpha - Supplies the scalar multiplier (see SGEMM definition).
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY MlasGemmFloatKernelAvx
|
||||
|
||||
push ebp
|
||||
push ebx
|
||||
push esi
|
||||
push edi
|
||||
mov edx,.LSgemmKernelFrame_MatrixB[esp]
|
||||
mov esi,.LSgemmKernelFrame_MatrixC[esp]
|
||||
mov ebp,.LSgemmKernelFrame_CountN[esp]
|
||||
|
||||
//
|
||||
// Process 2 rows of the matrices.
|
||||
//
|
||||
|
||||
cmp DWORD PTR .LSgemmKernelFrame_CountM[esp],2
|
||||
jb .LProcessCountMLessThan2
|
||||
mov BYTE PTR .LSgemmKernelFrame_CountM[esp],2
|
||||
mov eax,.LSgemmKernelFrame_ldc[esp]
|
||||
mov ebx,.LSgemmKernelFrame_lda[esp]
|
||||
shl eax,2 # convert ldc to bytes
|
||||
shl ebx,2 # convert lda to bytes
|
||||
cmp ebp,8
|
||||
jbe .LProcessRemainingCountN2
|
||||
|
||||
.LProcessNextColumnLoop16x2:
|
||||
mov edi,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ecx,.LSgemmKernelFrame_MatrixA[esp]
|
||||
vxorps xmm4,xmm4,xmm4 # clear block accumulators
|
||||
vxorps xmm5,xmm5,xmm5
|
||||
vxorps xmm6,xmm6,xmm6
|
||||
vxorps xmm7,xmm7,xmm7
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy16, 2
|
||||
vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
vmulps ymm4,ymm4,ymm2 # multiply by alpha
|
||||
vmulps ymm5,ymm5,ymm2
|
||||
vmulps ymm6,ymm6,ymm2
|
||||
vmulps ymm7,ymm7,ymm2
|
||||
sub ebp,16
|
||||
jb .LOutputMasked16x2Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateOutput16x2
|
||||
vaddps ymm4,ymm4,YMMWORD PTR [esi]
|
||||
vaddps ymm5,ymm5,YMMWORD PTR [esi+32]
|
||||
vaddps ymm6,ymm6,YMMWORD PTR [esi+eax]
|
||||
vaddps ymm7,ymm7,YMMWORD PTR [esi+eax+32]
|
||||
|
||||
.LSkipAccumulateOutput16x2:
|
||||
vmovups YMMWORD PTR [esi],ymm4
|
||||
vmovups YMMWORD PTR [esi+32],ymm5
|
||||
vmovups YMMWORD PTR [esi+eax],ymm6
|
||||
vmovups YMMWORD PTR [esi+eax+32],ymm7
|
||||
add esi,16*4 # advance matrix C by 16 columns
|
||||
cmp ebp,8
|
||||
ja .LProcessNextColumnLoop16x2
|
||||
test ebp,ebp
|
||||
jz .LExitKernel
|
||||
|
||||
.LProcessRemainingCountN2:
|
||||
mov edi,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ecx,.LSgemmKernelFrame_MatrixA[esp]
|
||||
vxorps xmm5,xmm5,xmm5 # clear block accumulators
|
||||
vxorps xmm7,xmm7,xmm7
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy8, 2
|
||||
vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
vmulps ymm5,ymm5,ymm2 # multiply by alpha
|
||||
vmulps ymm7,ymm7,ymm2
|
||||
cmp ebp,8
|
||||
jb .LOutputMasked8x2Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateOutput8x2
|
||||
vaddps ymm5,ymm5,YMMWORD PTR [esi]
|
||||
vaddps ymm7,ymm7,YMMWORD PTR [esi+eax]
|
||||
|
||||
.LSkipAccumulateOutput8x2:
|
||||
vmovups YMMWORD PTR [esi],ymm5
|
||||
vmovups YMMWORD PTR [esi+eax],ymm7
|
||||
|
||||
//
|
||||
// Restore non-volatile registers and return.
|
||||
//
|
||||
|
||||
.LExitKernel:
|
||||
movzx eax,BYTE PTR .LSgemmKernelFrame_CountM[esp]
|
||||
vzeroupper
|
||||
pop edi
|
||||
pop esi
|
||||
pop ebx
|
||||
pop ebp
|
||||
ret
|
||||
|
||||
.LOutputMasked16x2Block:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateMasked16x2Block
|
||||
vaddps ymm4,ymm4,YMMWORD PTR [esi]
|
||||
vaddps ymm6,ymm6,YMMWORD PTR [esi+eax]
|
||||
|
||||
.LSkipAccumulateMasked16x2Block:
|
||||
vmovups YMMWORD PTR [esi],ymm4
|
||||
vmovups YMMWORD PTR [esi+eax],ymm6
|
||||
add esi,8*4 # advance matrix C by 8 columns
|
||||
add ebp,8 # correct for over-subtract above
|
||||
|
||||
.LOutputMasked8x2Block:
|
||||
neg ebp
|
||||
LoadGlobalOffsetTable bx
|
||||
mov ebx,DWORD PTR C_UNDERSCORE(MlasMaskMoveTableAvx)@GOT[ebx]
|
||||
vmovdqu ymm0,YMMWORD PTR [ebx+ebp*4+8*4]
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateMasked8x2Block
|
||||
vmaskmovps ymm4,ymm0,YMMWORD PTR [esi]
|
||||
vmaskmovps ymm6,ymm0,YMMWORD PTR [esi+eax]
|
||||
vaddps ymm5,ymm5,ymm4
|
||||
vaddps ymm7,ymm7,ymm6
|
||||
|
||||
.LSkipAccumulateMasked8x2Block:
|
||||
vmaskmovps YMMWORD PTR [esi],ymm0,ymm5
|
||||
vmaskmovps YMMWORD PTR [esi+eax],ymm0,ymm7
|
||||
jmp .LExitKernel
|
||||
|
||||
//
|
||||
// Process 1 row of the matrices.
|
||||
//
|
||||
|
||||
.LProcessCountMLessThan2:
|
||||
mov BYTE PTR .LSgemmKernelFrame_CountM[esp],1
|
||||
mov ebx,.LSgemmKernelFrame_MatrixA[esp]
|
||||
vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
cmp ebp,8
|
||||
jbe .LProcessRemainingCountN1
|
||||
|
||||
.LProcessNextColumnLoop16x1:
|
||||
mov edi,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ecx,ebx # reload matrix A
|
||||
vxorps xmm4,xmm4,xmm4 # clear block accumulators
|
||||
vxorps xmm5,xmm5,xmm5
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy16, 1
|
||||
vmulps ymm4,ymm4,ymm2 # multiply by alpha
|
||||
vmulps ymm5,ymm5,ymm2
|
||||
sub ebp,16
|
||||
jb .LOutputMasked16x1Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulate16x1Block
|
||||
vaddps ymm4,ymm4,YMMWORD PTR [esi]
|
||||
vaddps ymm5,ymm5,YMMWORD PTR [esi+32]
|
||||
|
||||
.LSkipAccumulate16x1Block:
|
||||
vmovups YMMWORD PTR [esi],ymm4
|
||||
vmovups YMMWORD PTR [esi+32],ymm5
|
||||
add esi,16*4 # advance matrix C by 16 columns
|
||||
cmp ebp,8
|
||||
ja .LProcessNextColumnLoop16x1
|
||||
test ebp,ebp
|
||||
jz .LExitKernel
|
||||
|
||||
.LProcessRemainingCountN1:
|
||||
mov edi,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ecx,ebx # reload matrix A
|
||||
vxorps xmm5,xmm5,xmm5 # clear block accumulators
|
||||
ComputeBlockAvxLoop ComputeBlockAvxBy8, 1
|
||||
vmulps ymm5,ymm5,ymm2 # multiply by alpha
|
||||
cmp ebp,8
|
||||
jb .LOutputMasked8x1Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulate8x1Block
|
||||
vaddps ymm5,ymm5,YMMWORD PTR [esi]
|
||||
|
||||
.LSkipAccumulate8x1Block:
|
||||
vmovups YMMWORD PTR [esi],ymm5
|
||||
jmp .LExitKernel
|
||||
|
||||
.LOutputMasked16x1Block:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateMasked16x1Block
|
||||
vaddps ymm4,ymm4,YMMWORD PTR [esi]
|
||||
|
||||
.LSkipAccumulateMasked16x1Block:
|
||||
vmovups YMMWORD PTR [esi],ymm4
|
||||
add esi,8*4 # advance matrix C by 8 columns
|
||||
add ebp,8 # correct for over-subtract above
|
||||
|
||||
.LOutputMasked8x1Block:
|
||||
neg ebp
|
||||
LoadGlobalOffsetTable bx
|
||||
mov ebx,DWORD PTR C_UNDERSCORE(MlasMaskMoveTableAvx)@GOT[ebx]
|
||||
vmovdqu ymm0,YMMWORD PTR [ebx+ebp*4+8*4]
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateMasked8x1Block
|
||||
vmaskmovps ymm4,ymm0,YMMWORD PTR [esi]
|
||||
vaddps ymm5,ymm5,ymm4
|
||||
|
||||
.LSkipAccumulateMasked8x1Block:
|
||||
vmaskmovps YMMWORD PTR [esi],ymm0,ymm5
|
||||
jmp .LExitKernel
|
||||
|
||||
.end
|
||||
+406
@@ -0,0 +1,406 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
SgemmKernelSse2.s
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements the kernels for the single precision matrix/matrix
|
||||
multiply operation (SGEMM).
|
||||
|
||||
This implementation uses SSE2 instructions.
|
||||
|
||||
--*/
|
||||
|
||||
#include "asmmacro.h"
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
//
|
||||
// Stack frame layout for the SGEMM kernel.
|
||||
//
|
||||
|
||||
.equ .LSgemmKernelFrame_SavedEdi, 0
|
||||
.equ .LSgemmKernelFrame_SavedEsi, 4
|
||||
.equ .LSgemmKernelFrame_SavedEbx, 8
|
||||
.equ .LSgemmKernelFrame_SavedEbp, 12
|
||||
.equ .LSgemmKernelFrame_ReturnAddress, 16
|
||||
.equ .LSgemmKernelFrame_MatrixA, 20
|
||||
.equ .LSgemmKernelFrame_MatrixB, 24
|
||||
.equ .LSgemmKernelFrame_MatrixC, 28
|
||||
.equ .LSgemmKernelFrame_CountK, 32
|
||||
.equ .LSgemmKernelFrame_CountM, 36
|
||||
.equ .LSgemmKernelFrame_CountN, 40
|
||||
.equ .LSgemmKernelFrame_lda, 44
|
||||
.equ .LSgemmKernelFrame_ldc, 48
|
||||
.equ .LSgemmKernelFrame_alpha, 52
|
||||
.equ .LSgemmKernelFrame_ZeroMode, 56
|
||||
|
||||
.text
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro multiplies and accumulates for a Nx1 block of the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
VectorOffset - Supplies the byte offset from matrix B to fetch elements.
|
||||
|
||||
Shuffle - Supplies the shuffle mask to extract the element from matrix A.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the length in bytes of a row from matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
xmm2 - Supplies up to four elements loaded from matrix A.
|
||||
|
||||
xmm4-xmm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockSseBy4 VectorOffset, Shuffle
|
||||
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm4,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm5,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+32]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm6,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+48]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm7,xmm0
|
||||
|
||||
.endm
|
||||
|
||||
.macro ComputeBlockSseBy3 VectorOffset, Shuffle
|
||||
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm5,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm6,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+32]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm7,xmm0
|
||||
|
||||
.endm
|
||||
|
||||
.macro ComputeBlockSseBy2 VectorOffset, Shuffle
|
||||
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm6,xmm0
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm7,xmm0
|
||||
|
||||
.endm
|
||||
|
||||
.macro ComputeBlockSseBy1 VectorOffset, Shuffle
|
||||
|
||||
pshufd xmm3,xmm1,\Shuffle\()
|
||||
movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()]
|
||||
mulps xmm0,xmm3
|
||||
addps xmm7,xmm0
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro generates code to execute the block compute macro multiple
|
||||
times and advancing the matrix A and matrix B data pointers.
|
||||
|
||||
Arguments:
|
||||
|
||||
ComputeBlock - Supplies the macro to compute a single block.
|
||||
|
||||
RowCount - Supplies the number of rows to process.
|
||||
|
||||
Implicit Arguments:
|
||||
|
||||
ebx - Supplies the number of bytes to the next row of matrix A.
|
||||
|
||||
ecx - Supplies the address into the matrix A data.
|
||||
|
||||
edx - Supplies the address into the matrix B data.
|
||||
|
||||
edi - Supplies the number of columns from matrix A and the number of rows
|
||||
from matrix B to iterate over.
|
||||
|
||||
xmm4-xmm7 - Supplies the block accumulators.
|
||||
|
||||
--*/
|
||||
|
||||
.macro ComputeBlockSseLoop RowCount
|
||||
|
||||
sub edi,4
|
||||
jb .LProcessRemainingBlocks\@
|
||||
|
||||
.LComputeBlockBy4Loop\@:
|
||||
movups xmm1,XMMWORD PTR [ecx]
|
||||
ComputeBlockSseBy\RowCount\() 0, 0x00
|
||||
ComputeBlockSseBy\RowCount\() 16*4, 0x55
|
||||
sub edx,-32*4 # advance matrix B by 32 columns
|
||||
ComputeBlockSseBy\RowCount\() 0, 0xAA
|
||||
ComputeBlockSseBy\RowCount\() 16*4, 0xFF
|
||||
sub edx,-32*4 # advance matrix B by 32 columns
|
||||
add ecx,4*4 # advance matrix A by 4 columns
|
||||
sub edi,4
|
||||
jae .LComputeBlockBy4Loop\@
|
||||
|
||||
.LProcessRemainingBlocks\@:
|
||||
add edi,4 # correct for over-subtract above
|
||||
jz .LOutputBlock\@
|
||||
|
||||
.LComputeBlockBy1Loop\@:
|
||||
movss xmm1,DWORD PTR [ecx]
|
||||
ComputeBlockSseBy\RowCount\() 0, 0x00
|
||||
add edx,16*4 # advance matrix B by 16 columns
|
||||
add ecx,4 # advance matrix A by 1 column
|
||||
dec edi
|
||||
jne .LComputeBlockBy1Loop\@
|
||||
|
||||
.LOutputBlock\@:
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine is an inner kernel to compute matrix multiplication for a
|
||||
set of rows.
|
||||
|
||||
Arguments:
|
||||
|
||||
A - Supplies the address of matrix A.
|
||||
|
||||
B - Supplies the address of matrix B. The matrix data has been packed using
|
||||
MlasSgemmCopyPackB or MlasSgemmTransposePackB.
|
||||
|
||||
C - Supplies the address of matrix C.
|
||||
|
||||
CountK - Supplies the number of columns from matrix A and the number of
|
||||
rows from matrix B to iterate over.
|
||||
|
||||
CountM - Supplies the maximum number of rows that can be processed for
|
||||
matrix A and matrix C. The actual number of rows handled for this
|
||||
invocation depends on the kernel implementation.
|
||||
|
||||
CountN - Supplies the number of columns from matrix B and matrix C to
|
||||
iterate over.
|
||||
|
||||
lda - Supplies the first dimension of matrix A.
|
||||
|
||||
ldc - Supplies the first dimension of matrix C.
|
||||
|
||||
Alpha - Supplies the scalar multiplier (see SGEMM definition).
|
||||
|
||||
ZeroMode - Supplies true if the output matrix must be zero initialized,
|
||||
else false if the output matrix is accumulated into.
|
||||
|
||||
Return Value:
|
||||
|
||||
Returns the number of rows handled.
|
||||
|
||||
--*/
|
||||
|
||||
FUNCTION_ENTRY MlasGemmFloatKernelSse
|
||||
|
||||
push ebp
|
||||
push ebx
|
||||
push esi
|
||||
push edi
|
||||
mov edx,.LSgemmKernelFrame_MatrixB[esp]
|
||||
mov esi,.LSgemmKernelFrame_MatrixC[esp]
|
||||
mov ebp,.LSgemmKernelFrame_CountN[esp]
|
||||
|
||||
//
|
||||
// Process 1 row of the matrices.
|
||||
//
|
||||
|
||||
mov eax,.LSgemmKernelFrame_CountK[esp]
|
||||
mov ebx,.LSgemmKernelFrame_MatrixA[esp]
|
||||
cmp ebp,12
|
||||
jbe .LProcessRemainingCountN
|
||||
|
||||
.LProcessNextColumnLoop16x1:
|
||||
mov edi,eax # reload CountK
|
||||
mov ecx,ebx # reload matrix A
|
||||
xorps xmm4,xmm4 # clear block accumulators
|
||||
xorps xmm5,xmm5
|
||||
xorps xmm6,xmm6
|
||||
xorps xmm7,xmm7
|
||||
ComputeBlockSseLoop 4
|
||||
movss xmm2,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
shufps xmm2,xmm2,0
|
||||
mulps xmm4,xmm2 # multiply by alpha
|
||||
mulps xmm5,xmm2
|
||||
mulps xmm6,xmm2
|
||||
mulps xmm7,xmm2
|
||||
sub ebp,16
|
||||
jb .LOutputMasked16x1Block
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateOutput16x1
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
movups xmm1,XMMWORD PTR [esi+16]
|
||||
movups xmm2,XMMWORD PTR [esi+32]
|
||||
movups xmm3,XMMWORD PTR [esi+48]
|
||||
addps xmm4,xmm0
|
||||
addps xmm5,xmm1
|
||||
addps xmm6,xmm2
|
||||
addps xmm7,xmm3
|
||||
|
||||
.LSkipAccumulateOutput16x1:
|
||||
movups XMMWORD PTR [esi],xmm4
|
||||
movups XMMWORD PTR [esi+16],xmm5
|
||||
movups XMMWORD PTR [esi+32],xmm6
|
||||
movups XMMWORD PTR [esi+48],xmm7
|
||||
add esi,16*4 # advance matrix C by 16 columns
|
||||
cmp ebp,12
|
||||
ja .LProcessNextColumnLoop16x1
|
||||
test ebp,ebp
|
||||
jnz .LProcessRemainingCountN
|
||||
|
||||
//
|
||||
// Restore non-volatile registers and return.
|
||||
//
|
||||
|
||||
.LExitKernel:
|
||||
mov eax,1 # return 1 row handled
|
||||
pop edi
|
||||
pop esi
|
||||
pop ebx
|
||||
pop ebp
|
||||
ret
|
||||
|
||||
//
|
||||
// Process the remaining 1 to 12 columns of the matrices.
|
||||
//
|
||||
|
||||
.LProcessRemainingCountN:
|
||||
mov edi,eax # reload CountK
|
||||
mov ecx,ebx # reload matrix A
|
||||
movss xmm4,DWORD PTR .LSgemmKernelFrame_alpha[esp]
|
||||
shufps xmm4,xmm4,0
|
||||
xorps xmm5,xmm5 # clear block accumulators
|
||||
xorps xmm6,xmm6
|
||||
xorps xmm7,xmm7
|
||||
cmp ebp,4
|
||||
jbe .LProcessRemainingCountN4OrLess
|
||||
cmp ebp,8
|
||||
jbe .LProcessRemainingCountN8OrLess
|
||||
|
||||
.LProcessRemainingCountN12OrLess:
|
||||
ComputeBlockSseLoop 3
|
||||
mulps xmm5,xmm4 # multiply by alpha
|
||||
mulps xmm6,xmm4
|
||||
mulps xmm7,xmm4
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateLeadingN12OrLess
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
movups xmm1,XMMWORD PTR [esi+16]
|
||||
addps xmm5,xmm0
|
||||
addps xmm6,xmm1
|
||||
|
||||
.LSkipAccumulateLeadingN12OrLess:
|
||||
movups XMMWORD PTR [esi],xmm5
|
||||
movups XMMWORD PTR [esi+16],xmm6
|
||||
add esi,8*4 # advance matrix C by 8 columns
|
||||
jmp .LOutputTrailingBlock
|
||||
|
||||
.LProcessRemainingCountN8OrLess:
|
||||
ComputeBlockSseLoop 2
|
||||
mulps xmm6,xmm4 # multiply by alpha
|
||||
mulps xmm7,xmm4
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateLeadingN8OrLess
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
addps xmm6,xmm0
|
||||
|
||||
.LSkipAccumulateLeadingN8OrLess:
|
||||
movups XMMWORD PTR [esi],xmm6
|
||||
add esi,4*4 # advance matrix C by 4 columns
|
||||
jmp .LOutputTrailingBlock
|
||||
|
||||
.LProcessRemainingCountN4OrLess:
|
||||
ComputeBlockSseLoop 1
|
||||
mulps xmm7,xmm4 # multiply by alpha
|
||||
jmp .LOutputTrailingBlock
|
||||
|
||||
.LOutputMasked16x1Block:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateLeading16x1Block
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
movups xmm1,XMMWORD PTR [esi+16]
|
||||
movups xmm2,XMMWORD PTR [esi+32]
|
||||
addps xmm4,xmm0
|
||||
addps xmm5,xmm1
|
||||
addps xmm6,xmm2
|
||||
|
||||
.LSkipAccumulateLeading16x1Block:
|
||||
movups XMMWORD PTR [esi],xmm4
|
||||
movups XMMWORD PTR [esi+16],xmm5
|
||||
movups XMMWORD PTR [esi+32],xmm6
|
||||
add esi,12*4 # advance matrix C by 12 columns
|
||||
|
||||
.LOutputTrailingBlock:
|
||||
test ebp,3
|
||||
jz .LOutputTrailingBlock4Elements
|
||||
test ebp,2
|
||||
jz .LOutputTrailingBlock1Element
|
||||
|
||||
.LOutputTrailingBlock2Elements:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateTrailingBlock2Elements
|
||||
movsd xmm0,QWORD PTR [esi]
|
||||
addps xmm7,xmm0
|
||||
|
||||
.LSkipAccumulateTrailingBlock2Elements:
|
||||
movsd QWORD PTR [esi],xmm7
|
||||
test ebp,1
|
||||
jz .LExitKernel
|
||||
shufps xmm7,xmm7,0xAA # shuffle third float down
|
||||
add esi,2*4 # advance matrix C by 2 columns
|
||||
|
||||
.LOutputTrailingBlock1Element:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateTrailingBlock1Element
|
||||
movss xmm0,DWORD PTR [esi]
|
||||
addss xmm7,xmm0
|
||||
|
||||
.LSkipAccumulateTrailingBlock1Element:
|
||||
movss DWORD PTR [esi],xmm7
|
||||
jmp .LExitKernel
|
||||
|
||||
.LOutputTrailingBlock4Elements:
|
||||
cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0
|
||||
jnz .LSkipAccumulateTrailingBlock4Elements
|
||||
movups xmm0,XMMWORD PTR [esi]
|
||||
addps xmm7,xmm0
|
||||
|
||||
.LSkipAccumulateTrailingBlock4Elements:
|
||||
movups XMMWORD PTR [esi],xmm7
|
||||
jmp .LExitKernel
|
||||
|
||||
.end
|
||||
Vendored
+79
@@ -0,0 +1,79 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
asmmacro.h
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements common macros for the assembly modules.
|
||||
|
||||
--*/
|
||||
|
||||
#if defined(__APPLE__)
|
||||
#define C_UNDERSCORE(symbol) _##symbol
|
||||
#else
|
||||
#define C_UNDERSCORE(symbol) symbol
|
||||
#endif
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro emits the assembler directives to annotate a new function.
|
||||
|
||||
Arguments:
|
||||
|
||||
FunctionName - Supplies the name of the function.
|
||||
|
||||
--*/
|
||||
|
||||
.macro FUNCTION_ENTRY FunctionName
|
||||
|
||||
.p2align 4
|
||||
#if defined(__APPLE__)
|
||||
.globl _\FunctionName\()
|
||||
_\FunctionName\():
|
||||
#else
|
||||
.globl \FunctionName\()
|
||||
.type \FunctionName\(),@function
|
||||
\FunctionName\():
|
||||
#endif
|
||||
|
||||
.endm
|
||||
|
||||
/*++
|
||||
|
||||
Macro Description:
|
||||
|
||||
This macro emits the code to load the global offset table address into the
|
||||
supplied register.
|
||||
|
||||
Arguments:
|
||||
|
||||
TargetReg - Specifies the target register.
|
||||
|
||||
--*/
|
||||
|
||||
.macro LoadGlobalOffsetTable, TargetReg
|
||||
|
||||
//
|
||||
// The LLVM integrated assembler doesn't support the Intel syntax for OFFSET:
|
||||
//
|
||||
// add ebx,OFFSET _GLOBAL_OFFSET_TABLE_
|
||||
//
|
||||
// Workaround this by temporarily switching to AT&T syntax.
|
||||
//
|
||||
|
||||
.att_syntax
|
||||
|
||||
calll __x86.get_pc_thunk.\TargetReg\()
|
||||
addl $_GLOBAL_OFFSET_TABLE_,%e\TargetReg\()
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
.endm
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
/*++
|
||||
|
||||
Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
|
||||
Licensed under the MIT License.
|
||||
|
||||
Module Name:
|
||||
|
||||
x86.get_pc_thunk.S
|
||||
|
||||
Abstract:
|
||||
|
||||
This module implements __x86.get_pc_thunk.* to avoid external dependency.
|
||||
|
||||
--*/
|
||||
|
||||
.intel_syntax noprefix
|
||||
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
The routine loads its return address -- which is the address of the
|
||||
instruction that immediately follows -- into the ebx register.
|
||||
|
||||
--*/
|
||||
|
||||
.p2align 4
|
||||
.weak __x86.get_pc_thunk.bx
|
||||
// Hidden = non-preemptible, so the PC-relative call in the SGEMM
|
||||
// kernels binds locally in the .so (no R_386_PC32 link error).
|
||||
.hidden __x86.get_pc_thunk.bx
|
||||
.type __x86.get_pc_thunk.bx,@function
|
||||
__x86.get_pc_thunk.bx:
|
||||
mov ebx, [esp]
|
||||
ret
|
||||
+4
-8
@@ -77,14 +77,10 @@ if(OPENCV_WORKAROUND_CMAKE_20989)
|
||||
set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR_BACKUP})
|
||||
endif()
|
||||
|
||||
# Enable generic ASM at top-level scope: enable_language(ASM) from a subdir fails
|
||||
# (CMAKE_ASM_COMPILE_OBJECT unset) once ASM_NASM is enabled. Skip on Android (NDK).
|
||||
if(NOT ANDROID AND NOT CMAKE_SYSTEM_NAME STREQUAL "Android")
|
||||
include(CheckLanguage)
|
||||
check_language(ASM)
|
||||
if(CMAKE_ASM_COMPILER)
|
||||
enable_language(ASM)
|
||||
endif()
|
||||
include(CheckLanguage)
|
||||
check_language(ASM)
|
||||
if(CMAKE_ASM_COMPILER)
|
||||
enable_language(ASM)
|
||||
endif()
|
||||
|
||||
enable_testing()
|
||||
|
||||
Reference in New Issue
Block a user