diff --git a/3rdparty/mlas/CMakeLists.txt b/3rdparty/mlas/CMakeLists.txt index 381a5a162f..c773ec0004 100644 --- a/3rdparty/mlas/CMakeLists.txt +++ b/3rdparty/mlas/CMakeLists.txt @@ -83,15 +83,7 @@ set(_MLAS_REQUIRES_ASM FALSE) if(MLAS_X86_64 OR MLAS_X86 OR MLAS_ARM64 OR MLAS_LOONGARCH64) set(_MLAS_REQUIRES_ASM TRUE) endif() -if(ANDROID AND _MLAS_REQUIRES_ASM) - set(OPENCV_DNN_MLAS_SKIP_REASON - "skipped on Android ${CMAKE_ANDROID_ARCH_ABI}; enable_language(ASM) from subdir is unreliable" - CACHE INTERNAL "" FORCE) - message(STATUS "MLAS: skipping on Android ${CMAKE_ANDROID_ARCH_ABI} " - "(enable_language(ASM) from subdir is unreliable here); " - "DNN will use its built-in SGEMM") - return() -endif() + include(CheckLanguage) set(MLAS_HAS_ASM FALSE CACHE INTERNAL "" FORCE) check_language(ASM) @@ -261,12 +253,11 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang" "$<$:-Wa,--noexecstack>" ) endif() -# platform.cpp's __linux__ branch calls syscall() but upstream only includes -# , not (where syscall() is declared on glibc). -# Force-include unistd.h on non-Windows x86_64 builds. MSVC doesn't ship -# unistd.h and doesn't recognize the -include flag; its _WIN32 branch in -# platform.cpp doesn't need syscall() anyway. -if(MLAS_X86_64 AND NOT WIN32) +# platform.cpp's MlasInitAMX() calls syscall() but only includes , +# not where glibc declares it. It's compiled on both x86 and x86_64 +# (shared MLAS_TARGET_AMD64_IX86 block), so force-include unistd.h on both. +# MSVC has no unistd.h and its _WIN32 path doesn't use syscall() anyway. +if((MLAS_X86_64 OR MLAS_X86) AND NOT WIN32) set_source_files_properties(${MLAS_SRC_DIR}/platform.cpp PROPERTIES COMPILE_FLAGS "-include unistd.h") endif() diff --git a/3rdparty/mlas/lib/platform.cpp b/3rdparty/mlas/lib/platform.cpp index a515413331..52b5ac234c 100644 --- a/3rdparty/mlas/lib/platform.cpp +++ b/3rdparty/mlas/lib/platform.cpp @@ -363,7 +363,7 @@ MLAS_PLATFORM::MLAS_PLATFORM(void) this->KernelM1Routine = MlasSgemmKernelM1Avx; this->KernelM1TransposeBRoutine = MlasSgemmKernelM1TransposeBAvx; this->TransposePackB16x4Routine = MlasSgemmTransposePackB16x4Avx; -#endif + // FMA3/AVX512F kernels exist for x86_64 only; 32-bit x86 stops at AVX. unsigned Cpuid7[4]; #if defined(_WIN32) __cpuidex((int*)Cpuid7, 7, 0); @@ -378,6 +378,7 @@ MLAS_PLATFORM::MLAS_PLATFORM(void) this->GemmFloatKernel = MlasGemmFloatKernelAvx512F; } } +#endif // MLAS_TARGET_AMD64 } } #endif // MLAS_TARGET_AMD64_IX86 diff --git a/3rdparty/mlas/lib/x86/SgemmKernelAvx.S b/3rdparty/mlas/lib/x86/SgemmKernelAvx.S new file mode 100644 index 0000000000..7af2a9e118 --- /dev/null +++ b/3rdparty/mlas/lib/x86/SgemmKernelAvx.S @@ -0,0 +1,435 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelAvx.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). + + This implementation uses AVX instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + +// +// Stack frame layout for the SGEMM kernel. +// + + .equ .LSgemmKernelFrame_SavedEdi, 0 + .equ .LSgemmKernelFrame_SavedEsi, 4 + .equ .LSgemmKernelFrame_SavedEbx, 8 + .equ .LSgemmKernelFrame_SavedEbp, 12 + .equ .LSgemmKernelFrame_ReturnAddress, 16 + .equ .LSgemmKernelFrame_MatrixA, 20 + .equ .LSgemmKernelFrame_MatrixB, 24 + .equ .LSgemmKernelFrame_MatrixC, 28 + .equ .LSgemmKernelFrame_CountK, 32 + .equ .LSgemmKernelFrame_CountM, 36 + .equ .LSgemmKernelFrame_CountN, 40 + .equ .LSgemmKernelFrame_lda, 44 + .equ .LSgemmKernelFrame_ldc, 48 + .equ .LSgemmKernelFrame_alpha, 52 + .equ .LSgemmKernelFrame_ZeroMode, 56 + + .text + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 16xN block of the output matrix. + +Arguments: + + RowCount - Supplies the number of rows to process. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + ebx - Supplies the length in bytes of a row from matrix A. + + ecx - Supplies the address into the matrix A data. + + edx - Supplies the address into the matrix B data. + + ymm4-ymm7 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvxBy16 RowCount, VectorOffset, BroadcastOffset + +.if \RowCount\() == 1 + vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()] + vmulps ymm1,ymm3,YMMWORD PTR [edx+\VectorOffset\()] + vaddps ymm4,ymm1,ymm4 + vmulps ymm3,ymm3,YMMWORD PTR [edx+\VectorOffset\()+32] + vaddps ymm5,ymm3,ymm5 +.else + vmovaps ymm0,YMMWORD PTR [edx+\VectorOffset\()] + vmovaps ymm1,YMMWORD PTR [edx+\VectorOffset\()+32] + vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()] + vmulps ymm2,ymm3,ymm0 + vaddps ymm4,ymm2,ymm4 + vmulps ymm2,ymm3,ymm1 + vaddps ymm5,ymm2,ymm5 + vbroadcastss ymm3,DWORD PTR [ecx+ebx+\BroadcastOffset\()] + vmulps ymm2,ymm3,ymm0 + vaddps ymm6,ymm2,ymm6 + vmulps ymm2,ymm3,ymm1 + vaddps ymm7,ymm2,ymm7 +.endif + + .endm + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a 8xN block of the output matrix. + +Arguments: + + RowCount - Supplies the number of rows to process. + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + BroadcastOffset - Supplies the byte offset from matrix A to fetch elements. + +Implicit Arguments: + + ebx - Supplies the length in bytes of a row from matrix A. + + ecx - Supplies the address into the matrix A data. + + edx - Supplies the address into the matrix B data. + + ymm4-ymm7 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvxBy8 RowCount, VectorOffset, BroadcastOffset + +.if \RowCount\() == 1 + vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()] + vmulps ymm3,ymm3,YMMWORD PTR [edx+\VectorOffset\()] + vaddps ymm5,ymm3,ymm5 +.else + vmovaps ymm0,YMMWORD PTR [edx+\VectorOffset\()] + vbroadcastss ymm3,DWORD PTR [ecx+\BroadcastOffset\()] + vmulps ymm3,ymm3,ymm0 + vaddps ymm5,ymm3,ymm5 + vbroadcastss ymm3,DWORD PTR [ecx+ebx+\BroadcastOffset\()] + vmulps ymm3,ymm3,ymm0 + vaddps ymm7,ymm3,ymm7 +.endif + + .endm + +/*++ + +Macro Description: + + This macro generates code to execute the block compute macro multiple + times and advancing the matrix A and matrix B data pointers. + +Arguments: + + ComputeBlock - Supplies the macro to compute a single block. + + RowCount - Supplies the number of rows to process. + +Implicit Arguments: + + ebx - Supplies the number of bytes to the next row of matrix A. + + ecx - Supplies the address into the matrix A data. + + edx - Supplies the address into the matrix B data. + + edi - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + ymm4-ymm7 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockAvxLoop ComputeBlock, RowCount + + sub edi,4 + jb .LProcessRemainingBlocks\@ + +.LComputeBlockBy4Loop\@: + \ComputeBlock\() \RowCount\(), 0, 0 + \ComputeBlock\() \RowCount\(), 16*4, 4 + sub edx,-32*4 # advance matrix B by 32 columns + \ComputeBlock\() \RowCount\(), 0, 8 + \ComputeBlock\() \RowCount\(), 16*4, 12 + sub edx,-32*4 # advance matrix B by 32 columns + add ecx,4*4 # advance matrix A by 4 columns + sub edi,4 + jae .LComputeBlockBy4Loop\@ + +.LProcessRemainingBlocks\@: + add edi,4 # correct for over-subtract above + jz .LOutputBlock\@ + +.LComputeBlockBy1Loop\@: + \ComputeBlock\() \RowCount\(), 0, 0 + add edx,16*4 # advance matrix B by 16 columns + add ecx,4 # advance matrix A by 1 column + dec edi + jne .LComputeBlockBy1Loop\@ + +.LOutputBlock\@: + + .endm + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A - Supplies the address of matrix A. + + B - Supplies the address of matrix B. The matrix data has been packed using + MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C - Supplies the address of matrix C. + + CountK - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + CountM - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN - Supplies the number of columns from matrix B and matrix C to iterate + over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + Alpha - Supplies the scalar multiplier (see SGEMM definition). + + ZeroMode - Supplies true if the output matrix must be zero initialized, + else false if the output matrix is accumulated into. + +Return Value: + + Returns the number of rows handled. + +--*/ + + FUNCTION_ENTRY MlasGemmFloatKernelAvx + + push ebp + push ebx + push esi + push edi + mov edx,.LSgemmKernelFrame_MatrixB[esp] + mov esi,.LSgemmKernelFrame_MatrixC[esp] + mov ebp,.LSgemmKernelFrame_CountN[esp] + +// +// Process 2 rows of the matrices. +// + + cmp DWORD PTR .LSgemmKernelFrame_CountM[esp],2 + jb .LProcessCountMLessThan2 + mov BYTE PTR .LSgemmKernelFrame_CountM[esp],2 + mov eax,.LSgemmKernelFrame_ldc[esp] + mov ebx,.LSgemmKernelFrame_lda[esp] + shl eax,2 # convert ldc to bytes + shl ebx,2 # convert lda to bytes + cmp ebp,8 + jbe .LProcessRemainingCountN2 + +.LProcessNextColumnLoop16x2: + mov edi,.LSgemmKernelFrame_CountK[esp] + mov ecx,.LSgemmKernelFrame_MatrixA[esp] + vxorps xmm4,xmm4,xmm4 # clear block accumulators + vxorps xmm5,xmm5,xmm5 + vxorps xmm6,xmm6,xmm6 + vxorps xmm7,xmm7,xmm7 + ComputeBlockAvxLoop ComputeBlockAvxBy16, 2 + vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp] + vmulps ymm4,ymm4,ymm2 # multiply by alpha + vmulps ymm5,ymm5,ymm2 + vmulps ymm6,ymm6,ymm2 + vmulps ymm7,ymm7,ymm2 + sub ebp,16 + jb .LOutputMasked16x2Block + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateOutput16x2 + vaddps ymm4,ymm4,YMMWORD PTR [esi] + vaddps ymm5,ymm5,YMMWORD PTR [esi+32] + vaddps ymm6,ymm6,YMMWORD PTR [esi+eax] + vaddps ymm7,ymm7,YMMWORD PTR [esi+eax+32] + +.LSkipAccumulateOutput16x2: + vmovups YMMWORD PTR [esi],ymm4 + vmovups YMMWORD PTR [esi+32],ymm5 + vmovups YMMWORD PTR [esi+eax],ymm6 + vmovups YMMWORD PTR [esi+eax+32],ymm7 + add esi,16*4 # advance matrix C by 16 columns + cmp ebp,8 + ja .LProcessNextColumnLoop16x2 + test ebp,ebp + jz .LExitKernel + +.LProcessRemainingCountN2: + mov edi,.LSgemmKernelFrame_CountK[esp] + mov ecx,.LSgemmKernelFrame_MatrixA[esp] + vxorps xmm5,xmm5,xmm5 # clear block accumulators + vxorps xmm7,xmm7,xmm7 + ComputeBlockAvxLoop ComputeBlockAvxBy8, 2 + vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp] + vmulps ymm5,ymm5,ymm2 # multiply by alpha + vmulps ymm7,ymm7,ymm2 + cmp ebp,8 + jb .LOutputMasked8x2Block + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateOutput8x2 + vaddps ymm5,ymm5,YMMWORD PTR [esi] + vaddps ymm7,ymm7,YMMWORD PTR [esi+eax] + +.LSkipAccumulateOutput8x2: + vmovups YMMWORD PTR [esi],ymm5 + vmovups YMMWORD PTR [esi+eax],ymm7 + +// +// Restore non-volatile registers and return. +// + +.LExitKernel: + movzx eax,BYTE PTR .LSgemmKernelFrame_CountM[esp] + vzeroupper + pop edi + pop esi + pop ebx + pop ebp + ret + +.LOutputMasked16x2Block: + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateMasked16x2Block + vaddps ymm4,ymm4,YMMWORD PTR [esi] + vaddps ymm6,ymm6,YMMWORD PTR [esi+eax] + +.LSkipAccumulateMasked16x2Block: + vmovups YMMWORD PTR [esi],ymm4 + vmovups YMMWORD PTR [esi+eax],ymm6 + add esi,8*4 # advance matrix C by 8 columns + add ebp,8 # correct for over-subtract above + +.LOutputMasked8x2Block: + neg ebp + LoadGlobalOffsetTable bx + mov ebx,DWORD PTR C_UNDERSCORE(MlasMaskMoveTableAvx)@GOT[ebx] + vmovdqu ymm0,YMMWORD PTR [ebx+ebp*4+8*4] + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateMasked8x2Block + vmaskmovps ymm4,ymm0,YMMWORD PTR [esi] + vmaskmovps ymm6,ymm0,YMMWORD PTR [esi+eax] + vaddps ymm5,ymm5,ymm4 + vaddps ymm7,ymm7,ymm6 + +.LSkipAccumulateMasked8x2Block: + vmaskmovps YMMWORD PTR [esi],ymm0,ymm5 + vmaskmovps YMMWORD PTR [esi+eax],ymm0,ymm7 + jmp .LExitKernel + +// +// Process 1 row of the matrices. +// + +.LProcessCountMLessThan2: + mov BYTE PTR .LSgemmKernelFrame_CountM[esp],1 + mov ebx,.LSgemmKernelFrame_MatrixA[esp] + vbroadcastss ymm2,DWORD PTR .LSgemmKernelFrame_alpha[esp] + cmp ebp,8 + jbe .LProcessRemainingCountN1 + +.LProcessNextColumnLoop16x1: + mov edi,.LSgemmKernelFrame_CountK[esp] + mov ecx,ebx # reload matrix A + vxorps xmm4,xmm4,xmm4 # clear block accumulators + vxorps xmm5,xmm5,xmm5 + ComputeBlockAvxLoop ComputeBlockAvxBy16, 1 + vmulps ymm4,ymm4,ymm2 # multiply by alpha + vmulps ymm5,ymm5,ymm2 + sub ebp,16 + jb .LOutputMasked16x1Block + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulate16x1Block + vaddps ymm4,ymm4,YMMWORD PTR [esi] + vaddps ymm5,ymm5,YMMWORD PTR [esi+32] + +.LSkipAccumulate16x1Block: + vmovups YMMWORD PTR [esi],ymm4 + vmovups YMMWORD PTR [esi+32],ymm5 + add esi,16*4 # advance matrix C by 16 columns + cmp ebp,8 + ja .LProcessNextColumnLoop16x1 + test ebp,ebp + jz .LExitKernel + +.LProcessRemainingCountN1: + mov edi,.LSgemmKernelFrame_CountK[esp] + mov ecx,ebx # reload matrix A + vxorps xmm5,xmm5,xmm5 # clear block accumulators + ComputeBlockAvxLoop ComputeBlockAvxBy8, 1 + vmulps ymm5,ymm5,ymm2 # multiply by alpha + cmp ebp,8 + jb .LOutputMasked8x1Block + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulate8x1Block + vaddps ymm5,ymm5,YMMWORD PTR [esi] + +.LSkipAccumulate8x1Block: + vmovups YMMWORD PTR [esi],ymm5 + jmp .LExitKernel + +.LOutputMasked16x1Block: + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateMasked16x1Block + vaddps ymm4,ymm4,YMMWORD PTR [esi] + +.LSkipAccumulateMasked16x1Block: + vmovups YMMWORD PTR [esi],ymm4 + add esi,8*4 # advance matrix C by 8 columns + add ebp,8 # correct for over-subtract above + +.LOutputMasked8x1Block: + neg ebp + LoadGlobalOffsetTable bx + mov ebx,DWORD PTR C_UNDERSCORE(MlasMaskMoveTableAvx)@GOT[ebx] + vmovdqu ymm0,YMMWORD PTR [ebx+ebp*4+8*4] + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateMasked8x1Block + vmaskmovps ymm4,ymm0,YMMWORD PTR [esi] + vaddps ymm5,ymm5,ymm4 + +.LSkipAccumulateMasked8x1Block: + vmaskmovps YMMWORD PTR [esi],ymm0,ymm5 + jmp .LExitKernel + + .end diff --git a/3rdparty/mlas/lib/x86/SgemmKernelSse2.S b/3rdparty/mlas/lib/x86/SgemmKernelSse2.S new file mode 100644 index 0000000000..f42175ec6c --- /dev/null +++ b/3rdparty/mlas/lib/x86/SgemmKernelSse2.S @@ -0,0 +1,406 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + SgemmKernelSse2.s + +Abstract: + + This module implements the kernels for the single precision matrix/matrix + multiply operation (SGEMM). + + This implementation uses SSE2 instructions. + +--*/ + +#include "asmmacro.h" + + .intel_syntax noprefix + +// +// Stack frame layout for the SGEMM kernel. +// + + .equ .LSgemmKernelFrame_SavedEdi, 0 + .equ .LSgemmKernelFrame_SavedEsi, 4 + .equ .LSgemmKernelFrame_SavedEbx, 8 + .equ .LSgemmKernelFrame_SavedEbp, 12 + .equ .LSgemmKernelFrame_ReturnAddress, 16 + .equ .LSgemmKernelFrame_MatrixA, 20 + .equ .LSgemmKernelFrame_MatrixB, 24 + .equ .LSgemmKernelFrame_MatrixC, 28 + .equ .LSgemmKernelFrame_CountK, 32 + .equ .LSgemmKernelFrame_CountM, 36 + .equ .LSgemmKernelFrame_CountN, 40 + .equ .LSgemmKernelFrame_lda, 44 + .equ .LSgemmKernelFrame_ldc, 48 + .equ .LSgemmKernelFrame_alpha, 52 + .equ .LSgemmKernelFrame_ZeroMode, 56 + + .text + +/*++ + +Macro Description: + + This macro multiplies and accumulates for a Nx1 block of the output matrix. + +Arguments: + + VectorOffset - Supplies the byte offset from matrix B to fetch elements. + + Shuffle - Supplies the shuffle mask to extract the element from matrix A. + +Implicit Arguments: + + ebx - Supplies the length in bytes of a row from matrix A. + + ecx - Supplies the address into the matrix A data. + + edx - Supplies the address into the matrix B data. + + xmm2 - Supplies up to four elements loaded from matrix A. + + xmm4-xmm7 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockSseBy4 VectorOffset, Shuffle + + pshufd xmm3,xmm1,\Shuffle\() + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()] + mulps xmm0,xmm3 + addps xmm4,xmm0 + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16] + mulps xmm0,xmm3 + addps xmm5,xmm0 + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+32] + mulps xmm0,xmm3 + addps xmm6,xmm0 + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+48] + mulps xmm0,xmm3 + addps xmm7,xmm0 + + .endm + + .macro ComputeBlockSseBy3 VectorOffset, Shuffle + + pshufd xmm3,xmm1,\Shuffle\() + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()] + mulps xmm0,xmm3 + addps xmm5,xmm0 + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16] + mulps xmm0,xmm3 + addps xmm6,xmm0 + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+32] + mulps xmm0,xmm3 + addps xmm7,xmm0 + + .endm + + .macro ComputeBlockSseBy2 VectorOffset, Shuffle + + pshufd xmm3,xmm1,\Shuffle\() + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()] + mulps xmm0,xmm3 + addps xmm6,xmm0 + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()+16] + mulps xmm0,xmm3 + addps xmm7,xmm0 + + .endm + + .macro ComputeBlockSseBy1 VectorOffset, Shuffle + + pshufd xmm3,xmm1,\Shuffle\() + movaps xmm0,XMMWORD PTR [edx+\VectorOffset\()] + mulps xmm0,xmm3 + addps xmm7,xmm0 + + .endm + +/*++ + +Macro Description: + + This macro generates code to execute the block compute macro multiple + times and advancing the matrix A and matrix B data pointers. + +Arguments: + + ComputeBlock - Supplies the macro to compute a single block. + + RowCount - Supplies the number of rows to process. + +Implicit Arguments: + + ebx - Supplies the number of bytes to the next row of matrix A. + + ecx - Supplies the address into the matrix A data. + + edx - Supplies the address into the matrix B data. + + edi - Supplies the number of columns from matrix A and the number of rows + from matrix B to iterate over. + + xmm4-xmm7 - Supplies the block accumulators. + +--*/ + + .macro ComputeBlockSseLoop RowCount + + sub edi,4 + jb .LProcessRemainingBlocks\@ + +.LComputeBlockBy4Loop\@: + movups xmm1,XMMWORD PTR [ecx] + ComputeBlockSseBy\RowCount\() 0, 0x00 + ComputeBlockSseBy\RowCount\() 16*4, 0x55 + sub edx,-32*4 # advance matrix B by 32 columns + ComputeBlockSseBy\RowCount\() 0, 0xAA + ComputeBlockSseBy\RowCount\() 16*4, 0xFF + sub edx,-32*4 # advance matrix B by 32 columns + add ecx,4*4 # advance matrix A by 4 columns + sub edi,4 + jae .LComputeBlockBy4Loop\@ + +.LProcessRemainingBlocks\@: + add edi,4 # correct for over-subtract above + jz .LOutputBlock\@ + +.LComputeBlockBy1Loop\@: + movss xmm1,DWORD PTR [ecx] + ComputeBlockSseBy\RowCount\() 0, 0x00 + add edx,16*4 # advance matrix B by 16 columns + add ecx,4 # advance matrix A by 1 column + dec edi + jne .LComputeBlockBy1Loop\@ + +.LOutputBlock\@: + + .endm + +/*++ + +Routine Description: + + This routine is an inner kernel to compute matrix multiplication for a + set of rows. + +Arguments: + + A - Supplies the address of matrix A. + + B - Supplies the address of matrix B. The matrix data has been packed using + MlasSgemmCopyPackB or MlasSgemmTransposePackB. + + C - Supplies the address of matrix C. + + CountK - Supplies the number of columns from matrix A and the number of + rows from matrix B to iterate over. + + CountM - Supplies the maximum number of rows that can be processed for + matrix A and matrix C. The actual number of rows handled for this + invocation depends on the kernel implementation. + + CountN - Supplies the number of columns from matrix B and matrix C to + iterate over. + + lda - Supplies the first dimension of matrix A. + + ldc - Supplies the first dimension of matrix C. + + Alpha - Supplies the scalar multiplier (see SGEMM definition). + + ZeroMode - Supplies true if the output matrix must be zero initialized, + else false if the output matrix is accumulated into. + +Return Value: + + Returns the number of rows handled. + +--*/ + + FUNCTION_ENTRY MlasGemmFloatKernelSse + + push ebp + push ebx + push esi + push edi + mov edx,.LSgemmKernelFrame_MatrixB[esp] + mov esi,.LSgemmKernelFrame_MatrixC[esp] + mov ebp,.LSgemmKernelFrame_CountN[esp] + +// +// Process 1 row of the matrices. +// + + mov eax,.LSgemmKernelFrame_CountK[esp] + mov ebx,.LSgemmKernelFrame_MatrixA[esp] + cmp ebp,12 + jbe .LProcessRemainingCountN + +.LProcessNextColumnLoop16x1: + mov edi,eax # reload CountK + mov ecx,ebx # reload matrix A + xorps xmm4,xmm4 # clear block accumulators + xorps xmm5,xmm5 + xorps xmm6,xmm6 + xorps xmm7,xmm7 + ComputeBlockSseLoop 4 + movss xmm2,DWORD PTR .LSgemmKernelFrame_alpha[esp] + shufps xmm2,xmm2,0 + mulps xmm4,xmm2 # multiply by alpha + mulps xmm5,xmm2 + mulps xmm6,xmm2 + mulps xmm7,xmm2 + sub ebp,16 + jb .LOutputMasked16x1Block + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateOutput16x1 + movups xmm0,XMMWORD PTR [esi] + movups xmm1,XMMWORD PTR [esi+16] + movups xmm2,XMMWORD PTR [esi+32] + movups xmm3,XMMWORD PTR [esi+48] + addps xmm4,xmm0 + addps xmm5,xmm1 + addps xmm6,xmm2 + addps xmm7,xmm3 + +.LSkipAccumulateOutput16x1: + movups XMMWORD PTR [esi],xmm4 + movups XMMWORD PTR [esi+16],xmm5 + movups XMMWORD PTR [esi+32],xmm6 + movups XMMWORD PTR [esi+48],xmm7 + add esi,16*4 # advance matrix C by 16 columns + cmp ebp,12 + ja .LProcessNextColumnLoop16x1 + test ebp,ebp + jnz .LProcessRemainingCountN + +// +// Restore non-volatile registers and return. +// + +.LExitKernel: + mov eax,1 # return 1 row handled + pop edi + pop esi + pop ebx + pop ebp + ret + +// +// Process the remaining 1 to 12 columns of the matrices. +// + +.LProcessRemainingCountN: + mov edi,eax # reload CountK + mov ecx,ebx # reload matrix A + movss xmm4,DWORD PTR .LSgemmKernelFrame_alpha[esp] + shufps xmm4,xmm4,0 + xorps xmm5,xmm5 # clear block accumulators + xorps xmm6,xmm6 + xorps xmm7,xmm7 + cmp ebp,4 + jbe .LProcessRemainingCountN4OrLess + cmp ebp,8 + jbe .LProcessRemainingCountN8OrLess + +.LProcessRemainingCountN12OrLess: + ComputeBlockSseLoop 3 + mulps xmm5,xmm4 # multiply by alpha + mulps xmm6,xmm4 + mulps xmm7,xmm4 + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateLeadingN12OrLess + movups xmm0,XMMWORD PTR [esi] + movups xmm1,XMMWORD PTR [esi+16] + addps xmm5,xmm0 + addps xmm6,xmm1 + +.LSkipAccumulateLeadingN12OrLess: + movups XMMWORD PTR [esi],xmm5 + movups XMMWORD PTR [esi+16],xmm6 + add esi,8*4 # advance matrix C by 8 columns + jmp .LOutputTrailingBlock + +.LProcessRemainingCountN8OrLess: + ComputeBlockSseLoop 2 + mulps xmm6,xmm4 # multiply by alpha + mulps xmm7,xmm4 + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateLeadingN8OrLess + movups xmm0,XMMWORD PTR [esi] + addps xmm6,xmm0 + +.LSkipAccumulateLeadingN8OrLess: + movups XMMWORD PTR [esi],xmm6 + add esi,4*4 # advance matrix C by 4 columns + jmp .LOutputTrailingBlock + +.LProcessRemainingCountN4OrLess: + ComputeBlockSseLoop 1 + mulps xmm7,xmm4 # multiply by alpha + jmp .LOutputTrailingBlock + +.LOutputMasked16x1Block: + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateLeading16x1Block + movups xmm0,XMMWORD PTR [esi] + movups xmm1,XMMWORD PTR [esi+16] + movups xmm2,XMMWORD PTR [esi+32] + addps xmm4,xmm0 + addps xmm5,xmm1 + addps xmm6,xmm2 + +.LSkipAccumulateLeading16x1Block: + movups XMMWORD PTR [esi],xmm4 + movups XMMWORD PTR [esi+16],xmm5 + movups XMMWORD PTR [esi+32],xmm6 + add esi,12*4 # advance matrix C by 12 columns + +.LOutputTrailingBlock: + test ebp,3 + jz .LOutputTrailingBlock4Elements + test ebp,2 + jz .LOutputTrailingBlock1Element + +.LOutputTrailingBlock2Elements: + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateTrailingBlock2Elements + movsd xmm0,QWORD PTR [esi] + addps xmm7,xmm0 + +.LSkipAccumulateTrailingBlock2Elements: + movsd QWORD PTR [esi],xmm7 + test ebp,1 + jz .LExitKernel + shufps xmm7,xmm7,0xAA # shuffle third float down + add esi,2*4 # advance matrix C by 2 columns + +.LOutputTrailingBlock1Element: + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateTrailingBlock1Element + movss xmm0,DWORD PTR [esi] + addss xmm7,xmm0 + +.LSkipAccumulateTrailingBlock1Element: + movss DWORD PTR [esi],xmm7 + jmp .LExitKernel + +.LOutputTrailingBlock4Elements: + cmp BYTE PTR .LSgemmKernelFrame_ZeroMode[esp],0 + jnz .LSkipAccumulateTrailingBlock4Elements + movups xmm0,XMMWORD PTR [esi] + addps xmm7,xmm0 + +.LSkipAccumulateTrailingBlock4Elements: + movups XMMWORD PTR [esi],xmm7 + jmp .LExitKernel + + .end diff --git a/3rdparty/mlas/lib/x86/asmmacro.h b/3rdparty/mlas/lib/x86/asmmacro.h new file mode 100644 index 0000000000..4b80eea735 --- /dev/null +++ b/3rdparty/mlas/lib/x86/asmmacro.h @@ -0,0 +1,79 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + asmmacro.h + +Abstract: + + This module implements common macros for the assembly modules. + +--*/ + +#if defined(__APPLE__) +#define C_UNDERSCORE(symbol) _##symbol +#else +#define C_UNDERSCORE(symbol) symbol +#endif + +/*++ + +Macro Description: + + This macro emits the assembler directives to annotate a new function. + +Arguments: + + FunctionName - Supplies the name of the function. + +--*/ + + .macro FUNCTION_ENTRY FunctionName + + .p2align 4 +#if defined(__APPLE__) + .globl _\FunctionName\() +_\FunctionName\(): +#else + .globl \FunctionName\() + .type \FunctionName\(),@function +\FunctionName\(): +#endif + + .endm + +/*++ + +Macro Description: + + This macro emits the code to load the global offset table address into the + supplied register. + +Arguments: + + TargetReg - Specifies the target register. + +--*/ + + .macro LoadGlobalOffsetTable, TargetReg + +// +// The LLVM integrated assembler doesn't support the Intel syntax for OFFSET: +// +// add ebx,OFFSET _GLOBAL_OFFSET_TABLE_ +// +// Workaround this by temporarily switching to AT&T syntax. +// + + .att_syntax + + calll __x86.get_pc_thunk.\TargetReg\() + addl $_GLOBAL_OFFSET_TABLE_,%e\TargetReg\() + + .intel_syntax noprefix + + .endm diff --git a/3rdparty/mlas/lib/x86/x86.get_pc_thunk.S b/3rdparty/mlas/lib/x86/x86.get_pc_thunk.S new file mode 100644 index 0000000000..8043401d62 --- /dev/null +++ b/3rdparty/mlas/lib/x86/x86.get_pc_thunk.S @@ -0,0 +1,36 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + x86.get_pc_thunk.S + +Abstract: + + This module implements __x86.get_pc_thunk.* to avoid external dependency. + +--*/ + + .intel_syntax noprefix + +/*++ + +Routine Description: + + The routine loads its return address -- which is the address of the + instruction that immediately follows -- into the ebx register. + +--*/ + + .p2align 4 + .weak __x86.get_pc_thunk.bx + // Hidden = non-preemptible, so the PC-relative call in the SGEMM + // kernels binds locally in the .so (no R_386_PC32 link error). + .hidden __x86.get_pc_thunk.bx + .type __x86.get_pc_thunk.bx,@function +__x86.get_pc_thunk.bx: + mov ebx, [esp] + ret diff --git a/CMakeLists.txt b/CMakeLists.txt index 44e01f5724..d6812aa76e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,14 +77,10 @@ if(OPENCV_WORKAROUND_CMAKE_20989) set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR_BACKUP}) endif() -# Enable generic ASM at top-level scope: enable_language(ASM) from a subdir fails -# (CMAKE_ASM_COMPILE_OBJECT unset) once ASM_NASM is enabled. Skip on Android (NDK). -if(NOT ANDROID AND NOT CMAKE_SYSTEM_NAME STREQUAL "Android") - include(CheckLanguage) - check_language(ASM) - if(CMAKE_ASM_COMPILER) - enable_language(ASM) - endif() +include(CheckLanguage) +check_language(ASM) +if(CMAKE_ASM_COMPILER) + enable_language(ASM) endif() enable_testing()