--- mem_transfer_mmx.asm 2004/04/12 15:49:56 1.11 +++ mem_transfer_mmx.asm 2008/11/26 01:04:34 1.20 @@ -4,9 +4,8 @@ ; * - 8<->16 bit transfer functions - ; * ; * Copyright (C) 2001 Peter Ross -; * 2001 Michael Militzer +; * 2001-2008 Michael Militzer ; * 2002 Pascal Massimino -; * 2004 Jean-Marc Bastide ; * ; * This program is free software ; you can redistribute it and/or modify ; * it under the terms of the GNU General Public License as published by @@ -22,32 +21,19 @@ ; * along with this program ; if not, write to the Free Software ; * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ; * -; * $Id: mem_transfer_mmx.asm,v 1.11 2004/04/12 15:49:56 edgomez Exp $ +; * $Id: mem_transfer_mmx.asm,v 1.20 2008/11/26 01:04:34 Isibaar Exp $ ; * ; ***************************************************************************/ -BITS 32 - -%macro cglobal 1 - %ifdef PREFIX - global _%1 - %define %1 _%1 - %else - global %1 - %endif -%endmacro +%include "nasm.inc" ;============================================================================= ; Read only data ;============================================================================= -%ifdef FORMAT_COFF -SECTION .rodata data -%else -SECTION .rodata data align=16 -%endif +DATA -ALIGN 16 +ALIGN SECTION_ALIGN mmx_one: dw 1, 1, 1, 1 @@ -55,7 +41,7 @@ ; Code ;============================================================================= -SECTION .text +SECTION .rotext align=SECTION_ALIGN cglobal transfer_8to16copy_mmx cglobal transfer_16to8copy_mmx @@ -63,8 +49,10 @@ cglobal transfer_8to16subro_mmx cglobal transfer_8to16sub2_mmx cglobal transfer_8to16sub2_xmm +cglobal transfer_8to16sub2ro_xmm cglobal transfer_16to8add_mmx cglobal transfer8x8_copy_mmx +cglobal transfer8x4_copy_mmx ;----------------------------------------------------------------------------- ; @@ -75,27 +63,27 @@ ;----------------------------------------------------------------------------- %macro COPY_8_TO_16 1 - movq mm0, [eax] - movq mm1, [eax+edx] + movq mm0, [_EAX] + movq mm1, [_EAX+TMP1] movq mm2, mm0 movq mm3, mm1 punpcklbw mm0, mm7 - movq [ecx+%1*32], mm0 + movq [TMP0+%1*32], mm0 punpcklbw mm1, mm7 - movq [ecx+%1*32+16], mm1 + movq [TMP0+%1*32+16], mm1 punpckhbw mm2, mm7 punpckhbw mm3, mm7 - lea eax, [eax+2*edx] - movq [ecx+%1*32+8], mm2 - movq [ecx+%1*32+24], mm3 + lea _EAX, [_EAX+2*TMP1] + movq [TMP0+%1*32+8], mm2 + movq [TMP0+%1*32+24], mm3 %endmacro -ALIGN 16 +ALIGN SECTION_ALIGN transfer_8to16copy_mmx: - mov ecx, [esp+ 4] ; Dst - mov eax, [esp+ 8] ; Src - mov edx, [esp+12] ; Stride + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Src + mov TMP1, prm3 ; Stride pxor mm7, mm7 COPY_8_TO_16 0 @@ -103,6 +91,7 @@ COPY_8_TO_16 2 COPY_8_TO_16 3 ret +ENDFUNC ;----------------------------------------------------------------------------- ; @@ -113,31 +102,32 @@ ;----------------------------------------------------------------------------- %macro COPY_16_TO_8 1 - movq mm0, [eax+%1*32] - movq mm1, [eax+%1*32+8] + movq mm0, [_EAX+%1*32] + movq mm1, [_EAX+%1*32+8] packuswb mm0, mm1 - movq [ecx], mm0 - movq mm2, [eax+%1*32+16] - movq mm3, [eax+%1*32+24] + movq [TMP0], mm0 + movq mm2, [_EAX+%1*32+16] + movq mm3, [_EAX+%1*32+24] packuswb mm2, mm3 - movq [ecx+edx], mm2 + movq [TMP0+TMP1], mm2 %endmacro -ALIGN 16 +ALIGN SECTION_ALIGN transfer_16to8copy_mmx: - mov ecx, [esp+ 4] ; Dst - mov eax, [esp+ 8] ; Src - mov edx, [esp+12] ; Stride + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Src + mov TMP1, prm3 ; Stride COPY_16_TO_8 0 - lea ecx,[ecx+2*edx] + lea TMP0,[TMP0+2*TMP1] COPY_16_TO_8 1 - lea ecx,[ecx+2*edx] + lea TMP0,[TMP0+2*TMP1] COPY_16_TO_8 2 - lea ecx,[ecx+2*edx] + lea TMP0,[TMP0+2*TMP1] COPY_16_TO_8 3 ret +ENDFUNC ;----------------------------------------------------------------------------- ; @@ -148,45 +138,55 @@ ; ;----------------------------------------------------------------------------- -; when second argument == 1, reference (ebx) block is to current (eax) +; when second argument == 1, reference (ebx) block is to current (_EAX) %macro COPY_8_TO_16_SUB 2 - movq mm0, [eax] ; cur - movq mm2, [eax+edx] + movq mm0, [_EAX] ; cur + movq mm2, [_EAX+TMP1] + movq mm1, mm0 + movq mm3, mm2 - movq mm4, [ebx] ; ref - movq mm5, [ebx+edx] ; ref + punpcklbw mm0, mm7 + punpcklbw mm2, mm7 + movq mm4, [_EBX] ; ref + punpckhbw mm1, mm7 + punpckhbw mm3, mm7 + movq mm5, [_EBX+TMP1] ; ref + movq mm6, mm4 %if %2 == 1 - movq [eax], mm4 - movq [eax+edx], mm5 + movq [_EAX], mm4 + movq [_EAX+TMP1], mm5 %endif - - lea eax, [eax+2*edx] - - psubsb mm0,mm4 - psubsb mm2,mm5 - lea ebx,[ebx+2*edx] - - movq mm1,mm0 - movq mm3,mm2 - punpcklbw mm0,mm7 - punpckhbw mm1,mm7 - punpcklbw mm2,mm7 - punpckhbw mm3,mm7 - - movq [ecx+%1*32+ 0], mm0 ; dst - movq [ecx+%1*32+ 8], mm1 - movq [ecx+%1*32+16], mm2 - movq [ecx+%1*32+24], mm3 + punpcklbw mm4, mm7 + punpckhbw mm6, mm7 + psubsw mm0, mm4 + psubsw mm1, mm6 + movq mm6, mm5 + punpcklbw mm5, mm7 + punpckhbw mm6, mm7 + psubsw mm2, mm5 + lea _EAX, [_EAX+2*TMP1] + psubsw mm3, mm6 + lea _EBX,[_EBX+2*TMP1] + + movq [TMP0+%1*32+ 0], mm0 ; dst + movq [TMP0+%1*32+ 8], mm1 + movq [TMP0+%1*32+16], mm2 + movq [TMP0+%1*32+24], mm3 %endmacro -ALIGN 16 +ALIGN SECTION_ALIGN transfer_8to16sub_mmx: - mov ecx, [esp + 4] ; Dst - mov eax, [esp + 8] ; Cur - push ebx - mov ebx, [esp+4+12] ; Ref - mov edx, [esp+4+16] ; Stride + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Cur + mov TMP1, prm4 ; Stride + + push _EBX +%ifdef ARCH_IS_X86_64 + mov _EBX, prm3 +%else + mov _EBX, [_ESP+4+12] ; Ref +%endif pxor mm7, mm7 COPY_8_TO_16_SUB 0, 1 @@ -194,17 +194,23 @@ COPY_8_TO_16_SUB 2, 1 COPY_8_TO_16_SUB 3, 1 - pop ebx + pop _EBX ret +ENDFUNC -ALIGN 16 +ALIGN SECTION_ALIGN transfer_8to16subro_mmx: - mov ecx, [esp + 4] ; Dst - mov eax, [esp + 8] ; Cur - push ebx - mov ebx, [esp+4+12] ; Ref - mov edx, [esp+4+16] ; Stride + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Cur + mov TMP1, prm4 ; Stride + + push _EBX +%ifdef ARCH_IS_X86_64 + mov _EBX, prm3 +%else + mov _EBX, [_ESP+4+12] ; Ref +%endif pxor mm7, mm7 COPY_8_TO_16_SUB 0, 0 @@ -212,8 +218,9 @@ COPY_8_TO_16_SUB 2, 0 COPY_8_TO_16_SUB 3, 0 - pop ebx + pop _EBX ret +ENDFUNC ;----------------------------------------------------------------------------- @@ -227,72 +234,103 @@ ;----------------------------------------------------------------------------- %macro COPY_8_TO_16_SUB2_MMX 1 - movq mm0, [eax] ; cur - movq mm2, [eax+edx] - + movq mm0, [_EAX] ; cur + movq mm2, [_EAX+TMP1] + ; mm4 <- (ref1+ref2+1) / 2 - ;(a+b+1)/2 = (a|b)-((a^b)>>1) - movq mm4, [ebx] ; ref1 - movq mm1, [esi] ; ref2 - movq mm3, mm4 - pxor mm3,mm1 - por mm4,mm1 - pandn mm3,mm6 - psrlq mm3,1 - psubb mm4,mm3 - movq [eax],mm4 - - ; mm5 <- (ref1+ref2+1) / 2 - movq mm5, [ebx+edx] ; ref1 - movq mm1, [esi+edx] ; ref2 - movq mm3, mm5 - pxor mm3,mm1 - por mm5,mm1 - pandn mm3,mm6 - psrlq mm3,1 - psubb mm5,mm3 - movq [eax+edx],mm5 - - psubsb mm0,mm4 - psubsb mm2,mm5 - lea esi, [esi+2*edx] - movq mm1,mm0 - movq mm3,mm2 - lea eax, [eax+2*edx] - punpcklbw mm0,mm7 - punpcklbw mm2,mm7 - lea ebx, [ebx+2*edx] - punpckhbw mm1,mm7 - punpckhbw mm3,mm7 - - movq [ecx+%1*32+ 0], mm0 ; dst - movq [ecx+%1*32+ 8], mm1 - movq [ecx+%1*32+16], mm2 - movq [ecx+%1*32+24], mm3 + movq mm4, [_EBX] ; ref1 + movq mm1, [_ESI] ; ref2 + movq mm6, mm4 + movq mm3, mm1 + punpcklbw mm4, mm7 + punpcklbw mm1, mm7 + punpckhbw mm6, mm7 + punpckhbw mm3, mm7 + paddusw mm4, mm1 + paddusw mm6, mm3 + paddusw mm4, [mmx_one] + paddusw mm6, [mmx_one] + psrlw mm4, 1 + psrlw mm6, 1 + packuswb mm4, mm6 + movq [_EAX], mm4 + + ; mm5 <- (ref1+ref2+1) / 2 + movq mm5, [_EBX+TMP1] ; ref1 + movq mm1, [_ESI+TMP1] ; ref2 + movq mm6, mm5 + movq mm3, mm1 + punpcklbw mm5, mm7 + punpcklbw mm1, mm7 + punpckhbw mm6, mm7 + punpckhbw mm3, mm7 + paddusw mm5, mm1 + paddusw mm6, mm3 + paddusw mm5, [mmx_one] + paddusw mm6, [mmx_one] + lea _ESI, [_ESI+2*TMP1] + psrlw mm5, 1 + psrlw mm6, 1 + packuswb mm5, mm6 + movq [_EAX+TMP1], mm5 + + movq mm1, mm0 + movq mm3, mm2 + punpcklbw mm0, mm7 + punpcklbw mm2, mm7 + punpckhbw mm1, mm7 + punpckhbw mm3, mm7 + + movq mm6, mm4 + punpcklbw mm4, mm7 + punpckhbw mm6, mm7 + psubsw mm0, mm4 + psubsw mm1, mm6 + movq mm6, mm5 + punpcklbw mm5, mm7 + punpckhbw mm6, mm7 + psubsw mm2, mm5 + lea _EAX, [_EAX+2*TMP1] + psubsw mm3, mm6 + lea _EBX, [_EBX+2*TMP1] + + movq [TMP0+%1*32+ 0], mm0 ; dst + movq [TMP0+%1*32+ 8], mm1 + movq [TMP0+%1*32+16], mm2 + movq [TMP0+%1*32+24], mm3 %endmacro -ALIGN 16 +ALIGN SECTION_ALIGN transfer_8to16sub2_mmx: - mov ecx, [esp + 4] ; Dst - mov eax, [esp + 8] ; Cur - push ebx - mov ebx, [esp+4+12] ; Ref1 - push esi - mov esi, [esp+8+16] ; Ref2 - mov edx, [esp+8+20] ; Stride - pxor mm6,mm6 - pcmpeqb mm5,mm5 + mov TMP0, prm1 ; Dst + mov TMP1d, prm5d ; Stride + mov _EAX, prm2 ; Cur + + push _EBX +%ifdef ARCH_IS_X86_64 + mov _EBX, prm3 +%else + mov _EBX, [_ESP+4+12] ; Ref1 +%endif + + push _ESI +%ifdef ARCH_IS_X86_64 + mov _ESI, prm4 +%else + mov _ESI, [_ESP+8+16] ; Ref2 +%endif + pxor mm7, mm7 - psubb mm6, mm5; mm6=1 COPY_8_TO_16_SUB2_MMX 0 COPY_8_TO_16_SUB2_MMX 1 COPY_8_TO_16_SUB2_MMX 2 COPY_8_TO_16_SUB2_MMX 3 - pop esi - pop ebx + pop _ESI + pop _EBX ret +ENDFUNC ;----------------------------------------------------------------------------- ; @@ -305,45 +343,62 @@ ;----------------------------------------------------------------------------- %macro COPY_8_TO_16_SUB2_SSE 1 - movq mm0, [eax] ; cur - movq mm2, [eax+edx] + movq mm0, [_EAX] ; cur + movq mm2, [_EAX+TMP1] + movq mm1, mm0 + movq mm3, mm2 - movq mm4, [ebx] ; ref1 - pavgb mm4, [esi] ; ref2 - - movq mm5, [ebx+edx] ; ref - pavgb mm5, [esi+edx] ; ref2 - - movq [eax], mm4 - movq [eax+edx], mm5 - - psubsb mm0,mm4 - psubsb mm2,mm5 - lea esi, [esi+2*edx] - movq mm1,mm0 - movq mm3,mm2 - lea eax, [eax+2*edx] - punpcklbw mm0,mm7 - punpcklbw mm2,mm7 - lea ebx, [ebx+2*edx] - punpckhbw mm1,mm7 - punpckhbw mm3,mm7 - - movq [ecx+%1*32+ 0], mm0 ; dst - movq [ecx+%1*32+ 8], mm1 - movq [ecx+%1*32+16], mm2 - movq [ecx+%1*32+24], mm3 + punpcklbw mm0, mm7 + punpcklbw mm2, mm7 + movq mm4, [_EBX] ; ref1 + pavgb mm4, [_ESI] ; ref2 + movq [_EAX], mm4 + punpckhbw mm1, mm7 + punpckhbw mm3, mm7 + movq mm5, [_EBX+TMP1] ; ref + pavgb mm5, [_ESI+TMP1] ; ref2 + movq [_EAX+TMP1], mm5 + + movq mm6, mm4 + punpcklbw mm4, mm7 + punpckhbw mm6, mm7 + psubsw mm0, mm4 + psubsw mm1, mm6 + lea _ESI, [_ESI+2*TMP1] + movq mm6, mm5 + punpcklbw mm5, mm7 + punpckhbw mm6, mm7 + psubsw mm2, mm5 + lea _EAX, [_EAX+2*TMP1] + psubsw mm3, mm6 + lea _EBX, [_EBX+2*TMP1] + + movq [TMP0+%1*32+ 0], mm0 ; dst + movq [TMP0+%1*32+ 8], mm1 + movq [TMP0+%1*32+16], mm2 + movq [TMP0+%1*32+24], mm3 %endmacro -ALIGN 16 +ALIGN SECTION_ALIGN transfer_8to16sub2_xmm: - mov ecx, [esp + 4] ; Dst - mov eax, [esp + 8] ; Cur - push ebx - mov ebx, [esp+4+12] ; Ref1 - push esi - mov esi, [esp+8+16] ; Ref2 - mov edx, [esp+8+20] ; Stride + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Cur + mov TMP1d, prm5d ; Stride + + push _EBX +%ifdef ARCH_IS_X86_64 + mov _EBX, prm3 ; Ref1 +%else + mov _EBX, [_ESP+4+12] ; Ref1 +%endif + + push _ESI +%ifdef ARCH_IS_X86_64 + mov _ESI, prm4 ; Ref1 +%else + mov _ESI, [_ESP+8+16] ; Ref2 +%endif + pxor mm7, mm7 COPY_8_TO_16_SUB2_SSE 0 @@ -351,9 +406,88 @@ COPY_8_TO_16_SUB2_SSE 2 COPY_8_TO_16_SUB2_SSE 3 - pop esi - pop ebx + pop _ESI + pop _EBX ret +ENDFUNC + + +;----------------------------------------------------------------------------- +; +; void transfer_8to16sub2ro_xmm(int16_t * const dct, +; const uint8_t * const cur, +; const uint8_t * ref1, +; const uint8_t * ref2, +; const uint32_t stride) +; +;----------------------------------------------------------------------------- + +%macro COPY_8_TO_16_SUB2RO_SSE 1 + movq mm0, [_EAX] ; cur + movq mm2, [_EAX+TMP1] + movq mm1, mm0 + movq mm3, mm2 + + punpcklbw mm0, mm7 + punpcklbw mm2, mm7 + movq mm4, [_EBX] ; ref1 + pavgb mm4, [_ESI] ; ref2 + punpckhbw mm1, mm7 + punpckhbw mm3, mm7 + movq mm5, [_EBX+TMP1] ; ref + pavgb mm5, [_ESI+TMP1] ; ref2 + + movq mm6, mm4 + punpcklbw mm4, mm7 + punpckhbw mm6, mm7 + psubsw mm0, mm4 + psubsw mm1, mm6 + lea _ESI, [_ESI+2*TMP1] + movq mm6, mm5 + punpcklbw mm5, mm7 + punpckhbw mm6, mm7 + psubsw mm2, mm5 + lea _EAX, [_EAX+2*TMP1] + psubsw mm3, mm6 + lea _EBX, [_EBX+2*TMP1] + + movq [TMP0+%1*32+ 0], mm0 ; dst + movq [TMP0+%1*32+ 8], mm1 + movq [TMP0+%1*32+16], mm2 + movq [TMP0+%1*32+24], mm3 +%endmacro + +ALIGN SECTION_ALIGN +transfer_8to16sub2ro_xmm: + pxor mm7, mm7 + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Cur + mov TMP1d, prm5d ; Stride + + push _EBX +%ifdef ARCH_IS_X86_64 + mov _EBX, prm3 +%else + mov _EBX, [_ESP+4+12] ; Ref1 +%endif + + push _ESI +%ifdef ARCH_IS_X86_64 + mov _ESI, prm4 +%else + mov _ESI, [_ESP+8+16] ; Ref2 +%endif + + COPY_8_TO_16_SUB2RO_SSE 0 + COPY_8_TO_16_SUB2RO_SSE 1 + COPY_8_TO_16_SUB2RO_SSE 2 + COPY_8_TO_16_SUB2RO_SSE 3 + + pop _ESI + pop _EBX + ret +ENDFUNC + ;----------------------------------------------------------------------------- ; @@ -364,34 +498,41 @@ ;----------------------------------------------------------------------------- %macro COPY_16_TO_8_ADD 1 - movq mm0, [eax+%1*32+ 0] ;src - packuswb mm0,[eax+%1*32+8] - movq mm1, [eax+%1*32+ 16] - packuswb mm1,[eax+%1*32+24] - - paddusb mm0,[ecx] - paddusb mm1,[ecx+edx] - movq [ecx],mm0 - movq [ecx+edx],mm1 - + movq mm0, [TMP0] + movq mm2, [TMP0+TMP1] + movq mm1, mm0 + movq mm3, mm2 + punpcklbw mm0, mm7 + punpcklbw mm2, mm7 + punpckhbw mm1, mm7 + punpckhbw mm3, mm7 + paddsw mm0, [_EAX+%1*32+ 0] + paddsw mm1, [_EAX+%1*32+ 8] + paddsw mm2, [_EAX+%1*32+16] + paddsw mm3, [_EAX+%1*32+24] + packuswb mm0, mm1 + movq [TMP0], mm0 + packuswb mm2, mm3 + movq [TMP0+TMP1], mm2 %endmacro -ALIGN 16 +ALIGN SECTION_ALIGN transfer_16to8add_mmx: - mov ecx, [esp+ 4] ; Dst - mov eax, [esp+ 8] ; Src - mov edx, [esp+12] ; Stride - ; pxor mm7, mm7 + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Src + mov TMP1, prm3 ; Stride + pxor mm7, mm7 COPY_16_TO_8_ADD 0 - lea ecx,[ecx+2*edx] + lea TMP0,[TMP0+2*TMP1] COPY_16_TO_8_ADD 1 - lea ecx,[ecx+2*edx] + lea TMP0,[TMP0+2*TMP1] COPY_16_TO_8_ADD 2 - lea ecx,[ecx+2*edx] + lea TMP0,[TMP0+2*TMP1] COPY_16_TO_8_ADD 3 ret +ENDFUNC ;----------------------------------------------------------------------------- ; @@ -402,43 +543,53 @@ ; ;----------------------------------------------------------------------------- -ALIGN 16 +%macro COPY_8_TO_8 0 + movq mm0, [_EAX] + movq mm1, [_EAX+TMP1] + movq [TMP0], mm0 + lea _EAX, [_EAX+2*TMP1] + movq [TMP0+TMP1], mm1 +%endmacro + +ALIGN SECTION_ALIGN transfer8x8_copy_mmx: + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Src + mov TMP1, prm3 ; Stride + + COPY_8_TO_8 + lea TMP0,[TMP0+2*TMP1] + COPY_8_TO_8 + lea TMP0,[TMP0+2*TMP1] + COPY_8_TO_8 + lea TMP0,[TMP0+2*TMP1] + COPY_8_TO_8 + ret +ENDFUNC + +;----------------------------------------------------------------------------- +; +; void transfer8x4_copy_mmx(uint8_t * const dst, +; const uint8_t * const src, +; const uint32_t stride); +; +; +;----------------------------------------------------------------------------- + +ALIGN SECTION_ALIGN +transfer8x4_copy_mmx: + mov TMP0, prm1 ; Dst + mov _EAX, prm2 ; Src + mov TMP1, prm3 ; Stride + + COPY_8_TO_8 + lea TMP0,[TMP0+2*TMP1] + COPY_8_TO_8 + ret +ENDFUNC + + +%ifidn __OUTPUT_FORMAT__,elf +section ".note.GNU-stack" noalloc noexec nowrite progbits +%endif - mov eax, [esp+ 8] ; Src - mov ecx, [esp+ 4] ; Dst - mov edx, [esp+12] ; Stride - - movq mm0,[eax] - lea eax,[eax+edx] - movq mm1,[eax] - lea eax,[eax+edx] - movq mm2,[eax] - lea eax,[eax+edx] - movq mm3,[eax] - lea eax,[eax+edx] - movq mm4,[eax] - lea eax,[eax+edx] - movq mm5,[eax] - lea eax,[eax+edx] - movq mm6,[eax] - lea eax,[eax+edx] - movq mm7,[eax] - - movq [ecx],mm0 - lea ecx,[ecx+edx] - movq [ecx],mm1 - lea ecx,[ecx+edx] - movq [ecx],mm2 - lea ecx,[ecx+edx] - movq [ecx],mm3 - lea ecx,[ecx+edx] - movq [ecx],mm4 - lea ecx,[ecx+edx] - movq [ecx],mm5 - lea ecx,[ecx+edx] - movq [ecx],mm6 - lea ecx,[ecx+edx] - movq [ecx],mm7 - -ret