michael@0: ;
michael@0: ; jdsammmx.asm - upsampling (MMX)
michael@0: ;
michael@0: ; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
michael@0: ;
michael@0: ; Based on
michael@0: ; x86 SIMD extension for IJG JPEG library
michael@0: ; Copyright (C) 1999-2006, MIYASAKA Masaru.
michael@0: ; For conditions of distribution and use, see copyright notice in jsimdext.inc
michael@0: ;
michael@0: ; This file should be assembled with NASM (Netwide Assembler),
michael@0: ; can *not* be assembled with Microsoft's MASM or any compatible
michael@0: ; assembler (including Borland's Turbo Assembler).
michael@0: ; NASM is available from http://nasm.sourceforge.net/ or
michael@0: ; http://sourceforge.net/project/showfiles.php?group_id=6208
michael@0: ;
michael@0: ; [TAB8]
michael@0: 
michael@0: %include "jsimdext.inc"
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: 	SECTION	SEG_CONST
michael@0: 
michael@0: 	alignz	16
michael@0: 	global	EXTN(jconst_fancy_upsample_mmx)
michael@0: 
michael@0: EXTN(jconst_fancy_upsample_mmx):
michael@0: 
michael@0: PW_ONE		times 4 dw  1
michael@0: PW_TWO		times 4 dw  2
michael@0: PW_THREE	times 4 dw  3
michael@0: PW_SEVEN	times 4 dw  7
michael@0: PW_EIGHT	times 4 dw  8
michael@0: 
michael@0: 	alignz	16
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: 	SECTION	SEG_TEXT
michael@0: 	BITS	32
michael@0: ;
michael@0: ; Fancy processing for the common case of 2:1 horizontal and 1:1 vertical.
michael@0: ;
michael@0: ; The upsampling algorithm is linear interpolation between pixel centers,
michael@0: ; also known as a "triangle filter".  This is a good compromise between
michael@0: ; speed and visual quality.  The centers of the output pixels are 1/4 and 3/4
michael@0: ; of the way between input pixel centers.
michael@0: ;
michael@0: ; GLOBAL(void)
michael@0: ; jsimd_h2v1_fancy_upsample_mmx (int max_v_samp_factor,
michael@0: ;                                JDIMENSION downsampled_width,
michael@0: ;                                JSAMPARRAY input_data,
michael@0: ;                                JSAMPARRAY * output_data_ptr);
michael@0: ;
michael@0: 
michael@0: %define max_v_samp(b)		(b)+8			; int max_v_samp_factor
michael@0: %define downsamp_width(b)	(b)+12	; JDIMENSION downsampled_width
michael@0: %define input_data(b)		(b)+16		; JSAMPARRAY input_data
michael@0: %define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr
michael@0: 
michael@0: 	align	16
michael@0: 	global	EXTN(jsimd_h2v1_fancy_upsample_mmx)
michael@0: 
michael@0: EXTN(jsimd_h2v1_fancy_upsample_mmx):
michael@0: 	push	ebp
michael@0: 	mov	ebp,esp
michael@0: 	pushpic	ebx
michael@0: ;	push	ecx		; need not be preserved
michael@0: ;	push	edx		; need not be preserved
michael@0: 	push	esi
michael@0: 	push	edi
michael@0: 
michael@0: 	get_GOT	ebx		; get GOT address
michael@0: 
michael@0: 	mov	eax, JDIMENSION [downsamp_width(ebp)]  ; colctr
michael@0: 	test	eax,eax
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	ecx, INT [max_v_samp(ebp)]	; rowctr
michael@0: 	test	ecx,ecx
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
michael@0: 	mov	edi, POINTER [output_data_ptr(ebp)]
michael@0: 	mov	edi, JSAMPARRAY [edi]			; output_data
michael@0: 	alignx	16,7
michael@0: .rowloop:
michael@0: 	push	eax			; colctr
michael@0: 	push	edi
michael@0: 	push	esi
michael@0: 
michael@0: 	mov	esi, JSAMPROW [esi]	; inptr
michael@0: 	mov	edi, JSAMPROW [edi]	; outptr
michael@0: 
michael@0: 	test	eax, SIZEOF_MMWORD-1
michael@0: 	jz	short .skip
michael@0: 	mov	dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
michael@0: 	mov	JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl	; insert a dummy sample
michael@0: .skip:
michael@0: 	pxor	mm0,mm0			; mm0=(all 0's)
michael@0: 	pcmpeqb	mm7,mm7
michael@0: 	psrlq	mm7,(SIZEOF_MMWORD-1)*BYTE_BIT
michael@0: 	pand	mm7, MMWORD [esi+0*SIZEOF_MMWORD]
michael@0: 
michael@0: 	add	eax, byte SIZEOF_MMWORD-1
michael@0: 	and	eax, byte -SIZEOF_MMWORD
michael@0: 	cmp	eax, byte SIZEOF_MMWORD
michael@0: 	ja	short .columnloop
michael@0: 	alignx	16,7
michael@0: 
michael@0: .columnloop_last:
michael@0: 	pcmpeqb	mm6,mm6
michael@0: 	psllq	mm6,(SIZEOF_MMWORD-1)*BYTE_BIT
michael@0: 	pand	mm6, MMWORD [esi+0*SIZEOF_MMWORD]
michael@0: 	jmp	short .upsample
michael@0: 	alignx	16,7
michael@0: 
michael@0: .columnloop:
michael@0: 	movq	mm6, MMWORD [esi+1*SIZEOF_MMWORD]
michael@0: 	psllq	mm6,(SIZEOF_MMWORD-1)*BYTE_BIT
michael@0: 
michael@0: .upsample:
michael@0: 	movq	mm1, MMWORD [esi+0*SIZEOF_MMWORD]
michael@0: 	movq	mm2,mm1
michael@0: 	movq	mm3,mm1			; mm1=( 0 1 2 3 4 5 6 7)
michael@0: 	psllq	mm2,BYTE_BIT		; mm2=( - 0 1 2 3 4 5 6)
michael@0: 	psrlq	mm3,BYTE_BIT		; mm3=( 1 2 3 4 5 6 7 -)
michael@0: 
michael@0: 	por	mm2,mm7			; mm2=(-1 0 1 2 3 4 5 6)
michael@0: 	por	mm3,mm6			; mm3=( 1 2 3 4 5 6 7 8)
michael@0: 
michael@0: 	movq	mm7,mm1
michael@0: 	psrlq	mm7,(SIZEOF_MMWORD-1)*BYTE_BIT	; mm7=( 7 - - - - - - -)
michael@0: 
michael@0: 	movq      mm4,mm1
michael@0: 	punpcklbw mm1,mm0		; mm1=( 0 1 2 3)
michael@0: 	punpckhbw mm4,mm0		; mm4=( 4 5 6 7)
michael@0: 	movq      mm5,mm2
michael@0: 	punpcklbw mm2,mm0		; mm2=(-1 0 1 2)
michael@0: 	punpckhbw mm5,mm0		; mm5=( 3 4 5 6)
michael@0: 	movq      mm6,mm3
michael@0: 	punpcklbw mm3,mm0		; mm3=( 1 2 3 4)
michael@0: 	punpckhbw mm6,mm0		; mm6=( 5 6 7 8)
michael@0: 
michael@0: 	pmullw	mm1,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]
michael@0: 	paddw	mm2,[GOTOFF(ebx,PW_ONE)]
michael@0: 	paddw	mm5,[GOTOFF(ebx,PW_ONE)]
michael@0: 	paddw	mm3,[GOTOFF(ebx,PW_TWO)]
michael@0: 	paddw	mm6,[GOTOFF(ebx,PW_TWO)]
michael@0: 
michael@0: 	paddw	mm2,mm1
michael@0: 	paddw	mm5,mm4
michael@0: 	psrlw	mm2,2			; mm2=OutLE=( 0  2  4  6)
michael@0: 	psrlw	mm5,2			; mm5=OutHE=( 8 10 12 14)
michael@0: 	paddw	mm3,mm1
michael@0: 	paddw	mm6,mm4
michael@0: 	psrlw	mm3,2			; mm3=OutLO=( 1  3  5  7)
michael@0: 	psrlw	mm6,2			; mm6=OutHO=( 9 11 13 15)
michael@0: 
michael@0: 	psllw	mm3,BYTE_BIT
michael@0: 	psllw	mm6,BYTE_BIT
michael@0: 	por	mm2,mm3			; mm2=OutL=( 0  1  2  3  4  5  6  7)
michael@0: 	por	mm5,mm6			; mm5=OutH=( 8  9 10 11 12 13 14 15)
michael@0: 
michael@0: 	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm2
michael@0: 	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm5
michael@0: 
michael@0: 	sub	eax, byte SIZEOF_MMWORD
michael@0: 	add	esi, byte 1*SIZEOF_MMWORD	; inptr
michael@0: 	add	edi, byte 2*SIZEOF_MMWORD	; outptr
michael@0: 	cmp	eax, byte SIZEOF_MMWORD
michael@0: 	ja	near .columnloop
michael@0: 	test	eax,eax
michael@0: 	jnz	near .columnloop_last
michael@0: 
michael@0: 	pop	esi
michael@0: 	pop	edi
michael@0: 	pop	eax
michael@0: 
michael@0: 	add	esi, byte SIZEOF_JSAMPROW	; input_data
michael@0: 	add	edi, byte SIZEOF_JSAMPROW	; output_data
michael@0: 	dec	ecx				; rowctr
michael@0: 	jg	near .rowloop
michael@0: 
michael@0: 	emms		; empty MMX state
michael@0: 
michael@0: .return:
michael@0: 	pop	edi
michael@0: 	pop	esi
michael@0: ;	pop	edx		; need not be preserved
michael@0: ;	pop	ecx		; need not be preserved
michael@0: 	poppic	ebx
michael@0: 	pop	ebp
michael@0: 	ret
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: ;
michael@0: ; Fancy processing for the common case of 2:1 horizontal and 2:1 vertical.
michael@0: ; Again a triangle filter; see comments for h2v1 case, above.
michael@0: ;
michael@0: ; GLOBAL(void)
michael@0: ; jsimd_h2v2_fancy_upsample_mmx (int max_v_samp_factor,
michael@0: ;                                JDIMENSION downsampled_width,
michael@0: ;                                JSAMPARRAY input_data,
michael@0: ;                                JSAMPARRAY * output_data_ptr);
michael@0: ;
michael@0: 
michael@0: %define max_v_samp(b)		(b)+8			; int max_v_samp_factor
michael@0: %define downsamp_width(b)	(b)+12	; JDIMENSION downsampled_width
michael@0: %define input_data(b)		(b)+16		; JSAMPARRAY input_data
michael@0: %define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr
michael@0: 
michael@0: %define original_ebp	ebp+0
michael@0: %define wk(i)		ebp-(WK_NUM-(i))*SIZEOF_MMWORD	; mmword wk[WK_NUM]
michael@0: %define WK_NUM		4
michael@0: %define gotptr		wk(0)-SIZEOF_POINTER	; void * gotptr
michael@0: 
michael@0: 	align	16
michael@0: 	global	EXTN(jsimd_h2v2_fancy_upsample_mmx)
michael@0: 
michael@0: EXTN(jsimd_h2v2_fancy_upsample_mmx):
michael@0: 	push	ebp
michael@0: 	mov	eax,esp				; eax = original ebp
michael@0: 	sub	esp, byte 4
michael@0: 	and	esp, byte (-SIZEOF_MMWORD)	; align to 64 bits
michael@0: 	mov	[esp],eax
michael@0: 	mov	ebp,esp				; ebp = aligned ebp
michael@0: 	lea	esp, [wk(0)]
michael@0: 	pushpic	eax		; make a room for GOT address
michael@0: 	push	ebx
michael@0: ;	push	ecx		; need not be preserved
michael@0: ;	push	edx		; need not be preserved
michael@0: 	push	esi
michael@0: 	push	edi
michael@0: 
michael@0: 	get_GOT	ebx			; get GOT address
michael@0: 	movpic	POINTER [gotptr], ebx	; save GOT address
michael@0: 
michael@0: 	mov	edx,eax				; edx = original ebp
michael@0: 	mov	eax, JDIMENSION [downsamp_width(edx)]  ; colctr
michael@0: 	test	eax,eax
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	ecx, INT [max_v_samp(edx)]	; rowctr
michael@0: 	test	ecx,ecx
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	esi, JSAMPARRAY [input_data(edx)]	; input_data
michael@0: 	mov	edi, POINTER [output_data_ptr(edx)]
michael@0: 	mov	edi, JSAMPARRAY [edi]			; output_data
michael@0: 	alignx	16,7
michael@0: .rowloop:
michael@0: 	push	eax					; colctr
michael@0: 	push	ecx
michael@0: 	push	edi
michael@0: 	push	esi
michael@0: 
michael@0: 	mov	ecx, JSAMPROW [esi-1*SIZEOF_JSAMPROW]	; inptr1(above)
michael@0: 	mov	ebx, JSAMPROW [esi+0*SIZEOF_JSAMPROW]	; inptr0
michael@0: 	mov	esi, JSAMPROW [esi+1*SIZEOF_JSAMPROW]	; inptr1(below)
michael@0: 	mov	edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]	; outptr0
michael@0: 	mov	edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]	; outptr1
michael@0: 
michael@0: 	test	eax, SIZEOF_MMWORD-1
michael@0: 	jz	short .skip
michael@0: 	push	edx
michael@0: 	mov	dl, JSAMPLE [ecx+(eax-1)*SIZEOF_JSAMPLE]
michael@0: 	mov	JSAMPLE [ecx+eax*SIZEOF_JSAMPLE], dl
michael@0: 	mov	dl, JSAMPLE [ebx+(eax-1)*SIZEOF_JSAMPLE]
michael@0: 	mov	JSAMPLE [ebx+eax*SIZEOF_JSAMPLE], dl
michael@0: 	mov	dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
michael@0: 	mov	JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl	; insert a dummy sample
michael@0: 	pop	edx
michael@0: .skip:
michael@0: 	; -- process the first column block
michael@0: 
michael@0: 	movq	mm0, MMWORD [ebx+0*SIZEOF_MMWORD]	; mm0=row[ 0][0]
michael@0: 	movq	mm1, MMWORD [ecx+0*SIZEOF_MMWORD]	; mm1=row[-1][0]
michael@0: 	movq	mm2, MMWORD [esi+0*SIZEOF_MMWORD]	; mm2=row[+1][0]
michael@0: 
michael@0: 	pushpic	ebx
michael@0: 	movpic	ebx, POINTER [gotptr]	; load GOT address
michael@0: 
michael@0: 	pxor      mm3,mm3		; mm3=(all 0's)
michael@0: 	movq      mm4,mm0
michael@0: 	punpcklbw mm0,mm3		; mm0=row[ 0][0]( 0 1 2 3)
michael@0: 	punpckhbw mm4,mm3		; mm4=row[ 0][0]( 4 5 6 7)
michael@0: 	movq      mm5,mm1
michael@0: 	punpcklbw mm1,mm3		; mm1=row[-1][0]( 0 1 2 3)
michael@0: 	punpckhbw mm5,mm3		; mm5=row[-1][0]( 4 5 6 7)
michael@0: 	movq      mm6,mm2
michael@0: 	punpcklbw mm2,mm3		; mm2=row[+1][0]( 0 1 2 3)
michael@0: 	punpckhbw mm6,mm3		; mm6=row[+1][0]( 4 5 6 7)
michael@0: 
michael@0: 	pmullw	mm0,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]
michael@0: 
michael@0: 	pcmpeqb	mm7,mm7
michael@0: 	psrlq	mm7,(SIZEOF_MMWORD-2)*BYTE_BIT
michael@0: 
michael@0: 	paddw	mm1,mm0			; mm1=Int0L=( 0 1 2 3)
michael@0: 	paddw	mm5,mm4			; mm5=Int0H=( 4 5 6 7)
michael@0: 	paddw	mm2,mm0			; mm2=Int1L=( 0 1 2 3)
michael@0: 	paddw	mm6,mm4			; mm6=Int1H=( 4 5 6 7)
michael@0: 
michael@0: 	movq	MMWORD [edx+0*SIZEOF_MMWORD], mm1	; temporarily save
michael@0: 	movq	MMWORD [edx+1*SIZEOF_MMWORD], mm5	; the intermediate data
michael@0: 	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm2
michael@0: 	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm6
michael@0: 
michael@0: 	pand	mm1,mm7			; mm1=( 0 - - -)
michael@0: 	pand	mm2,mm7			; mm2=( 0 - - -)
michael@0: 
michael@0: 	movq	MMWORD [wk(0)], mm1
michael@0: 	movq	MMWORD [wk(1)], mm2
michael@0: 
michael@0: 	poppic	ebx
michael@0: 
michael@0: 	add	eax, byte SIZEOF_MMWORD-1
michael@0: 	and	eax, byte -SIZEOF_MMWORD
michael@0: 	cmp	eax, byte SIZEOF_MMWORD
michael@0: 	ja	short .columnloop
michael@0: 	alignx	16,7
michael@0: 
michael@0: .columnloop_last:
michael@0: 	; -- process the last column block
michael@0: 
michael@0: 	pushpic	ebx
michael@0: 	movpic	ebx, POINTER [gotptr]	; load GOT address
michael@0: 
michael@0: 	pcmpeqb	mm1,mm1
michael@0: 	psllq	mm1,(SIZEOF_MMWORD-2)*BYTE_BIT
michael@0: 	movq	mm2,mm1
michael@0: 
michael@0: 	pand	mm1, MMWORD [edx+1*SIZEOF_MMWORD]	; mm1=( - - - 7)
michael@0: 	pand	mm2, MMWORD [edi+1*SIZEOF_MMWORD]	; mm2=( - - - 7)
michael@0: 
michael@0: 	movq	MMWORD [wk(2)], mm1
michael@0: 	movq	MMWORD [wk(3)], mm2
michael@0: 
michael@0: 	jmp	short .upsample
michael@0: 	alignx	16,7
michael@0: 
michael@0: .columnloop:
michael@0: 	; -- process the next column block
michael@0: 
michael@0: 	movq	mm0, MMWORD [ebx+1*SIZEOF_MMWORD]	; mm0=row[ 0][1]
michael@0: 	movq	mm1, MMWORD [ecx+1*SIZEOF_MMWORD]	; mm1=row[-1][1]
michael@0: 	movq	mm2, MMWORD [esi+1*SIZEOF_MMWORD]	; mm2=row[+1][1]
michael@0: 
michael@0: 	pushpic	ebx
michael@0: 	movpic	ebx, POINTER [gotptr]	; load GOT address
michael@0: 
michael@0: 	pxor      mm3,mm3		; mm3=(all 0's)
michael@0: 	movq      mm4,mm0
michael@0: 	punpcklbw mm0,mm3		; mm0=row[ 0][1]( 0 1 2 3)
michael@0: 	punpckhbw mm4,mm3		; mm4=row[ 0][1]( 4 5 6 7)
michael@0: 	movq      mm5,mm1
michael@0: 	punpcklbw mm1,mm3		; mm1=row[-1][1]( 0 1 2 3)
michael@0: 	punpckhbw mm5,mm3		; mm5=row[-1][1]( 4 5 6 7)
michael@0: 	movq      mm6,mm2
michael@0: 	punpcklbw mm2,mm3		; mm2=row[+1][1]( 0 1 2 3)
michael@0: 	punpckhbw mm6,mm3		; mm6=row[+1][1]( 4 5 6 7)
michael@0: 
michael@0: 	pmullw	mm0,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]
michael@0: 
michael@0: 	paddw	mm1,mm0			; mm1=Int0L=( 0 1 2 3)
michael@0: 	paddw	mm5,mm4			; mm5=Int0H=( 4 5 6 7)
michael@0: 	paddw	mm2,mm0			; mm2=Int1L=( 0 1 2 3)
michael@0: 	paddw	mm6,mm4			; mm6=Int1H=( 4 5 6 7)
michael@0: 
michael@0: 	movq	MMWORD [edx+2*SIZEOF_MMWORD], mm1	; temporarily save
michael@0: 	movq	MMWORD [edx+3*SIZEOF_MMWORD], mm5	; the intermediate data
michael@0: 	movq	MMWORD [edi+2*SIZEOF_MMWORD], mm2
michael@0: 	movq	MMWORD [edi+3*SIZEOF_MMWORD], mm6
michael@0: 
michael@0: 	psllq	mm1,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm1=( - - - 0)
michael@0: 	psllq	mm2,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm2=( - - - 0)
michael@0: 
michael@0: 	movq	MMWORD [wk(2)], mm1
michael@0: 	movq	MMWORD [wk(3)], mm2
michael@0: 
michael@0: .upsample:
michael@0: 	; -- process the upper row
michael@0: 
michael@0: 	movq	mm7, MMWORD [edx+0*SIZEOF_MMWORD]	; mm7=Int0L=( 0 1 2 3)
michael@0: 	movq	mm3, MMWORD [edx+1*SIZEOF_MMWORD]	; mm3=Int0H=( 4 5 6 7)
michael@0: 
michael@0: 	movq	mm0,mm7
michael@0: 	movq	mm4,mm3
michael@0: 	psrlq	mm0,2*BYTE_BIT			; mm0=( 1 2 3 -)
michael@0: 	psllq	mm4,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm4=( - - - 4)
michael@0: 	movq	mm5,mm7
michael@0: 	movq	mm6,mm3
michael@0: 	psrlq	mm5,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm5=( 3 - - -)
michael@0: 	psllq	mm6,2*BYTE_BIT			; mm6=( - 4 5 6)
michael@0: 
michael@0: 	por	mm0,mm4				; mm0=( 1 2 3 4)
michael@0: 	por	mm5,mm6				; mm5=( 3 4 5 6)
michael@0: 
michael@0: 	movq	mm1,mm7
michael@0: 	movq	mm2,mm3
michael@0: 	psllq	mm1,2*BYTE_BIT			; mm1=( - 0 1 2)
michael@0: 	psrlq	mm2,2*BYTE_BIT			; mm2=( 5 6 7 -)
michael@0: 	movq	mm4,mm3
michael@0: 	psrlq	mm4,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm4=( 7 - - -)
michael@0: 
michael@0: 	por	mm1, MMWORD [wk(0)]		; mm1=(-1 0 1 2)
michael@0: 	por	mm2, MMWORD [wk(2)]		; mm2=( 5 6 7 8)
michael@0: 
michael@0: 	movq	MMWORD [wk(0)], mm4
michael@0: 
michael@0: 	pmullw	mm7,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	mm3,[GOTOFF(ebx,PW_THREE)]
michael@0: 	paddw	mm1,[GOTOFF(ebx,PW_EIGHT)]
michael@0: 	paddw	mm5,[GOTOFF(ebx,PW_EIGHT)]
michael@0: 	paddw	mm0,[GOTOFF(ebx,PW_SEVEN)]
michael@0: 	paddw	mm2,[GOTOFF(ebx,PW_SEVEN)]
michael@0: 
michael@0: 	paddw	mm1,mm7
michael@0: 	paddw	mm5,mm3
michael@0: 	psrlw	mm1,4			; mm1=Out0LE=( 0  2  4  6)
michael@0: 	psrlw	mm5,4			; mm5=Out0HE=( 8 10 12 14)
michael@0: 	paddw	mm0,mm7
michael@0: 	paddw	mm2,mm3
michael@0: 	psrlw	mm0,4			; mm0=Out0LO=( 1  3  5  7)
michael@0: 	psrlw	mm2,4			; mm2=Out0HO=( 9 11 13 15)
michael@0: 
michael@0: 	psllw	mm0,BYTE_BIT
michael@0: 	psllw	mm2,BYTE_BIT
michael@0: 	por	mm1,mm0			; mm1=Out0L=( 0  1  2  3  4  5  6  7)
michael@0: 	por	mm5,mm2			; mm5=Out0H=( 8  9 10 11 12 13 14 15)
michael@0: 
michael@0: 	movq	MMWORD [edx+0*SIZEOF_MMWORD], mm1
michael@0: 	movq	MMWORD [edx+1*SIZEOF_MMWORD], mm5
michael@0: 
michael@0: 	; -- process the lower row
michael@0: 
michael@0: 	movq	mm6, MMWORD [edi+0*SIZEOF_MMWORD]	; mm6=Int1L=( 0 1 2 3)
michael@0: 	movq	mm4, MMWORD [edi+1*SIZEOF_MMWORD]	; mm4=Int1H=( 4 5 6 7)
michael@0: 
michael@0: 	movq	mm7,mm6
michael@0: 	movq	mm3,mm4
michael@0: 	psrlq	mm7,2*BYTE_BIT			; mm7=( 1 2 3 -)
michael@0: 	psllq	mm3,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm3=( - - - 4)
michael@0: 	movq	mm0,mm6
michael@0: 	movq	mm2,mm4
michael@0: 	psrlq	mm0,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm0=( 3 - - -)
michael@0: 	psllq	mm2,2*BYTE_BIT			; mm2=( - 4 5 6)
michael@0: 
michael@0: 	por	mm7,mm3				; mm7=( 1 2 3 4)
michael@0: 	por	mm0,mm2				; mm0=( 3 4 5 6)
michael@0: 
michael@0: 	movq	mm1,mm6
michael@0: 	movq	mm5,mm4
michael@0: 	psllq	mm1,2*BYTE_BIT			; mm1=( - 0 1 2)
michael@0: 	psrlq	mm5,2*BYTE_BIT			; mm5=( 5 6 7 -)
michael@0: 	movq	mm3,mm4
michael@0: 	psrlq	mm3,(SIZEOF_MMWORD-2)*BYTE_BIT	; mm3=( 7 - - -)
michael@0: 
michael@0: 	por	mm1, MMWORD [wk(1)]		; mm1=(-1 0 1 2)
michael@0: 	por	mm5, MMWORD [wk(3)]		; mm5=( 5 6 7 8)
michael@0: 
michael@0: 	movq	MMWORD [wk(1)], mm3
michael@0: 
michael@0: 	pmullw	mm6,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	mm4,[GOTOFF(ebx,PW_THREE)]
michael@0: 	paddw	mm1,[GOTOFF(ebx,PW_EIGHT)]
michael@0: 	paddw	mm0,[GOTOFF(ebx,PW_EIGHT)]
michael@0: 	paddw	mm7,[GOTOFF(ebx,PW_SEVEN)]
michael@0: 	paddw	mm5,[GOTOFF(ebx,PW_SEVEN)]
michael@0: 
michael@0: 	paddw	mm1,mm6
michael@0: 	paddw	mm0,mm4
michael@0: 	psrlw	mm1,4			; mm1=Out1LE=( 0  2  4  6)
michael@0: 	psrlw	mm0,4			; mm0=Out1HE=( 8 10 12 14)
michael@0: 	paddw	mm7,mm6
michael@0: 	paddw	mm5,mm4
michael@0: 	psrlw	mm7,4			; mm7=Out1LO=( 1  3  5  7)
michael@0: 	psrlw	mm5,4			; mm5=Out1HO=( 9 11 13 15)
michael@0: 
michael@0: 	psllw	mm7,BYTE_BIT
michael@0: 	psllw	mm5,BYTE_BIT
michael@0: 	por	mm1,mm7			; mm1=Out1L=( 0  1  2  3  4  5  6  7)
michael@0: 	por	mm0,mm5			; mm0=Out1H=( 8  9 10 11 12 13 14 15)
michael@0: 
michael@0: 	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm1
michael@0: 	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm0
michael@0: 
michael@0: 	poppic	ebx
michael@0: 
michael@0: 	sub	eax, byte SIZEOF_MMWORD
michael@0: 	add	ecx, byte 1*SIZEOF_MMWORD	; inptr1(above)
michael@0: 	add	ebx, byte 1*SIZEOF_MMWORD	; inptr0
michael@0: 	add	esi, byte 1*SIZEOF_MMWORD	; inptr1(below)
michael@0: 	add	edx, byte 2*SIZEOF_MMWORD	; outptr0
michael@0: 	add	edi, byte 2*SIZEOF_MMWORD	; outptr1
michael@0: 	cmp	eax, byte SIZEOF_MMWORD
michael@0: 	ja	near .columnloop
michael@0: 	test	eax,eax
michael@0: 	jnz	near .columnloop_last
michael@0: 
michael@0: 	pop	esi
michael@0: 	pop	edi
michael@0: 	pop	ecx
michael@0: 	pop	eax
michael@0: 
michael@0: 	add	esi, byte 1*SIZEOF_JSAMPROW	; input_data
michael@0: 	add	edi, byte 2*SIZEOF_JSAMPROW	; output_data
michael@0: 	sub	ecx, byte 2			; rowctr
michael@0: 	jg	near .rowloop
michael@0: 
michael@0: 	emms		; empty MMX state
michael@0: 
michael@0: .return:
michael@0: 	pop	edi
michael@0: 	pop	esi
michael@0: ;	pop	edx		; need not be preserved
michael@0: ;	pop	ecx		; need not be preserved
michael@0: 	pop	ebx
michael@0: 	mov	esp,ebp		; esp <- aligned ebp
michael@0: 	pop	esp		; esp <- original ebp
michael@0: 	pop	ebp
michael@0: 	ret
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: ;
michael@0: ; Fast processing for the common case of 2:1 horizontal and 1:1 vertical.
michael@0: ; It's still a box filter.
michael@0: ;
michael@0: ; GLOBAL(void)
michael@0: ; jsimd_h2v1_upsample_mmx (int max_v_samp_factor,
michael@0: ;                          JDIMENSION output_width,
michael@0: ;                          JSAMPARRAY input_data,
michael@0: ;                          JSAMPARRAY * output_data_ptr);
michael@0: ;
michael@0: 
michael@0: %define max_v_samp(b)		(b)+8			; int max_v_samp_factor
michael@0: %define output_width(b)	(b)+12		; JDIMENSION output_width
michael@0: %define input_data(b)		(b)+16		; JSAMPARRAY input_data
michael@0: %define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr
michael@0: 
michael@0: 	align	16
michael@0: 	global	EXTN(jsimd_h2v1_upsample_mmx)
michael@0: 
michael@0: EXTN(jsimd_h2v1_upsample_mmx):
michael@0: 	push	ebp
michael@0: 	mov	ebp,esp
michael@0: ;	push	ebx		; unused
michael@0: ;	push	ecx		; need not be preserved
michael@0: ;	push	edx		; need not be preserved
michael@0: 	push	esi
michael@0: 	push	edi
michael@0: 
michael@0: 	mov	edx, JDIMENSION [output_width(ebp)]
michael@0: 	add	edx, byte (2*SIZEOF_MMWORD)-1
michael@0: 	and	edx, byte -(2*SIZEOF_MMWORD)
michael@0: 	jz	short .return
michael@0: 
michael@0: 	mov	ecx, INT [max_v_samp(ebp)]	; rowctr
michael@0: 	test	ecx,ecx
michael@0: 	jz	short .return
michael@0: 
michael@0: 	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
michael@0: 	mov	edi, POINTER [output_data_ptr(ebp)]
michael@0: 	mov	edi, JSAMPARRAY [edi]			; output_data
michael@0: 	alignx	16,7
michael@0: .rowloop:
michael@0: 	push	edi
michael@0: 	push	esi
michael@0: 
michael@0: 	mov	esi, JSAMPROW [esi]		; inptr
michael@0: 	mov	edi, JSAMPROW [edi]		; outptr
michael@0: 	mov	eax,edx				; colctr
michael@0: 	alignx	16,7
michael@0: .columnloop:
michael@0: 
michael@0: 	movq	mm0, MMWORD [esi+0*SIZEOF_MMWORD]
michael@0: 
michael@0: 	movq      mm1,mm0
michael@0: 	punpcklbw mm0,mm0
michael@0: 	punpckhbw mm1,mm1
michael@0: 
michael@0: 	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm0
michael@0: 	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm1
michael@0: 
michael@0: 	sub	eax, byte 2*SIZEOF_MMWORD
michael@0: 	jz	short .nextrow
michael@0: 
michael@0: 	movq	mm2, MMWORD [esi+1*SIZEOF_MMWORD]
michael@0: 
michael@0: 	movq      mm3,mm2
michael@0: 	punpcklbw mm2,mm2
michael@0: 	punpckhbw mm3,mm3
michael@0: 
michael@0: 	movq	MMWORD [edi+2*SIZEOF_MMWORD], mm2
michael@0: 	movq	MMWORD [edi+3*SIZEOF_MMWORD], mm3
michael@0: 
michael@0: 	sub	eax, byte 2*SIZEOF_MMWORD
michael@0: 	jz	short .nextrow
michael@0: 
michael@0: 	add	esi, byte 2*SIZEOF_MMWORD	; inptr
michael@0: 	add	edi, byte 4*SIZEOF_MMWORD	; outptr
michael@0: 	jmp	short .columnloop
michael@0: 	alignx	16,7
michael@0: 
michael@0: .nextrow:
michael@0: 	pop	esi
michael@0: 	pop	edi
michael@0: 
michael@0: 	add	esi, byte SIZEOF_JSAMPROW	; input_data
michael@0: 	add	edi, byte SIZEOF_JSAMPROW	; output_data
michael@0: 	dec	ecx				; rowctr
michael@0: 	jg	short .rowloop
michael@0: 
michael@0: 	emms		; empty MMX state
michael@0: 
michael@0: .return:
michael@0: 	pop	edi
michael@0: 	pop	esi
michael@0: ;	pop	edx		; need not be preserved
michael@0: ;	pop	ecx		; need not be preserved
michael@0: ;	pop	ebx		; unused
michael@0: 	pop	ebp
michael@0: 	ret
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: ;
michael@0: ; Fast processing for the common case of 2:1 horizontal and 2:1 vertical.
michael@0: ; It's still a box filter.
michael@0: ;
michael@0: ; GLOBAL(void)
michael@0: ; jsimd_h2v2_upsample_mmx (int max_v_samp_factor,
michael@0: ;                          JDIMENSION output_width,
michael@0: ;                          JSAMPARRAY input_data,
michael@0: ;                          JSAMPARRAY * output_data_ptr);
michael@0: ;
michael@0: 
michael@0: %define max_v_samp(b)		(b)+8			; int max_v_samp_factor
michael@0: %define output_width(b)	(b)+12		; JDIMENSION output_width
michael@0: %define input_data(b)		(b)+16		; JSAMPARRAY input_data
michael@0: %define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr
michael@0: 
michael@0: 	align	16
michael@0: 	global	EXTN(jsimd_h2v2_upsample_mmx)
michael@0: 
michael@0: EXTN(jsimd_h2v2_upsample_mmx):
michael@0: 	push	ebp
michael@0: 	mov	ebp,esp
michael@0: 	push	ebx
michael@0: ;	push	ecx		; need not be preserved
michael@0: ;	push	edx		; need not be preserved
michael@0: 	push	esi
michael@0: 	push	edi
michael@0: 
michael@0: 	mov	edx, JDIMENSION [output_width(ebp)]
michael@0: 	add	edx, byte (2*SIZEOF_MMWORD)-1
michael@0: 	and	edx, byte -(2*SIZEOF_MMWORD)
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	ecx, INT [max_v_samp(ebp)]	; rowctr
michael@0: 	test	ecx,ecx
michael@0: 	jz	short .return
michael@0: 
michael@0: 	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
michael@0: 	mov	edi, POINTER [output_data_ptr(ebp)]
michael@0: 	mov	edi, JSAMPARRAY [edi]			; output_data
michael@0: 	alignx	16,7
michael@0: .rowloop:
michael@0: 	push	edi
michael@0: 	push	esi
michael@0: 
michael@0: 	mov	esi, JSAMPROW [esi]			; inptr
michael@0: 	mov	ebx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]	; outptr0
michael@0: 	mov	edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]	; outptr1
michael@0: 	mov	eax,edx					; colctr
michael@0: 	alignx	16,7
michael@0: .columnloop:
michael@0: 
michael@0: 	movq	mm0, MMWORD [esi+0*SIZEOF_MMWORD]
michael@0: 
michael@0: 	movq      mm1,mm0
michael@0: 	punpcklbw mm0,mm0
michael@0: 	punpckhbw mm1,mm1
michael@0: 
michael@0: 	movq	MMWORD [ebx+0*SIZEOF_MMWORD], mm0
michael@0: 	movq	MMWORD [ebx+1*SIZEOF_MMWORD], mm1
michael@0: 	movq	MMWORD [edi+0*SIZEOF_MMWORD], mm0
michael@0: 	movq	MMWORD [edi+1*SIZEOF_MMWORD], mm1
michael@0: 
michael@0: 	sub	eax, byte 2*SIZEOF_MMWORD
michael@0: 	jz	short .nextrow
michael@0: 
michael@0: 	movq	mm2, MMWORD [esi+1*SIZEOF_MMWORD]
michael@0: 
michael@0: 	movq      mm3,mm2
michael@0: 	punpcklbw mm2,mm2
michael@0: 	punpckhbw mm3,mm3
michael@0: 
michael@0: 	movq	MMWORD [ebx+2*SIZEOF_MMWORD], mm2
michael@0: 	movq	MMWORD [ebx+3*SIZEOF_MMWORD], mm3
michael@0: 	movq	MMWORD [edi+2*SIZEOF_MMWORD], mm2
michael@0: 	movq	MMWORD [edi+3*SIZEOF_MMWORD], mm3
michael@0: 
michael@0: 	sub	eax, byte 2*SIZEOF_MMWORD
michael@0: 	jz	short .nextrow
michael@0: 
michael@0: 	add	esi, byte 2*SIZEOF_MMWORD	; inptr
michael@0: 	add	ebx, byte 4*SIZEOF_MMWORD	; outptr0
michael@0: 	add	edi, byte 4*SIZEOF_MMWORD	; outptr1
michael@0: 	jmp	short .columnloop
michael@0: 	alignx	16,7
michael@0: 
michael@0: .nextrow:
michael@0: 	pop	esi
michael@0: 	pop	edi
michael@0: 
michael@0: 	add	esi, byte 1*SIZEOF_JSAMPROW	; input_data
michael@0: 	add	edi, byte 2*SIZEOF_JSAMPROW	; output_data
michael@0: 	sub	ecx, byte 2			; rowctr
michael@0: 	jg	short .rowloop
michael@0: 
michael@0: 	emms		; empty MMX state
michael@0: 
michael@0: .return:
michael@0: 	pop	edi
michael@0: 	pop	esi
michael@0: ;	pop	edx		; need not be preserved
michael@0: ;	pop	ecx		; need not be preserved
michael@0: 	pop	ebx
michael@0: 	pop	ebp
michael@0: 	ret
michael@0: 
michael@0: ; For some reason, the OS X linker does not honor the request to align the
michael@0: ; segment unless we do this.
michael@0: 	align	16