michael@0: ;
michael@0: ; jdsamss2.asm - upsampling (SSE2)
michael@0: ;
michael@0: ; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
michael@0: ;
michael@0: ; Based on
michael@0: ; x86 SIMD extension for IJG JPEG library
michael@0: ; Copyright (C) 1999-2006, MIYASAKA Masaru.
michael@0: ; For conditions of distribution and use, see copyright notice in jsimdext.inc
michael@0: ;
michael@0: ; This file should be assembled with NASM (Netwide Assembler),
michael@0: ; can *not* be assembled with Microsoft's MASM or any compatible
michael@0: ; assembler (including Borland's Turbo Assembler).
michael@0: ; NASM is available from http://nasm.sourceforge.net/ or
michael@0: ; http://sourceforge.net/project/showfiles.php?group_id=6208
michael@0: ;
michael@0: ; [TAB8]
michael@0: 
michael@0: %include "jsimdext.inc"
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: 	SECTION	SEG_CONST
michael@0: 
michael@0: 	alignz	16
michael@0: 	global	EXTN(jconst_fancy_upsample_sse2)
michael@0: 
michael@0: EXTN(jconst_fancy_upsample_sse2):
michael@0: 
michael@0: PW_ONE		times 8 dw  1
michael@0: PW_TWO		times 8 dw  2
michael@0: PW_THREE	times 8 dw  3
michael@0: PW_SEVEN	times 8 dw  7
michael@0: PW_EIGHT	times 8 dw  8
michael@0: 
michael@0: 	alignz	16
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: 	SECTION	SEG_TEXT
michael@0: 	BITS	32
michael@0: ;
michael@0: ; Fancy processing for the common case of 2:1 horizontal and 1:1 vertical.
michael@0: ;
michael@0: ; The upsampling algorithm is linear interpolation between pixel centers,
michael@0: ; also known as a "triangle filter".  This is a good compromise between
michael@0: ; speed and visual quality.  The centers of the output pixels are 1/4 and 3/4
michael@0: ; of the way between input pixel centers.
michael@0: ;
michael@0: ; GLOBAL(void)
michael@0: ; jsimd_h2v1_fancy_upsample_sse2 (int max_v_samp_factor,
michael@0: ;                                 JDIMENSION downsampled_width,
michael@0: ;                                 JSAMPARRAY input_data,
michael@0: ;                                 JSAMPARRAY * output_data_ptr);
michael@0: ;
michael@0: 
michael@0: %define max_v_samp(b)		(b)+8			; int max_v_samp_factor
michael@0: %define downsamp_width(b)	(b)+12	; JDIMENSION downsampled_width
michael@0: %define input_data(b)		(b)+16		; JSAMPARRAY input_data
michael@0: %define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr
michael@0: 
michael@0: 	align	16
michael@0: 	global	EXTN(jsimd_h2v1_fancy_upsample_sse2)
michael@0: 
michael@0: EXTN(jsimd_h2v1_fancy_upsample_sse2):
michael@0: 	push	ebp
michael@0: 	mov	ebp,esp
michael@0: 	pushpic	ebx
michael@0: ;	push	ecx		; need not be preserved
michael@0: ;	push	edx		; need not be preserved
michael@0: 	push	esi
michael@0: 	push	edi
michael@0: 
michael@0: 	get_GOT	ebx		; get GOT address
michael@0: 
michael@0: 	mov	eax, JDIMENSION [downsamp_width(ebp)]  ; colctr
michael@0: 	test	eax,eax
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	ecx, INT [max_v_samp(ebp)]	; rowctr
michael@0: 	test	ecx,ecx
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
michael@0: 	mov	edi, POINTER [output_data_ptr(ebp)]
michael@0: 	mov	edi, JSAMPARRAY [edi]			; output_data
michael@0: 	alignx	16,7
michael@0: .rowloop:
michael@0: 	push	eax			; colctr
michael@0: 	push	edi
michael@0: 	push	esi
michael@0: 
michael@0: 	mov	esi, JSAMPROW [esi]	; inptr
michael@0: 	mov	edi, JSAMPROW [edi]	; outptr
michael@0: 
michael@0: 	test	eax, SIZEOF_XMMWORD-1
michael@0: 	jz	short .skip
michael@0: 	mov	dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
michael@0: 	mov	JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl	; insert a dummy sample
michael@0: .skip:
michael@0: 	pxor	xmm0,xmm0		; xmm0=(all 0's)
michael@0: 	pcmpeqb	xmm7,xmm7
michael@0: 	psrldq	xmm7,(SIZEOF_XMMWORD-1)
michael@0: 	pand	xmm7, XMMWORD [esi+0*SIZEOF_XMMWORD]
michael@0: 
michael@0: 	add	eax, byte SIZEOF_XMMWORD-1
michael@0: 	and	eax, byte -SIZEOF_XMMWORD
michael@0: 	cmp	eax, byte SIZEOF_XMMWORD
michael@0: 	ja	short .columnloop
michael@0: 	alignx	16,7
michael@0: 
michael@0: .columnloop_last:
michael@0: 	pcmpeqb	xmm6,xmm6
michael@0: 	pslldq	xmm6,(SIZEOF_XMMWORD-1)
michael@0: 	pand	xmm6, XMMWORD [esi+0*SIZEOF_XMMWORD]
michael@0: 	jmp	short .upsample
michael@0: 	alignx	16,7
michael@0: 
michael@0: .columnloop:
michael@0: 	movdqa	xmm6, XMMWORD [esi+1*SIZEOF_XMMWORD]
michael@0: 	pslldq	xmm6,(SIZEOF_XMMWORD-1)
michael@0: 
michael@0: .upsample:
michael@0: 	movdqa	xmm1, XMMWORD [esi+0*SIZEOF_XMMWORD]
michael@0: 	movdqa	xmm2,xmm1
michael@0: 	movdqa	xmm3,xmm1		; xmm1=( 0  1  2 ... 13 14 15)
michael@0: 	pslldq	xmm2,1			; xmm2=(--  0  1 ... 12 13 14)
michael@0: 	psrldq	xmm3,1			; xmm3=( 1  2  3 ... 14 15 --)
michael@0: 
michael@0: 	por	xmm2,xmm7		; xmm2=(-1  0  1 ... 12 13 14)
michael@0: 	por	xmm3,xmm6		; xmm3=( 1  2  3 ... 14 15 16)
michael@0: 
michael@0: 	movdqa	xmm7,xmm1
michael@0: 	psrldq	xmm7,(SIZEOF_XMMWORD-1)	; xmm7=(15 -- -- ... -- -- --)
michael@0: 
michael@0: 	movdqa    xmm4,xmm1
michael@0: 	punpcklbw xmm1,xmm0		; xmm1=( 0  1  2  3  4  5  6  7)
michael@0: 	punpckhbw xmm4,xmm0		; xmm4=( 8  9 10 11 12 13 14 15)
michael@0: 	movdqa    xmm5,xmm2
michael@0: 	punpcklbw xmm2,xmm0		; xmm2=(-1  0  1  2  3  4  5  6)
michael@0: 	punpckhbw xmm5,xmm0		; xmm5=( 7  8  9 10 11 12 13 14)
michael@0: 	movdqa    xmm6,xmm3
michael@0: 	punpcklbw xmm3,xmm0		; xmm3=( 1  2  3  4  5  6  7  8)
michael@0: 	punpckhbw xmm6,xmm0		; xmm6=( 9 10 11 12 13 14 15 16)
michael@0: 
michael@0: 	pmullw	xmm1,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	xmm4,[GOTOFF(ebx,PW_THREE)]
michael@0: 	paddw	xmm2,[GOTOFF(ebx,PW_ONE)]
michael@0: 	paddw	xmm5,[GOTOFF(ebx,PW_ONE)]
michael@0: 	paddw	xmm3,[GOTOFF(ebx,PW_TWO)]
michael@0: 	paddw	xmm6,[GOTOFF(ebx,PW_TWO)]
michael@0: 
michael@0: 	paddw	xmm2,xmm1
michael@0: 	paddw	xmm5,xmm4
michael@0: 	psrlw	xmm2,2			; xmm2=OutLE=( 0  2  4  6  8 10 12 14)
michael@0: 	psrlw	xmm5,2			; xmm5=OutHE=(16 18 20 22 24 26 28 30)
michael@0: 	paddw	xmm3,xmm1
michael@0: 	paddw	xmm6,xmm4
michael@0: 	psrlw	xmm3,2			; xmm3=OutLO=( 1  3  5  7  9 11 13 15)
michael@0: 	psrlw	xmm6,2			; xmm6=OutHO=(17 19 21 23 25 27 29 31)
michael@0: 
michael@0: 	psllw	xmm3,BYTE_BIT
michael@0: 	psllw	xmm6,BYTE_BIT
michael@0: 	por	xmm2,xmm3		; xmm2=OutL=( 0  1  2 ... 13 14 15)
michael@0: 	por	xmm5,xmm6		; xmm5=OutH=(16 17 18 ... 29 30 31)
michael@0: 
michael@0: 	movdqa	XMMWORD [edi+0*SIZEOF_XMMWORD], xmm2
michael@0: 	movdqa	XMMWORD [edi+1*SIZEOF_XMMWORD], xmm5
michael@0: 
michael@0: 	sub	eax, byte SIZEOF_XMMWORD
michael@0: 	add	esi, byte 1*SIZEOF_XMMWORD	; inptr
michael@0: 	add	edi, byte 2*SIZEOF_XMMWORD	; outptr
michael@0: 	cmp	eax, byte SIZEOF_XMMWORD
michael@0: 	ja	near .columnloop
michael@0: 	test	eax,eax
michael@0: 	jnz	near .columnloop_last
michael@0: 
michael@0: 	pop	esi
michael@0: 	pop	edi
michael@0: 	pop	eax
michael@0: 
michael@0: 	add	esi, byte SIZEOF_JSAMPROW	; input_data
michael@0: 	add	edi, byte SIZEOF_JSAMPROW	; output_data
michael@0: 	dec	ecx				; rowctr
michael@0: 	jg	near .rowloop
michael@0: 
michael@0: .return:
michael@0: 	pop	edi
michael@0: 	pop	esi
michael@0: ;	pop	edx		; need not be preserved
michael@0: ;	pop	ecx		; need not be preserved
michael@0: 	poppic	ebx
michael@0: 	pop	ebp
michael@0: 	ret
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: ;
michael@0: ; Fancy processing for the common case of 2:1 horizontal and 2:1 vertical.
michael@0: ; Again a triangle filter; see comments for h2v1 case, above.
michael@0: ;
michael@0: ; GLOBAL(void)
michael@0: ; jsimd_h2v2_fancy_upsample_sse2 (int max_v_samp_factor,
michael@0: ;                                 JDIMENSION downsampled_width,
michael@0: ;                                 JSAMPARRAY input_data,
michael@0: ;                                 JSAMPARRAY * output_data_ptr);
michael@0: ;
michael@0: 
michael@0: %define max_v_samp(b)		(b)+8			; int max_v_samp_factor
michael@0: %define downsamp_width(b)	(b)+12	; JDIMENSION downsampled_width
michael@0: %define input_data(b)		(b)+16		; JSAMPARRAY input_data
michael@0: %define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr
michael@0: 
michael@0: %define original_ebp	ebp+0
michael@0: %define wk(i)		ebp-(WK_NUM-(i))*SIZEOF_XMMWORD	; xmmword wk[WK_NUM]
michael@0: %define WK_NUM		4
michael@0: %define gotptr		wk(0)-SIZEOF_POINTER	; void * gotptr
michael@0: 
michael@0: 	align	16
michael@0: 	global	EXTN(jsimd_h2v2_fancy_upsample_sse2)
michael@0: 
michael@0: EXTN(jsimd_h2v2_fancy_upsample_sse2):
michael@0: 	push	ebp
michael@0: 	mov	eax,esp				; eax = original ebp
michael@0: 	sub	esp, byte 4
michael@0: 	and	esp, byte (-SIZEOF_XMMWORD)	; align to 128 bits
michael@0: 	mov	[esp],eax
michael@0: 	mov	ebp,esp				; ebp = aligned ebp
michael@0: 	lea	esp, [wk(0)]
michael@0: 	pushpic	eax		; make a room for GOT address
michael@0: 	push	ebx
michael@0: ;	push	ecx		; need not be preserved
michael@0: ;	push	edx		; need not be preserved
michael@0: 	push	esi
michael@0: 	push	edi
michael@0: 
michael@0: 	get_GOT	ebx			; get GOT address
michael@0: 	movpic	POINTER [gotptr], ebx	; save GOT address
michael@0: 
michael@0: 	mov	edx,eax				; edx = original ebp
michael@0: 	mov	eax, JDIMENSION [downsamp_width(edx)]  ; colctr
michael@0: 	test	eax,eax
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	ecx, INT [max_v_samp(edx)]	; rowctr
michael@0: 	test	ecx,ecx
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	esi, JSAMPARRAY [input_data(edx)]	; input_data
michael@0: 	mov	edi, POINTER [output_data_ptr(edx)]
michael@0: 	mov	edi, JSAMPARRAY [edi]			; output_data
michael@0: 	alignx	16,7
michael@0: .rowloop:
michael@0: 	push	eax					; colctr
michael@0: 	push	ecx
michael@0: 	push	edi
michael@0: 	push	esi
michael@0: 
michael@0: 	mov	ecx, JSAMPROW [esi-1*SIZEOF_JSAMPROW]	; inptr1(above)
michael@0: 	mov	ebx, JSAMPROW [esi+0*SIZEOF_JSAMPROW]	; inptr0
michael@0: 	mov	esi, JSAMPROW [esi+1*SIZEOF_JSAMPROW]	; inptr1(below)
michael@0: 	mov	edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]	; outptr0
michael@0: 	mov	edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]	; outptr1
michael@0: 
michael@0: 	test	eax, SIZEOF_XMMWORD-1
michael@0: 	jz	short .skip
michael@0: 	push	edx
michael@0: 	mov	dl, JSAMPLE [ecx+(eax-1)*SIZEOF_JSAMPLE]
michael@0: 	mov	JSAMPLE [ecx+eax*SIZEOF_JSAMPLE], dl
michael@0: 	mov	dl, JSAMPLE [ebx+(eax-1)*SIZEOF_JSAMPLE]
michael@0: 	mov	JSAMPLE [ebx+eax*SIZEOF_JSAMPLE], dl
michael@0: 	mov	dl, JSAMPLE [esi+(eax-1)*SIZEOF_JSAMPLE]
michael@0: 	mov	JSAMPLE [esi+eax*SIZEOF_JSAMPLE], dl	; insert a dummy sample
michael@0: 	pop	edx
michael@0: .skip:
michael@0: 	; -- process the first column block
michael@0: 
michael@0: 	movdqa	xmm0, XMMWORD [ebx+0*SIZEOF_XMMWORD]	; xmm0=row[ 0][0]
michael@0: 	movdqa	xmm1, XMMWORD [ecx+0*SIZEOF_XMMWORD]	; xmm1=row[-1][0]
michael@0: 	movdqa	xmm2, XMMWORD [esi+0*SIZEOF_XMMWORD]	; xmm2=row[+1][0]
michael@0: 
michael@0: 	pushpic	ebx
michael@0: 	movpic	ebx, POINTER [gotptr]	; load GOT address
michael@0: 
michael@0: 	pxor      xmm3,xmm3		; xmm3=(all 0's)
michael@0: 	movdqa    xmm4,xmm0
michael@0: 	punpcklbw xmm0,xmm3		; xmm0=row[ 0]( 0  1  2  3  4  5  6  7)
michael@0: 	punpckhbw xmm4,xmm3		; xmm4=row[ 0]( 8  9 10 11 12 13 14 15)
michael@0: 	movdqa    xmm5,xmm1
michael@0: 	punpcklbw xmm1,xmm3		; xmm1=row[-1]( 0  1  2  3  4  5  6  7)
michael@0: 	punpckhbw xmm5,xmm3		; xmm5=row[-1]( 8  9 10 11 12 13 14 15)
michael@0: 	movdqa    xmm6,xmm2
michael@0: 	punpcklbw xmm2,xmm3		; xmm2=row[+1]( 0  1  2  3  4  5  6  7)
michael@0: 	punpckhbw xmm6,xmm3		; xmm6=row[+1]( 8  9 10 11 12 13 14 15)
michael@0: 
michael@0: 	pmullw	xmm0,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	xmm4,[GOTOFF(ebx,PW_THREE)]
michael@0: 
michael@0: 	pcmpeqb	xmm7,xmm7
michael@0: 	psrldq	xmm7,(SIZEOF_XMMWORD-2)
michael@0: 
michael@0: 	paddw	xmm1,xmm0		; xmm1=Int0L=( 0  1  2  3  4  5  6  7)
michael@0: 	paddw	xmm5,xmm4		; xmm5=Int0H=( 8  9 10 11 12 13 14 15)
michael@0: 	paddw	xmm2,xmm0		; xmm2=Int1L=( 0  1  2  3  4  5  6  7)
michael@0: 	paddw	xmm6,xmm4		; xmm6=Int1H=( 8  9 10 11 12 13 14 15)
michael@0: 
michael@0: 	movdqa	XMMWORD [edx+0*SIZEOF_XMMWORD], xmm1	; temporarily save
michael@0: 	movdqa	XMMWORD [edx+1*SIZEOF_XMMWORD], xmm5	; the intermediate data
michael@0: 	movdqa	XMMWORD [edi+0*SIZEOF_XMMWORD], xmm2
michael@0: 	movdqa	XMMWORD [edi+1*SIZEOF_XMMWORD], xmm6
michael@0: 
michael@0: 	pand	xmm1,xmm7		; xmm1=( 0 -- -- -- -- -- -- --)
michael@0: 	pand	xmm2,xmm7		; xmm2=( 0 -- -- -- -- -- -- --)
michael@0: 
michael@0: 	movdqa	XMMWORD [wk(0)], xmm1
michael@0: 	movdqa	XMMWORD [wk(1)], xmm2
michael@0: 
michael@0: 	poppic	ebx
michael@0: 
michael@0: 	add	eax, byte SIZEOF_XMMWORD-1
michael@0: 	and	eax, byte -SIZEOF_XMMWORD
michael@0: 	cmp	eax, byte SIZEOF_XMMWORD
michael@0: 	ja	short .columnloop
michael@0: 	alignx	16,7
michael@0: 
michael@0: .columnloop_last:
michael@0: 	; -- process the last column block
michael@0: 
michael@0: 	pushpic	ebx
michael@0: 	movpic	ebx, POINTER [gotptr]	; load GOT address
michael@0: 
michael@0: 	pcmpeqb	xmm1,xmm1
michael@0: 	pslldq	xmm1,(SIZEOF_XMMWORD-2)
michael@0: 	movdqa	xmm2,xmm1
michael@0: 
michael@0: 	pand	xmm1, XMMWORD [edx+1*SIZEOF_XMMWORD]
michael@0: 	pand	xmm2, XMMWORD [edi+1*SIZEOF_XMMWORD]
michael@0: 
michael@0: 	movdqa	XMMWORD [wk(2)], xmm1	; xmm1=(-- -- -- -- -- -- -- 15)
michael@0: 	movdqa	XMMWORD [wk(3)], xmm2	; xmm2=(-- -- -- -- -- -- -- 15)
michael@0: 
michael@0: 	jmp	near .upsample
michael@0: 	alignx	16,7
michael@0: 
michael@0: .columnloop:
michael@0: 	; -- process the next column block
michael@0: 
michael@0: 	movdqa	xmm0, XMMWORD [ebx+1*SIZEOF_XMMWORD]	; xmm0=row[ 0][1]
michael@0: 	movdqa	xmm1, XMMWORD [ecx+1*SIZEOF_XMMWORD]	; xmm1=row[-1][1]
michael@0: 	movdqa	xmm2, XMMWORD [esi+1*SIZEOF_XMMWORD]	; xmm2=row[+1][1]
michael@0: 
michael@0: 	pushpic	ebx
michael@0: 	movpic	ebx, POINTER [gotptr]	; load GOT address
michael@0: 
michael@0: 	pxor      xmm3,xmm3		; xmm3=(all 0's)
michael@0: 	movdqa    xmm4,xmm0
michael@0: 	punpcklbw xmm0,xmm3		; xmm0=row[ 0]( 0  1  2  3  4  5  6  7)
michael@0: 	punpckhbw xmm4,xmm3		; xmm4=row[ 0]( 8  9 10 11 12 13 14 15)
michael@0: 	movdqa    xmm5,xmm1
michael@0: 	punpcklbw xmm1,xmm3		; xmm1=row[-1]( 0  1  2  3  4  5  6  7)
michael@0: 	punpckhbw xmm5,xmm3		; xmm5=row[-1]( 8  9 10 11 12 13 14 15)
michael@0: 	movdqa    xmm6,xmm2
michael@0: 	punpcklbw xmm2,xmm3		; xmm2=row[+1]( 0  1  2  3  4  5  6  7)
michael@0: 	punpckhbw xmm6,xmm3		; xmm6=row[+1]( 8  9 10 11 12 13 14 15)
michael@0: 
michael@0: 	pmullw	xmm0,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	xmm4,[GOTOFF(ebx,PW_THREE)]
michael@0: 
michael@0: 	paddw	xmm1,xmm0		; xmm1=Int0L=( 0  1  2  3  4  5  6  7)
michael@0: 	paddw	xmm5,xmm4		; xmm5=Int0H=( 8  9 10 11 12 13 14 15)
michael@0: 	paddw	xmm2,xmm0		; xmm2=Int1L=( 0  1  2  3  4  5  6  7)
michael@0: 	paddw	xmm6,xmm4		; xmm6=Int1H=( 8  9 10 11 12 13 14 15)
michael@0: 
michael@0: 	movdqa	XMMWORD [edx+2*SIZEOF_XMMWORD], xmm1	; temporarily save
michael@0: 	movdqa	XMMWORD [edx+3*SIZEOF_XMMWORD], xmm5	; the intermediate data
michael@0: 	movdqa	XMMWORD [edi+2*SIZEOF_XMMWORD], xmm2
michael@0: 	movdqa	XMMWORD [edi+3*SIZEOF_XMMWORD], xmm6
michael@0: 
michael@0: 	pslldq	xmm1,(SIZEOF_XMMWORD-2)	; xmm1=(-- -- -- -- -- -- --  0)
michael@0: 	pslldq	xmm2,(SIZEOF_XMMWORD-2)	; xmm2=(-- -- -- -- -- -- --  0)
michael@0: 
michael@0: 	movdqa	XMMWORD [wk(2)], xmm1
michael@0: 	movdqa	XMMWORD [wk(3)], xmm2
michael@0: 
michael@0: .upsample:
michael@0: 	; -- process the upper row
michael@0: 
michael@0: 	movdqa	xmm7, XMMWORD [edx+0*SIZEOF_XMMWORD]
michael@0: 	movdqa	xmm3, XMMWORD [edx+1*SIZEOF_XMMWORD]
michael@0: 
michael@0: 	movdqa	xmm0,xmm7		; xmm7=Int0L=( 0  1  2  3  4  5  6  7)
michael@0: 	movdqa	xmm4,xmm3		; xmm3=Int0H=( 8  9 10 11 12 13 14 15)
michael@0: 	psrldq	xmm0,2			; xmm0=( 1  2  3  4  5  6  7 --)
michael@0: 	pslldq	xmm4,(SIZEOF_XMMWORD-2)	; xmm4=(-- -- -- -- -- -- --  8)
michael@0: 	movdqa	xmm5,xmm7
michael@0: 	movdqa	xmm6,xmm3
michael@0: 	psrldq	xmm5,(SIZEOF_XMMWORD-2)	; xmm5=( 7 -- -- -- -- -- -- --)
michael@0: 	pslldq	xmm6,2			; xmm6=(--  8  9 10 11 12 13 14)
michael@0: 
michael@0: 	por	xmm0,xmm4		; xmm0=( 1  2  3  4  5  6  7  8)
michael@0: 	por	xmm5,xmm6		; xmm5=( 7  8  9 10 11 12 13 14)
michael@0: 
michael@0: 	movdqa	xmm1,xmm7
michael@0: 	movdqa	xmm2,xmm3
michael@0: 	pslldq	xmm1,2			; xmm1=(--  0  1  2  3  4  5  6)
michael@0: 	psrldq	xmm2,2			; xmm2=( 9 10 11 12 13 14 15 --)
michael@0: 	movdqa	xmm4,xmm3
michael@0: 	psrldq	xmm4,(SIZEOF_XMMWORD-2)	; xmm4=(15 -- -- -- -- -- -- --)
michael@0: 
michael@0: 	por	xmm1, XMMWORD [wk(0)]	; xmm1=(-1  0  1  2  3  4  5  6)
michael@0: 	por	xmm2, XMMWORD [wk(2)]	; xmm2=( 9 10 11 12 13 14 15 16)
michael@0: 
michael@0: 	movdqa	XMMWORD [wk(0)], xmm4
michael@0: 
michael@0: 	pmullw	xmm7,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	xmm3,[GOTOFF(ebx,PW_THREE)]
michael@0: 	paddw	xmm1,[GOTOFF(ebx,PW_EIGHT)]
michael@0: 	paddw	xmm5,[GOTOFF(ebx,PW_EIGHT)]
michael@0: 	paddw	xmm0,[GOTOFF(ebx,PW_SEVEN)]
michael@0: 	paddw	xmm2,[GOTOFF(ebx,PW_SEVEN)]
michael@0: 
michael@0: 	paddw	xmm1,xmm7
michael@0: 	paddw	xmm5,xmm3
michael@0: 	psrlw	xmm1,4			; xmm1=Out0LE=( 0  2  4  6  8 10 12 14)
michael@0: 	psrlw	xmm5,4			; xmm5=Out0HE=(16 18 20 22 24 26 28 30)
michael@0: 	paddw	xmm0,xmm7
michael@0: 	paddw	xmm2,xmm3
michael@0: 	psrlw	xmm0,4			; xmm0=Out0LO=( 1  3  5  7  9 11 13 15)
michael@0: 	psrlw	xmm2,4			; xmm2=Out0HO=(17 19 21 23 25 27 29 31)
michael@0: 
michael@0: 	psllw	xmm0,BYTE_BIT
michael@0: 	psllw	xmm2,BYTE_BIT
michael@0: 	por	xmm1,xmm0		; xmm1=Out0L=( 0  1  2 ... 13 14 15)
michael@0: 	por	xmm5,xmm2		; xmm5=Out0H=(16 17 18 ... 29 30 31)
michael@0: 
michael@0: 	movdqa	XMMWORD [edx+0*SIZEOF_XMMWORD], xmm1
michael@0: 	movdqa	XMMWORD [edx+1*SIZEOF_XMMWORD], xmm5
michael@0: 
michael@0: 	; -- process the lower row
michael@0: 
michael@0: 	movdqa	xmm6, XMMWORD [edi+0*SIZEOF_XMMWORD]
michael@0: 	movdqa	xmm4, XMMWORD [edi+1*SIZEOF_XMMWORD]
michael@0: 
michael@0: 	movdqa	xmm7,xmm6		; xmm6=Int1L=( 0  1  2  3  4  5  6  7)
michael@0: 	movdqa	xmm3,xmm4		; xmm4=Int1H=( 8  9 10 11 12 13 14 15)
michael@0: 	psrldq	xmm7,2			; xmm7=( 1  2  3  4  5  6  7 --)
michael@0: 	pslldq	xmm3,(SIZEOF_XMMWORD-2)	; xmm3=(-- -- -- -- -- -- --  8)
michael@0: 	movdqa	xmm0,xmm6
michael@0: 	movdqa	xmm2,xmm4
michael@0: 	psrldq	xmm0,(SIZEOF_XMMWORD-2)	; xmm0=( 7 -- -- -- -- -- -- --)
michael@0: 	pslldq	xmm2,2			; xmm2=(--  8  9 10 11 12 13 14)
michael@0: 
michael@0: 	por	xmm7,xmm3		; xmm7=( 1  2  3  4  5  6  7  8)
michael@0: 	por	xmm0,xmm2		; xmm0=( 7  8  9 10 11 12 13 14)
michael@0: 
michael@0: 	movdqa	xmm1,xmm6
michael@0: 	movdqa	xmm5,xmm4
michael@0: 	pslldq	xmm1,2			; xmm1=(--  0  1  2  3  4  5  6)
michael@0: 	psrldq	xmm5,2			; xmm5=( 9 10 11 12 13 14 15 --)
michael@0: 	movdqa	xmm3,xmm4
michael@0: 	psrldq	xmm3,(SIZEOF_XMMWORD-2)	; xmm3=(15 -- -- -- -- -- -- --)
michael@0: 
michael@0: 	por	xmm1, XMMWORD [wk(1)]	; xmm1=(-1  0  1  2  3  4  5  6)
michael@0: 	por	xmm5, XMMWORD [wk(3)]	; xmm5=( 9 10 11 12 13 14 15 16)
michael@0: 
michael@0: 	movdqa	XMMWORD [wk(1)], xmm3
michael@0: 
michael@0: 	pmullw	xmm6,[GOTOFF(ebx,PW_THREE)]
michael@0: 	pmullw	xmm4,[GOTOFF(ebx,PW_THREE)]
michael@0: 	paddw	xmm1,[GOTOFF(ebx,PW_EIGHT)]
michael@0: 	paddw	xmm0,[GOTOFF(ebx,PW_EIGHT)]
michael@0: 	paddw	xmm7,[GOTOFF(ebx,PW_SEVEN)]
michael@0: 	paddw	xmm5,[GOTOFF(ebx,PW_SEVEN)]
michael@0: 
michael@0: 	paddw	xmm1,xmm6
michael@0: 	paddw	xmm0,xmm4
michael@0: 	psrlw	xmm1,4			; xmm1=Out1LE=( 0  2  4  6  8 10 12 14)
michael@0: 	psrlw	xmm0,4			; xmm0=Out1HE=(16 18 20 22 24 26 28 30)
michael@0: 	paddw	xmm7,xmm6
michael@0: 	paddw	xmm5,xmm4
michael@0: 	psrlw	xmm7,4			; xmm7=Out1LO=( 1  3  5  7  9 11 13 15)
michael@0: 	psrlw	xmm5,4			; xmm5=Out1HO=(17 19 21 23 25 27 29 31)
michael@0: 
michael@0: 	psllw	xmm7,BYTE_BIT
michael@0: 	psllw	xmm5,BYTE_BIT
michael@0: 	por	xmm1,xmm7		; xmm1=Out1L=( 0  1  2 ... 13 14 15)
michael@0: 	por	xmm0,xmm5		; xmm0=Out1H=(16 17 18 ... 29 30 31)
michael@0: 
michael@0: 	movdqa	XMMWORD [edi+0*SIZEOF_XMMWORD], xmm1
michael@0: 	movdqa	XMMWORD [edi+1*SIZEOF_XMMWORD], xmm0
michael@0: 
michael@0: 	poppic	ebx
michael@0: 
michael@0: 	sub	eax, byte SIZEOF_XMMWORD
michael@0: 	add	ecx, byte 1*SIZEOF_XMMWORD	; inptr1(above)
michael@0: 	add	ebx, byte 1*SIZEOF_XMMWORD	; inptr0
michael@0: 	add	esi, byte 1*SIZEOF_XMMWORD	; inptr1(below)
michael@0: 	add	edx, byte 2*SIZEOF_XMMWORD	; outptr0
michael@0: 	add	edi, byte 2*SIZEOF_XMMWORD	; outptr1
michael@0: 	cmp	eax, byte SIZEOF_XMMWORD
michael@0: 	ja	near .columnloop
michael@0: 	test	eax,eax
michael@0: 	jnz	near .columnloop_last
michael@0: 
michael@0: 	pop	esi
michael@0: 	pop	edi
michael@0: 	pop	ecx
michael@0: 	pop	eax
michael@0: 
michael@0: 	add	esi, byte 1*SIZEOF_JSAMPROW	; input_data
michael@0: 	add	edi, byte 2*SIZEOF_JSAMPROW	; output_data
michael@0: 	sub	ecx, byte 2			; rowctr
michael@0: 	jg	near .rowloop
michael@0: 
michael@0: .return:
michael@0: 	pop	edi
michael@0: 	pop	esi
michael@0: ;	pop	edx		; need not be preserved
michael@0: ;	pop	ecx		; need not be preserved
michael@0: 	pop	ebx
michael@0: 	mov	esp,ebp		; esp <- aligned ebp
michael@0: 	pop	esp		; esp <- original ebp
michael@0: 	pop	ebp
michael@0: 	ret
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: ;
michael@0: ; Fast processing for the common case of 2:1 horizontal and 1:1 vertical.
michael@0: ; It's still a box filter.
michael@0: ;
michael@0: ; GLOBAL(void)
michael@0: ; jsimd_h2v1_upsample_sse2 (int max_v_samp_factor,
michael@0: ;                           JDIMENSION output_width,
michael@0: ;                           JSAMPARRAY input_data,
michael@0: ;                           JSAMPARRAY * output_data_ptr);
michael@0: ;
michael@0: 
michael@0: %define max_v_samp(b)		(b)+8			; int max_v_samp_factor
michael@0: %define output_width(b)	(b)+12		; JDIMENSION output_width
michael@0: %define input_data(b)		(b)+16		; JSAMPARRAY input_data
michael@0: %define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr
michael@0: 
michael@0: 	align	16
michael@0: 	global	EXTN(jsimd_h2v1_upsample_sse2)
michael@0: 
michael@0: EXTN(jsimd_h2v1_upsample_sse2):
michael@0: 	push	ebp
michael@0: 	mov	ebp,esp
michael@0: ;	push	ebx		; unused
michael@0: ;	push	ecx		; need not be preserved
michael@0: ;	push	edx		; need not be preserved
michael@0: 	push	esi
michael@0: 	push	edi
michael@0: 
michael@0: 	mov	edx, JDIMENSION [output_width(ebp)]
michael@0: 	add	edx, byte (2*SIZEOF_XMMWORD)-1
michael@0: 	and	edx, byte -(2*SIZEOF_XMMWORD)
michael@0: 	jz	short .return
michael@0: 
michael@0: 	mov	ecx, INT [max_v_samp(ebp)]	; rowctr
michael@0: 	test	ecx,ecx
michael@0: 	jz	short .return
michael@0: 
michael@0: 	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
michael@0: 	mov	edi, POINTER [output_data_ptr(ebp)]
michael@0: 	mov	edi, JSAMPARRAY [edi]			; output_data
michael@0: 	alignx	16,7
michael@0: .rowloop:
michael@0: 	push	edi
michael@0: 	push	esi
michael@0: 
michael@0: 	mov	esi, JSAMPROW [esi]		; inptr
michael@0: 	mov	edi, JSAMPROW [edi]		; outptr
michael@0: 	mov	eax,edx				; colctr
michael@0: 	alignx	16,7
michael@0: .columnloop:
michael@0: 
michael@0: 	movdqa	xmm0, XMMWORD [esi+0*SIZEOF_XMMWORD]
michael@0: 
michael@0: 	movdqa    xmm1,xmm0
michael@0: 	punpcklbw xmm0,xmm0
michael@0: 	punpckhbw xmm1,xmm1
michael@0: 
michael@0: 	movdqa	XMMWORD [edi+0*SIZEOF_XMMWORD], xmm0
michael@0: 	movdqa	XMMWORD [edi+1*SIZEOF_XMMWORD], xmm1
michael@0: 
michael@0: 	sub	eax, byte 2*SIZEOF_XMMWORD
michael@0: 	jz	short .nextrow
michael@0: 
michael@0: 	movdqa	xmm2, XMMWORD [esi+1*SIZEOF_XMMWORD]
michael@0: 
michael@0: 	movdqa    xmm3,xmm2
michael@0: 	punpcklbw xmm2,xmm2
michael@0: 	punpckhbw xmm3,xmm3
michael@0: 
michael@0: 	movdqa	XMMWORD [edi+2*SIZEOF_XMMWORD], xmm2
michael@0: 	movdqa	XMMWORD [edi+3*SIZEOF_XMMWORD], xmm3
michael@0: 
michael@0: 	sub	eax, byte 2*SIZEOF_XMMWORD
michael@0: 	jz	short .nextrow
michael@0: 
michael@0: 	add	esi, byte 2*SIZEOF_XMMWORD	; inptr
michael@0: 	add	edi, byte 4*SIZEOF_XMMWORD	; outptr
michael@0: 	jmp	short .columnloop
michael@0: 	alignx	16,7
michael@0: 
michael@0: .nextrow:
michael@0: 	pop	esi
michael@0: 	pop	edi
michael@0: 
michael@0: 	add	esi, byte SIZEOF_JSAMPROW	; input_data
michael@0: 	add	edi, byte SIZEOF_JSAMPROW	; output_data
michael@0: 	dec	ecx				; rowctr
michael@0: 	jg	short .rowloop
michael@0: 
michael@0: .return:
michael@0: 	pop	edi
michael@0: 	pop	esi
michael@0: ;	pop	edx		; need not be preserved
michael@0: ;	pop	ecx		; need not be preserved
michael@0: ;	pop	ebx		; unused
michael@0: 	pop	ebp
michael@0: 	ret
michael@0: 
michael@0: ; --------------------------------------------------------------------------
michael@0: ;
michael@0: ; Fast processing for the common case of 2:1 horizontal and 2:1 vertical.
michael@0: ; It's still a box filter.
michael@0: ;
michael@0: ; GLOBAL(void)
michael@0: ; jsimd_h2v2_upsample_sse2 (nt max_v_samp_factor,
michael@0: ;                           JDIMENSION output_width,
michael@0: ;                           JSAMPARRAY input_data,
michael@0: ;                           JSAMPARRAY * output_data_ptr);
michael@0: ;
michael@0: 
michael@0: %define max_v_samp(b)		(b)+8			; int max_v_samp_factor
michael@0: %define output_width(b)	(b)+12		; JDIMENSION output_width
michael@0: %define input_data(b)		(b)+16		; JSAMPARRAY input_data
michael@0: %define output_data_ptr(b)	(b)+20		; JSAMPARRAY * output_data_ptr
michael@0: 
michael@0: 	align	16
michael@0: 	global	EXTN(jsimd_h2v2_upsample_sse2)
michael@0: 
michael@0: EXTN(jsimd_h2v2_upsample_sse2):
michael@0: 	push	ebp
michael@0: 	mov	ebp,esp
michael@0: 	push	ebx
michael@0: ;	push	ecx		; need not be preserved
michael@0: ;	push	edx		; need not be preserved
michael@0: 	push	esi
michael@0: 	push	edi
michael@0: 
michael@0: 	mov	edx, JDIMENSION [output_width(ebp)]
michael@0: 	add	edx, byte (2*SIZEOF_XMMWORD)-1
michael@0: 	and	edx, byte -(2*SIZEOF_XMMWORD)
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	ecx, INT [max_v_samp(ebp)]	; rowctr
michael@0: 	test	ecx,ecx
michael@0: 	jz	near .return
michael@0: 
michael@0: 	mov	esi, JSAMPARRAY [input_data(ebp)]	; input_data
michael@0: 	mov	edi, POINTER [output_data_ptr(ebp)]
michael@0: 	mov	edi, JSAMPARRAY [edi]			; output_data
michael@0: 	alignx	16,7
michael@0: .rowloop:
michael@0: 	push	edi
michael@0: 	push	esi
michael@0: 
michael@0: 	mov	esi, JSAMPROW [esi]			; inptr
michael@0: 	mov	ebx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]	; outptr0
michael@0: 	mov	edi, JSAMPROW [edi+1*SIZEOF_JSAMPROW]	; outptr1
michael@0: 	mov	eax,edx					; colctr
michael@0: 	alignx	16,7
michael@0: .columnloop:
michael@0: 
michael@0: 	movdqa	xmm0, XMMWORD [esi+0*SIZEOF_XMMWORD]
michael@0: 
michael@0: 	movdqa    xmm1,xmm0
michael@0: 	punpcklbw xmm0,xmm0
michael@0: 	punpckhbw xmm1,xmm1
michael@0: 
michael@0: 	movdqa	XMMWORD [ebx+0*SIZEOF_XMMWORD], xmm0
michael@0: 	movdqa	XMMWORD [ebx+1*SIZEOF_XMMWORD], xmm1
michael@0: 	movdqa	XMMWORD [edi+0*SIZEOF_XMMWORD], xmm0
michael@0: 	movdqa	XMMWORD [edi+1*SIZEOF_XMMWORD], xmm1
michael@0: 
michael@0: 	sub	eax, byte 2*SIZEOF_XMMWORD
michael@0: 	jz	short .nextrow
michael@0: 
michael@0: 	movdqa	xmm2, XMMWORD [esi+1*SIZEOF_XMMWORD]
michael@0: 
michael@0: 	movdqa    xmm3,xmm2
michael@0: 	punpcklbw xmm2,xmm2
michael@0: 	punpckhbw xmm3,xmm3
michael@0: 
michael@0: 	movdqa	XMMWORD [ebx+2*SIZEOF_XMMWORD], xmm2
michael@0: 	movdqa	XMMWORD [ebx+3*SIZEOF_XMMWORD], xmm3
michael@0: 	movdqa	XMMWORD [edi+2*SIZEOF_XMMWORD], xmm2
michael@0: 	movdqa	XMMWORD [edi+3*SIZEOF_XMMWORD], xmm3
michael@0: 
michael@0: 	sub	eax, byte 2*SIZEOF_XMMWORD
michael@0: 	jz	short .nextrow
michael@0: 
michael@0: 	add	esi, byte 2*SIZEOF_XMMWORD	; inptr
michael@0: 	add	ebx, byte 4*SIZEOF_XMMWORD	; outptr0
michael@0: 	add	edi, byte 4*SIZEOF_XMMWORD	; outptr1
michael@0: 	jmp	short .columnloop
michael@0: 	alignx	16,7
michael@0: 
michael@0: .nextrow:
michael@0: 	pop	esi
michael@0: 	pop	edi
michael@0: 
michael@0: 	add	esi, byte 1*SIZEOF_JSAMPROW	; input_data
michael@0: 	add	edi, byte 2*SIZEOF_JSAMPROW	; output_data
michael@0: 	sub	ecx, byte 2			; rowctr
michael@0: 	jg	short .rowloop
michael@0: 
michael@0: .return:
michael@0: 	pop	edi
michael@0: 	pop	esi
michael@0: ;	pop	edx		; need not be preserved
michael@0: ;	pop	ecx		; need not be preserved
michael@0: 	pop	ebx
michael@0: 	pop	ebp
michael@0: 	ret
michael@0: 
michael@0: ; For some reason, the OS X linker does not honor the request to align the
michael@0: ; segment unless we do this.
michael@0: 	align	16