• file
• revisions
• annotate
• diff
• comparison
• raw

media/libjpeg/simd/ji3dnflt.asm@6474c204b198 (annotated)

media/libjpeg/simd/ji3dnflt.asm

Wed, 31 Dec 2014 06:09:35 +0100

author

Michael Schloh von Bennewitz <michael@schloh.com>

date

Wed, 31 Dec 2014 06:09:35 +0100

changeset 0

6474c204b198

permissions

-rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

michael@0	1	;
michael@0	2	; ji3dnflt.asm - floating-point IDCT (3DNow! & MMX)
michael@0	3	;
michael@0	4	; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
michael@0	5	;
michael@0	6	; Based on
michael@0	7	; x86 SIMD extension for IJG JPEG library
michael@0	8	; Copyright (C) 1999-2006, MIYASAKA Masaru.
michael@0	9	; For conditions of distribution and use, see copyright notice in jsimdext.inc
michael@0	10	;
michael@0	11	; This file should be assembled with NASM (Netwide Assembler),
michael@0	12	; can *not* be assembled with Microsoft's MASM or any compatible
michael@0	13	; assembler (including Borland's Turbo Assembler).
michael@0	14	; NASM is available from http://nasm.sourceforge.net/ or
michael@0	15	; http://sourceforge.net/project/showfiles.php?group_id=6208
michael@0	16	;
michael@0	17	; This file contains a floating-point implementation of the inverse DCT
michael@0	18	; (Discrete Cosine Transform). The following code is based directly on
michael@0	19	; the IJG's original jidctflt.c; see the jidctflt.c for more details.
michael@0	20	;
michael@0	21	; [TAB8]
michael@0	22
michael@0	23	%include "jsimdext.inc"
michael@0	24	%include "jdct.inc"
michael@0	25
michael@0	26	; --------------------------------------------------------------------------
michael@0	27	SECTION SEG_CONST
michael@0	28
michael@0	29	alignz 16
michael@0	30	global EXTN(jconst_idct_float_3dnow)
michael@0	31
michael@0	32	EXTN(jconst_idct_float_3dnow):
michael@0	33
michael@0	34	PD_1_414 times 2 dd 1.414213562373095048801689
michael@0	35	PD_1_847 times 2 dd 1.847759065022573512256366
michael@0	36	PD_1_082 times 2 dd 1.082392200292393968799446
michael@0	37	PD_2_613 times 2 dd 2.613125929752753055713286
michael@0	38	PD_RNDINT_MAGIC times 2 dd 100663296.0 ; (float)(0x00C00000 << 3)
michael@0	39	PB_CENTERJSAMP times 8 db CENTERJSAMPLE
michael@0	40
michael@0	41	alignz 16
michael@0	42
michael@0	43	; --------------------------------------------------------------------------
michael@0	44	SECTION SEG_TEXT
michael@0	45	BITS 32
michael@0	46	;
michael@0	47	; Perform dequantization and inverse DCT on one block of coefficients.
michael@0	48	;
michael@0	49	; GLOBAL(void)
michael@0	50	; jsimd_idct_float_3dnow (void * dct_table, JCOEFPTR coef_block,
michael@0	51	; JSAMPARRAY output_buf, JDIMENSION output_col)
michael@0	52	;
michael@0	53
michael@0	54	%define dct_table(b) (b)+8 ; void * dct_table
michael@0	55	%define coef_block(b) (b)+12 ; JCOEFPTR coef_block
michael@0	56	%define output_buf(b) (b)+16 ; JSAMPARRAY output_buf
michael@0	57	%define output_col(b) (b)+20 ; JDIMENSION output_col
michael@0	58
michael@0	59	%define original_ebp ebp+0
michael@0	60	%define wk(i) ebp-(WK_NUM-(i))*SIZEOF_MMWORD ; mmword wk[WK_NUM]
michael@0	61	%define WK_NUM 2
michael@0	62	%define workspace wk(0)-DCTSIZE2*SIZEOF_FAST_FLOAT
michael@0	63	; FAST_FLOAT workspace[DCTSIZE2]
michael@0	64
michael@0	65	align 16
michael@0	66	global EXTN(jsimd_idct_float_3dnow)
michael@0	67
michael@0	68	EXTN(jsimd_idct_float_3dnow):
michael@0	69	push ebp
michael@0	70	mov eax,esp ; eax = original ebp
michael@0	71	sub esp, byte 4
michael@0	72	and esp, byte (-SIZEOF_MMWORD) ; align to 64 bits
michael@0	73	mov [esp],eax
michael@0	74	mov ebp,esp ; ebp = aligned ebp
michael@0	75	lea esp, [workspace]
michael@0	76	push ebx
michael@0	77	; push ecx ; need not be preserved
michael@0	78	; push edx ; need not be preserved
michael@0	79	push esi
michael@0	80	push edi
michael@0	81
michael@0	82	get_GOT ebx ; get GOT address
michael@0	83
michael@0	84	; ---- Pass 1: process columns from input, store into work array.
michael@0	85
michael@0	86	; mov eax, [original_ebp]
michael@0	87	mov edx, POINTER [dct_table(eax)] ; quantptr
michael@0	88	mov esi, JCOEFPTR [coef_block(eax)] ; inptr
michael@0	89	lea edi, [workspace] ; FAST_FLOAT * wsptr
michael@0	90	mov ecx, DCTSIZE/2 ; ctr
michael@0	91	alignx 16,7
michael@0	92	.columnloop:
michael@0	93	%ifndef NO_ZERO_COLUMN_TEST_FLOAT_3DNOW
michael@0	94	mov eax, DWORD [DWBLOCK(1,0,esi,SIZEOF_JCOEF)]
michael@0	95	or eax, DWORD [DWBLOCK(2,0,esi,SIZEOF_JCOEF)]
michael@0	96	jnz short .columnDCT
michael@0	97
michael@0	98	pushpic ebx ; save GOT address
michael@0	99	mov ebx, DWORD [DWBLOCK(3,0,esi,SIZEOF_JCOEF)]
michael@0	100	mov eax, DWORD [DWBLOCK(4,0,esi,SIZEOF_JCOEF)]
michael@0	101	or ebx, DWORD [DWBLOCK(5,0,esi,SIZEOF_JCOEF)]
michael@0	102	or eax, DWORD [DWBLOCK(6,0,esi,SIZEOF_JCOEF)]
michael@0	103	or ebx, DWORD [DWBLOCK(7,0,esi,SIZEOF_JCOEF)]
michael@0	104	or eax,ebx
michael@0	105	poppic ebx ; restore GOT address
michael@0	106	jnz short .columnDCT
michael@0	107
michael@0	108	; -- AC terms all zero
michael@0	109
michael@0	110	movd mm0, DWORD [DWBLOCK(0,0,esi,SIZEOF_JCOEF)]
michael@0	111
michael@0	112	punpcklwd mm0,mm0
michael@0	113	psrad mm0,(DWORD_BIT-WORD_BIT)
michael@0	114	pi2fd mm0,mm0
michael@0	115
michael@0	116	pfmul mm0, MMWORD [MMBLOCK(0,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	117
michael@0	118	movq mm1,mm0
michael@0	119	punpckldq mm0,mm0
michael@0	120	punpckhdq mm1,mm1
michael@0	121
michael@0	122	movq MMWORD [MMBLOCK(0,0,edi,SIZEOF_FAST_FLOAT)], mm0
michael@0	123	movq MMWORD [MMBLOCK(0,1,edi,SIZEOF_FAST_FLOAT)], mm0
michael@0	124	movq MMWORD [MMBLOCK(0,2,edi,SIZEOF_FAST_FLOAT)], mm0
michael@0	125	movq MMWORD [MMBLOCK(0,3,edi,SIZEOF_FAST_FLOAT)], mm0
michael@0	126	movq MMWORD [MMBLOCK(1,0,edi,SIZEOF_FAST_FLOAT)], mm1
michael@0	127	movq MMWORD [MMBLOCK(1,1,edi,SIZEOF_FAST_FLOAT)], mm1
michael@0	128	movq MMWORD [MMBLOCK(1,2,edi,SIZEOF_FAST_FLOAT)], mm1
michael@0	129	movq MMWORD [MMBLOCK(1,3,edi,SIZEOF_FAST_FLOAT)], mm1
michael@0	130	jmp near .nextcolumn
michael@0	131	alignx 16,7
michael@0	132	%endif
michael@0	133	.columnDCT:
michael@0	134
michael@0	135	; -- Even part
michael@0	136
michael@0	137	movd mm0, DWORD [DWBLOCK(0,0,esi,SIZEOF_JCOEF)]
michael@0	138	movd mm1, DWORD [DWBLOCK(2,0,esi,SIZEOF_JCOEF)]
michael@0	139	movd mm2, DWORD [DWBLOCK(4,0,esi,SIZEOF_JCOEF)]
michael@0	140	movd mm3, DWORD [DWBLOCK(6,0,esi,SIZEOF_JCOEF)]
michael@0	141
michael@0	142	punpcklwd mm0,mm0
michael@0	143	punpcklwd mm1,mm1
michael@0	144	psrad mm0,(DWORD_BIT-WORD_BIT)
michael@0	145	psrad mm1,(DWORD_BIT-WORD_BIT)
michael@0	146	pi2fd mm0,mm0
michael@0	147	pi2fd mm1,mm1
michael@0	148
michael@0	149	pfmul mm0, MMWORD [MMBLOCK(0,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	150	pfmul mm1, MMWORD [MMBLOCK(2,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	151
michael@0	152	punpcklwd mm2,mm2
michael@0	153	punpcklwd mm3,mm3
michael@0	154	psrad mm2,(DWORD_BIT-WORD_BIT)
michael@0	155	psrad mm3,(DWORD_BIT-WORD_BIT)
michael@0	156	pi2fd mm2,mm2
michael@0	157	pi2fd mm3,mm3
michael@0	158
michael@0	159	pfmul mm2, MMWORD [MMBLOCK(4,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	160	pfmul mm3, MMWORD [MMBLOCK(6,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	161
michael@0	162	movq mm4,mm0
michael@0	163	movq mm5,mm1
michael@0	164	pfsub mm0,mm2 ; mm0=tmp11
michael@0	165	pfsub mm1,mm3
michael@0	166	pfadd mm4,mm2 ; mm4=tmp10
michael@0	167	pfadd mm5,mm3 ; mm5=tmp13
michael@0	168
michael@0	169	pfmul mm1,[GOTOFF(ebx,PD_1_414)]
michael@0	170	pfsub mm1,mm5 ; mm1=tmp12
michael@0	171
michael@0	172	movq mm6,mm4
michael@0	173	movq mm7,mm0
michael@0	174	pfsub mm4,mm5 ; mm4=tmp3
michael@0	175	pfsub mm0,mm1 ; mm0=tmp2
michael@0	176	pfadd mm6,mm5 ; mm6=tmp0
michael@0	177	pfadd mm7,mm1 ; mm7=tmp1
michael@0	178
michael@0	179	movq MMWORD [wk(1)], mm4 ; tmp3
michael@0	180	movq MMWORD [wk(0)], mm0 ; tmp2
michael@0	181
michael@0	182	; -- Odd part
michael@0	183
michael@0	184	movd mm2, DWORD [DWBLOCK(1,0,esi,SIZEOF_JCOEF)]
michael@0	185	movd mm3, DWORD [DWBLOCK(3,0,esi,SIZEOF_JCOEF)]
michael@0	186	movd mm5, DWORD [DWBLOCK(5,0,esi,SIZEOF_JCOEF)]
michael@0	187	movd mm1, DWORD [DWBLOCK(7,0,esi,SIZEOF_JCOEF)]
michael@0	188
michael@0	189	punpcklwd mm2,mm2
michael@0	190	punpcklwd mm3,mm3
michael@0	191	psrad mm2,(DWORD_BIT-WORD_BIT)
michael@0	192	psrad mm3,(DWORD_BIT-WORD_BIT)
michael@0	193	pi2fd mm2,mm2
michael@0	194	pi2fd mm3,mm3
michael@0	195
michael@0	196	pfmul mm2, MMWORD [MMBLOCK(1,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	197	pfmul mm3, MMWORD [MMBLOCK(3,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	198
michael@0	199	punpcklwd mm5,mm5
michael@0	200	punpcklwd mm1,mm1
michael@0	201	psrad mm5,(DWORD_BIT-WORD_BIT)
michael@0	202	psrad mm1,(DWORD_BIT-WORD_BIT)
michael@0	203	pi2fd mm5,mm5
michael@0	204	pi2fd mm1,mm1
michael@0	205
michael@0	206	pfmul mm5, MMWORD [MMBLOCK(5,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	207	pfmul mm1, MMWORD [MMBLOCK(7,0,edx,SIZEOF_FLOAT_MULT_TYPE)]
michael@0	208
michael@0	209	movq mm4,mm2
michael@0	210	movq mm0,mm5
michael@0	211	pfadd mm2,mm1 ; mm2=z11
michael@0	212	pfadd mm5,mm3 ; mm5=z13
michael@0	213	pfsub mm4,mm1 ; mm4=z12
michael@0	214	pfsub mm0,mm3 ; mm0=z10
michael@0	215
michael@0	216	movq mm1,mm2
michael@0	217	pfsub mm2,mm5
michael@0	218	pfadd mm1,mm5 ; mm1=tmp7
michael@0	219
michael@0	220	pfmul mm2,[GOTOFF(ebx,PD_1_414)] ; mm2=tmp11
michael@0	221
michael@0	222	movq mm3,mm0
michael@0	223	pfadd mm0,mm4
michael@0	224	pfmul mm0,[GOTOFF(ebx,PD_1_847)] ; mm0=z5
michael@0	225	pfmul mm3,[GOTOFF(ebx,PD_2_613)] ; mm3=(z10 * 2.613125930)
michael@0	226	pfmul mm4,[GOTOFF(ebx,PD_1_082)] ; mm4=(z12 * 1.082392200)
michael@0	227	pfsubr mm3,mm0 ; mm3=tmp12
michael@0	228	pfsub mm4,mm0 ; mm4=tmp10
michael@0	229
michael@0	230	; -- Final output stage
michael@0	231
michael@0	232	pfsub mm3,mm1 ; mm3=tmp6
michael@0	233	movq mm5,mm6
michael@0	234	movq mm0,mm7
michael@0	235	pfadd mm6,mm1 ; mm6=data0=(00 01)
michael@0	236	pfadd mm7,mm3 ; mm7=data1=(10 11)
michael@0	237	pfsub mm5,mm1 ; mm5=data7=(70 71)
michael@0	238	pfsub mm0,mm3 ; mm0=data6=(60 61)
michael@0	239	pfsub mm2,mm3 ; mm2=tmp5
michael@0	240
michael@0	241	movq mm1,mm6 ; transpose coefficients
michael@0	242	punpckldq mm6,mm7 ; mm6=(00 10)
michael@0	243	punpckhdq mm1,mm7 ; mm1=(01 11)
michael@0	244	movq mm3,mm0 ; transpose coefficients
michael@0	245	punpckldq mm0,mm5 ; mm0=(60 70)
michael@0	246	punpckhdq mm3,mm5 ; mm3=(61 71)
michael@0	247
michael@0	248	movq MMWORD [MMBLOCK(0,0,edi,SIZEOF_FAST_FLOAT)], mm6
michael@0	249	movq MMWORD [MMBLOCK(1,0,edi,SIZEOF_FAST_FLOAT)], mm1
michael@0	250	movq MMWORD [MMBLOCK(0,3,edi,SIZEOF_FAST_FLOAT)], mm0
michael@0	251	movq MMWORD [MMBLOCK(1,3,edi,SIZEOF_FAST_FLOAT)], mm3
michael@0	252
michael@0	253	movq mm7, MMWORD [wk(0)] ; mm7=tmp2
michael@0	254	movq mm5, MMWORD [wk(1)] ; mm5=tmp3
michael@0	255
michael@0	256	pfadd mm4,mm2 ; mm4=tmp4
michael@0	257	movq mm6,mm7
michael@0	258	movq mm1,mm5
michael@0	259	pfadd mm7,mm2 ; mm7=data2=(20 21)
michael@0	260	pfadd mm5,mm4 ; mm5=data4=(40 41)
michael@0	261	pfsub mm6,mm2 ; mm6=data5=(50 51)
michael@0	262	pfsub mm1,mm4 ; mm1=data3=(30 31)
michael@0	263
michael@0	264	movq mm0,mm7 ; transpose coefficients
michael@0	265	punpckldq mm7,mm1 ; mm7=(20 30)
michael@0	266	punpckhdq mm0,mm1 ; mm0=(21 31)
michael@0	267	movq mm3,mm5 ; transpose coefficients
michael@0	268	punpckldq mm5,mm6 ; mm5=(40 50)
michael@0	269	punpckhdq mm3,mm6 ; mm3=(41 51)
michael@0	270
michael@0	271	movq MMWORD [MMBLOCK(0,1,edi,SIZEOF_FAST_FLOAT)], mm7
michael@0	272	movq MMWORD [MMBLOCK(1,1,edi,SIZEOF_FAST_FLOAT)], mm0
michael@0	273	movq MMWORD [MMBLOCK(0,2,edi,SIZEOF_FAST_FLOAT)], mm5
michael@0	274	movq MMWORD [MMBLOCK(1,2,edi,SIZEOF_FAST_FLOAT)], mm3
michael@0	275
michael@0	276	.nextcolumn:
michael@0	277	add esi, byte 2*SIZEOF_JCOEF ; coef_block
michael@0	278	add edx, byte 2*SIZEOF_FLOAT_MULT_TYPE ; quantptr
michael@0	279	add edi, byte 2*DCTSIZE*SIZEOF_FAST_FLOAT ; wsptr
michael@0	280	dec ecx ; ctr
michael@0	281	jnz near .columnloop
michael@0	282
michael@0	283	; -- Prefetch the next coefficient block
michael@0	284
michael@0	285	prefetch [esi + (DCTSIZE2-8)*SIZEOF_JCOEF + 0*32]
michael@0	286	prefetch [esi + (DCTSIZE2-8)*SIZEOF_JCOEF + 1*32]
michael@0	287	prefetch [esi + (DCTSIZE2-8)*SIZEOF_JCOEF + 2*32]
michael@0	288	prefetch [esi + (DCTSIZE2-8)*SIZEOF_JCOEF + 3*32]
michael@0	289
michael@0	290	; ---- Pass 2: process rows from work array, store into output array.
michael@0	291
michael@0	292	mov eax, [original_ebp]
michael@0	293	lea esi, [workspace] ; FAST_FLOAT * wsptr
michael@0	294	mov edi, JSAMPARRAY [output_buf(eax)] ; (JSAMPROW *)
michael@0	295	mov eax, JDIMENSION [output_col(eax)]
michael@0	296	mov ecx, DCTSIZE/2 ; ctr
michael@0	297	alignx 16,7
michael@0	298	.rowloop:
michael@0	299
michael@0	300	; -- Even part
michael@0	301
michael@0	302	movq mm0, MMWORD [MMBLOCK(0,0,esi,SIZEOF_FAST_FLOAT)]
michael@0	303	movq mm1, MMWORD [MMBLOCK(2,0,esi,SIZEOF_FAST_FLOAT)]
michael@0	304	movq mm2, MMWORD [MMBLOCK(4,0,esi,SIZEOF_FAST_FLOAT)]
michael@0	305	movq mm3, MMWORD [MMBLOCK(6,0,esi,SIZEOF_FAST_FLOAT)]
michael@0	306
michael@0	307	movq mm4,mm0
michael@0	308	movq mm5,mm1
michael@0	309	pfsub mm0,mm2 ; mm0=tmp11
michael@0	310	pfsub mm1,mm3
michael@0	311	pfadd mm4,mm2 ; mm4=tmp10
michael@0	312	pfadd mm5,mm3 ; mm5=tmp13
michael@0	313
michael@0	314	pfmul mm1,[GOTOFF(ebx,PD_1_414)]
michael@0	315	pfsub mm1,mm5 ; mm1=tmp12
michael@0	316
michael@0	317	movq mm6,mm4
michael@0	318	movq mm7,mm0
michael@0	319	pfsub mm4,mm5 ; mm4=tmp3
michael@0	320	pfsub mm0,mm1 ; mm0=tmp2
michael@0	321	pfadd mm6,mm5 ; mm6=tmp0
michael@0	322	pfadd mm7,mm1 ; mm7=tmp1
michael@0	323
michael@0	324	movq MMWORD [wk(1)], mm4 ; tmp3
michael@0	325	movq MMWORD [wk(0)], mm0 ; tmp2
michael@0	326
michael@0	327	; -- Odd part
michael@0	328
michael@0	329	movq mm2, MMWORD [MMBLOCK(1,0,esi,SIZEOF_FAST_FLOAT)]
michael@0	330	movq mm3, MMWORD [MMBLOCK(3,0,esi,SIZEOF_FAST_FLOAT)]
michael@0	331	movq mm5, MMWORD [MMBLOCK(5,0,esi,SIZEOF_FAST_FLOAT)]
michael@0	332	movq mm1, MMWORD [MMBLOCK(7,0,esi,SIZEOF_FAST_FLOAT)]
michael@0	333
michael@0	334	movq mm4,mm2
michael@0	335	movq mm0,mm5
michael@0	336	pfadd mm2,mm1 ; mm2=z11
michael@0	337	pfadd mm5,mm3 ; mm5=z13
michael@0	338	pfsub mm4,mm1 ; mm4=z12
michael@0	339	pfsub mm0,mm3 ; mm0=z10
michael@0	340
michael@0	341	movq mm1,mm2
michael@0	342	pfsub mm2,mm5
michael@0	343	pfadd mm1,mm5 ; mm1=tmp7
michael@0	344
michael@0	345	pfmul mm2,[GOTOFF(ebx,PD_1_414)] ; mm2=tmp11
michael@0	346
michael@0	347	movq mm3,mm0
michael@0	348	pfadd mm0,mm4
michael@0	349	pfmul mm0,[GOTOFF(ebx,PD_1_847)] ; mm0=z5
michael@0	350	pfmul mm3,[GOTOFF(ebx,PD_2_613)] ; mm3=(z10 * 2.613125930)
michael@0	351	pfmul mm4,[GOTOFF(ebx,PD_1_082)] ; mm4=(z12 * 1.082392200)
michael@0	352	pfsubr mm3,mm0 ; mm3=tmp12
michael@0	353	pfsub mm4,mm0 ; mm4=tmp10
michael@0	354
michael@0	355	; -- Final output stage
michael@0	356
michael@0	357	pfsub mm3,mm1 ; mm3=tmp6
michael@0	358	movq mm5,mm6
michael@0	359	movq mm0,mm7
michael@0	360	pfadd mm6,mm1 ; mm6=data0=(00 10)
michael@0	361	pfadd mm7,mm3 ; mm7=data1=(01 11)
michael@0	362	pfsub mm5,mm1 ; mm5=data7=(07 17)
michael@0	363	pfsub mm0,mm3 ; mm0=data6=(06 16)
michael@0	364	pfsub mm2,mm3 ; mm2=tmp5
michael@0	365
michael@0	366	movq mm1,[GOTOFF(ebx,PD_RNDINT_MAGIC)] ; mm1=[PD_RNDINT_MAGIC]
michael@0	367	pcmpeqd mm3,mm3
michael@0	368	psrld mm3,WORD_BIT ; mm3={0xFFFF 0x0000 0xFFFF 0x0000}
michael@0	369
michael@0	370	pfadd mm6,mm1 ; mm6=roundint(data0/8)=(00 ** 10 **)
michael@0	371	pfadd mm7,mm1 ; mm7=roundint(data1/8)=(01 ** 11 **)
michael@0	372	pfadd mm0,mm1 ; mm0=roundint(data6/8)=(06 ** 16 **)
michael@0	373	pfadd mm5,mm1 ; mm5=roundint(data7/8)=(07 ** 17 **)
michael@0	374
michael@0	375	pand mm6,mm3 ; mm6=(00 -- 10 --)
michael@0	376	pslld mm7,WORD_BIT ; mm7=(-- 01 -- 11)
michael@0	377	pand mm0,mm3 ; mm0=(06 -- 16 --)
michael@0	378	pslld mm5,WORD_BIT ; mm5=(-- 07 -- 17)
michael@0	379	por mm6,mm7 ; mm6=(00 01 10 11)
michael@0	380	por mm0,mm5 ; mm0=(06 07 16 17)
michael@0	381
michael@0	382	movq mm1, MMWORD [wk(0)] ; mm1=tmp2
michael@0	383	movq mm3, MMWORD [wk(1)] ; mm3=tmp3
michael@0	384
michael@0	385	pfadd mm4,mm2 ; mm4=tmp4
michael@0	386	movq mm7,mm1
michael@0	387	movq mm5,mm3
michael@0	388	pfadd mm1,mm2 ; mm1=data2=(02 12)
michael@0	389	pfadd mm3,mm4 ; mm3=data4=(04 14)
michael@0	390	pfsub mm7,mm2 ; mm7=data5=(05 15)
michael@0	391	pfsub mm5,mm4 ; mm5=data3=(03 13)
michael@0	392
michael@0	393	movq mm2,[GOTOFF(ebx,PD_RNDINT_MAGIC)] ; mm2=[PD_RNDINT_MAGIC]
michael@0	394	pcmpeqd mm4,mm4
michael@0	395	psrld mm4,WORD_BIT ; mm4={0xFFFF 0x0000 0xFFFF 0x0000}
michael@0	396
michael@0	397	pfadd mm3,mm2 ; mm3=roundint(data4/8)=(04 ** 14 **)
michael@0	398	pfadd mm7,mm2 ; mm7=roundint(data5/8)=(05 ** 15 **)
michael@0	399	pfadd mm1,mm2 ; mm1=roundint(data2/8)=(02 ** 12 **)
michael@0	400	pfadd mm5,mm2 ; mm5=roundint(data3/8)=(03 ** 13 **)
michael@0	401
michael@0	402	pand mm3,mm4 ; mm3=(04 -- 14 --)
michael@0	403	pslld mm7,WORD_BIT ; mm7=(-- 05 -- 15)
michael@0	404	pand mm1,mm4 ; mm1=(02 -- 12 --)
michael@0	405	pslld mm5,WORD_BIT ; mm5=(-- 03 -- 13)
michael@0	406	por mm3,mm7 ; mm3=(04 05 14 15)
michael@0	407	por mm1,mm5 ; mm1=(02 03 12 13)
michael@0	408
michael@0	409	movq mm2,[GOTOFF(ebx,PB_CENTERJSAMP)] ; mm2=[PB_CENTERJSAMP]
michael@0	410
michael@0	411	packsswb mm6,mm3 ; mm6=(00 01 10 11 04 05 14 15)
michael@0	412	packsswb mm1,mm0 ; mm1=(02 03 12 13 06 07 16 17)
michael@0	413	paddb mm6,mm2
michael@0	414	paddb mm1,mm2
michael@0	415
michael@0	416	movq mm4,mm6 ; transpose coefficients(phase 2)
michael@0	417	punpcklwd mm6,mm1 ; mm6=(00 01 02 03 10 11 12 13)
michael@0	418	punpckhwd mm4,mm1 ; mm4=(04 05 06 07 14 15 16 17)
michael@0	419
michael@0	420	movq mm7,mm6 ; transpose coefficients(phase 3)
michael@0	421	punpckldq mm6,mm4 ; mm6=(00 01 02 03 04 05 06 07)
michael@0	422	punpckhdq mm7,mm4 ; mm7=(10 11 12 13 14 15 16 17)
michael@0	423
michael@0	424	pushpic ebx ; save GOT address
michael@0	425
michael@0	426	mov edx, JSAMPROW [edi+0*SIZEOF_JSAMPROW]
michael@0	427	mov ebx, JSAMPROW [edi+1*SIZEOF_JSAMPROW]
michael@0	428	movq MMWORD [edx+eax*SIZEOF_JSAMPLE], mm6
michael@0	429	movq MMWORD [ebx+eax*SIZEOF_JSAMPLE], mm7
michael@0	430
michael@0	431	poppic ebx ; restore GOT address
michael@0	432
michael@0	433	add esi, byte 2*SIZEOF_FAST_FLOAT ; wsptr
michael@0	434	add edi, byte 2*SIZEOF_JSAMPROW
michael@0	435	dec ecx ; ctr
michael@0	436	jnz near .rowloop
michael@0	437
michael@0	438	femms ; empty MMX/3DNow! state
michael@0	439
michael@0	440	pop edi
michael@0	441	pop esi
michael@0	442	; pop edx ; need not be preserved
michael@0	443	; pop ecx ; need not be preserved
michael@0	444	pop ebx
michael@0	445	mov esp,ebp ; esp <- aligned ebp
michael@0	446	pop esp ; esp <- original ebp
michael@0	447	pop ebp
michael@0	448	ret
michael@0	449
michael@0	450	; For some reason, the OS X linker does not honor the request to align the
michael@0	451	; segment unless we do this.
michael@0	452	align 16