• file
• revisions
• annotate
• diff
• comparison
• raw

media/libjpeg/simd/jfss2fst-64.asm@6474c204b198 (annotated)

media/libjpeg/simd/jfss2fst-64.asm

Wed, 31 Dec 2014 06:09:35 +0100

author

Michael Schloh von Bennewitz <michael@schloh.com>

date

Wed, 31 Dec 2014 06:09:35 +0100

changeset 0

6474c204b198

permissions

-rw-r--r--

Cloned upstream origin tor-browser at tor-browser-31.3.0esr-4.5-1-build1
revision ID fc1c9ff7c1b2defdbc039f12214767608f46423f for hacking purpose.

michael@0	1	;
michael@0	2	; jfss2fst-64.asm - fast integer FDCT (64-bit SSE2)
michael@0	3	;
michael@0	4	; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
michael@0	5	; Copyright 2009 D. R. Commander
michael@0	6	;
michael@0	7	; Based on
michael@0	8	; x86 SIMD extension for IJG JPEG library
michael@0	9	; Copyright (C) 1999-2006, MIYASAKA Masaru.
michael@0	10	; For conditions of distribution and use, see copyright notice in jsimdext.inc
michael@0	11	;
michael@0	12	; This file should be assembled with NASM (Netwide Assembler),
michael@0	13	; can *not* be assembled with Microsoft's MASM or any compatible
michael@0	14	; assembler (including Borland's Turbo Assembler).
michael@0	15	; NASM is available from http://nasm.sourceforge.net/ or
michael@0	16	; http://sourceforge.net/project/showfiles.php?group_id=6208
michael@0	17	;
michael@0	18	; This file contains a fast, not so accurate integer implementation of
michael@0	19	; the forward DCT (Discrete Cosine Transform). The following code is
michael@0	20	; based directly on the IJG's original jfdctfst.c; see the jfdctfst.c
michael@0	21	; for more details.
michael@0	22	;
michael@0	23	; [TAB8]
michael@0	24
michael@0	25	%include "jsimdext.inc"
michael@0	26	%include "jdct.inc"
michael@0	27
michael@0	28	; --------------------------------------------------------------------------
michael@0	29
michael@0	30	%define CONST_BITS 8 ; 14 is also OK.
michael@0	31
michael@0	32	%if CONST_BITS == 8
michael@0	33	F_0_382 equ 98 ; FIX(0.382683433)
michael@0	34	F_0_541 equ 139 ; FIX(0.541196100)
michael@0	35	F_0_707 equ 181 ; FIX(0.707106781)
michael@0	36	F_1_306 equ 334 ; FIX(1.306562965)
michael@0	37	%else
michael@0	38	; NASM cannot do compile-time arithmetic on floating-point constants.
michael@0	39	%define DESCALE(x,n) (((x)+(1<<((n)-1)))>>(n))
michael@0	40	F_0_382 equ DESCALE( 410903207,30-CONST_BITS) ; FIX(0.382683433)
michael@0	41	F_0_541 equ DESCALE( 581104887,30-CONST_BITS) ; FIX(0.541196100)
michael@0	42	F_0_707 equ DESCALE( 759250124,30-CONST_BITS) ; FIX(0.707106781)
michael@0	43	F_1_306 equ DESCALE(1402911301,30-CONST_BITS) ; FIX(1.306562965)
michael@0	44	%endif
michael@0	45
michael@0	46	; --------------------------------------------------------------------------
michael@0	47	SECTION SEG_CONST
michael@0	48
michael@0	49	; PRE_MULTIPLY_SCALE_BITS <= 2 (to avoid overflow)
michael@0	50	; CONST_BITS + CONST_SHIFT + PRE_MULTIPLY_SCALE_BITS == 16 (for pmulhw)
michael@0	51
michael@0	52	%define PRE_MULTIPLY_SCALE_BITS 2
michael@0	53	%define CONST_SHIFT (16 - PRE_MULTIPLY_SCALE_BITS - CONST_BITS)
michael@0	54
michael@0	55	alignz 16
michael@0	56	global EXTN(jconst_fdct_ifast_sse2)
michael@0	57
michael@0	58	EXTN(jconst_fdct_ifast_sse2):
michael@0	59
michael@0	60	PW_F0707 times 8 dw F_0_707 << CONST_SHIFT
michael@0	61	PW_F0382 times 8 dw F_0_382 << CONST_SHIFT
michael@0	62	PW_F0541 times 8 dw F_0_541 << CONST_SHIFT
michael@0	63	PW_F1306 times 8 dw F_1_306 << CONST_SHIFT
michael@0	64
michael@0	65	alignz 16
michael@0	66
michael@0	67	; --------------------------------------------------------------------------
michael@0	68	SECTION SEG_TEXT
michael@0	69	BITS 64
michael@0	70	;
michael@0	71	; Perform the forward DCT on one block of samples.
michael@0	72	;
michael@0	73	; GLOBAL(void)
michael@0	74	; jsimd_fdct_ifast_sse2 (DCTELEM * data)
michael@0	75	;
michael@0	76
michael@0	77	; r10 = DCTELEM * data
michael@0	78
michael@0	79	%define wk(i) rbp-(WK_NUM-(i))*SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
michael@0	80	%define WK_NUM 2
michael@0	81
michael@0	82	align 16
michael@0	83	global EXTN(jsimd_fdct_ifast_sse2)
michael@0	84
michael@0	85	EXTN(jsimd_fdct_ifast_sse2):
michael@0	86	push rbp
michael@0	87	mov rax,rsp ; rax = original rbp
michael@0	88	sub rsp, byte 4
michael@0	89	and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
michael@0	90	mov [rsp],rax
michael@0	91	mov rbp,rsp ; rbp = aligned rbp
michael@0	92	lea rsp, [wk(0)]
michael@0	93	collect_args
michael@0	94
michael@0	95	; ---- Pass 1: process rows.
michael@0	96
michael@0	97	mov rdx, r10 ; (DCTELEM *)
michael@0	98
michael@0	99	movdqa xmm0, XMMWORD [XMMBLOCK(0,0,rdx,SIZEOF_DCTELEM)]
michael@0	100	movdqa xmm1, XMMWORD [XMMBLOCK(1,0,rdx,SIZEOF_DCTELEM)]
michael@0	101	movdqa xmm2, XMMWORD [XMMBLOCK(2,0,rdx,SIZEOF_DCTELEM)]
michael@0	102	movdqa xmm3, XMMWORD [XMMBLOCK(3,0,rdx,SIZEOF_DCTELEM)]
michael@0	103
michael@0	104	; xmm0=(00 01 02 03 04 05 06 07), xmm2=(20 21 22 23 24 25 26 27)
michael@0	105	; xmm1=(10 11 12 13 14 15 16 17), xmm3=(30 31 32 33 34 35 36 37)
michael@0	106
michael@0	107	movdqa xmm4,xmm0 ; transpose coefficients(phase 1)
michael@0	108	punpcklwd xmm0,xmm1 ; xmm0=(00 10 01 11 02 12 03 13)
michael@0	109	punpckhwd xmm4,xmm1 ; xmm4=(04 14 05 15 06 16 07 17)
michael@0	110	movdqa xmm5,xmm2 ; transpose coefficients(phase 1)
michael@0	111	punpcklwd xmm2,xmm3 ; xmm2=(20 30 21 31 22 32 23 33)
michael@0	112	punpckhwd xmm5,xmm3 ; xmm5=(24 34 25 35 26 36 27 37)
michael@0	113
michael@0	114	movdqa xmm6, XMMWORD [XMMBLOCK(4,0,rdx,SIZEOF_DCTELEM)]
michael@0	115	movdqa xmm7, XMMWORD [XMMBLOCK(5,0,rdx,SIZEOF_DCTELEM)]
michael@0	116	movdqa xmm1, XMMWORD [XMMBLOCK(6,0,rdx,SIZEOF_DCTELEM)]
michael@0	117	movdqa xmm3, XMMWORD [XMMBLOCK(7,0,rdx,SIZEOF_DCTELEM)]
michael@0	118
michael@0	119	; xmm6=( 4 12 20 28 36 44 52 60), xmm1=( 6 14 22 30 38 46 54 62)
michael@0	120	; xmm7=( 5 13 21 29 37 45 53 61), xmm3=( 7 15 23 31 39 47 55 63)
michael@0	121
michael@0	122	movdqa XMMWORD [wk(0)], xmm2 ; wk(0)=(20 30 21 31 22 32 23 33)
michael@0	123	movdqa XMMWORD [wk(1)], xmm5 ; wk(1)=(24 34 25 35 26 36 27 37)
michael@0	124
michael@0	125	movdqa xmm2,xmm6 ; transpose coefficients(phase 1)
michael@0	126	punpcklwd xmm6,xmm7 ; xmm6=(40 50 41 51 42 52 43 53)
michael@0	127	punpckhwd xmm2,xmm7 ; xmm2=(44 54 45 55 46 56 47 57)
michael@0	128	movdqa xmm5,xmm1 ; transpose coefficients(phase 1)
michael@0	129	punpcklwd xmm1,xmm3 ; xmm1=(60 70 61 71 62 72 63 73)
michael@0	130	punpckhwd xmm5,xmm3 ; xmm5=(64 74 65 75 66 76 67 77)
michael@0	131
michael@0	132	movdqa xmm7,xmm6 ; transpose coefficients(phase 2)
michael@0	133	punpckldq xmm6,xmm1 ; xmm6=(40 50 60 70 41 51 61 71)
michael@0	134	punpckhdq xmm7,xmm1 ; xmm7=(42 52 62 72 43 53 63 73)
michael@0	135	movdqa xmm3,xmm2 ; transpose coefficients(phase 2)
michael@0	136	punpckldq xmm2,xmm5 ; xmm2=(44 54 64 74 45 55 65 75)
michael@0	137	punpckhdq xmm3,xmm5 ; xmm3=(46 56 66 76 47 57 67 77)
michael@0	138
michael@0	139	movdqa xmm1, XMMWORD [wk(0)] ; xmm1=(20 30 21 31 22 32 23 33)
michael@0	140	movdqa xmm5, XMMWORD [wk(1)] ; xmm5=(24 34 25 35 26 36 27 37)
michael@0	141	movdqa XMMWORD [wk(0)], xmm7 ; wk(0)=(42 52 62 72 43 53 63 73)
michael@0	142	movdqa XMMWORD [wk(1)], xmm2 ; wk(1)=(44 54 64 74 45 55 65 75)
michael@0	143
michael@0	144	movdqa xmm7,xmm0 ; transpose coefficients(phase 2)
michael@0	145	punpckldq xmm0,xmm1 ; xmm0=(00 10 20 30 01 11 21 31)
michael@0	146	punpckhdq xmm7,xmm1 ; xmm7=(02 12 22 32 03 13 23 33)
michael@0	147	movdqa xmm2,xmm4 ; transpose coefficients(phase 2)
michael@0	148	punpckldq xmm4,xmm5 ; xmm4=(04 14 24 34 05 15 25 35)
michael@0	149	punpckhdq xmm2,xmm5 ; xmm2=(06 16 26 36 07 17 27 37)
michael@0	150
michael@0	151	movdqa xmm1,xmm0 ; transpose coefficients(phase 3)
michael@0	152	punpcklqdq xmm0,xmm6 ; xmm0=(00 10 20 30 40 50 60 70)=data0
michael@0	153	punpckhqdq xmm1,xmm6 ; xmm1=(01 11 21 31 41 51 61 71)=data1
michael@0	154	movdqa xmm5,xmm2 ; transpose coefficients(phase 3)
michael@0	155	punpcklqdq xmm2,xmm3 ; xmm2=(06 16 26 36 46 56 66 76)=data6
michael@0	156	punpckhqdq xmm5,xmm3 ; xmm5=(07 17 27 37 47 57 67 77)=data7
michael@0	157
michael@0	158	movdqa xmm6,xmm1
michael@0	159	movdqa xmm3,xmm0
michael@0	160	psubw xmm1,xmm2 ; xmm1=data1-data6=tmp6
michael@0	161	psubw xmm0,xmm5 ; xmm0=data0-data7=tmp7
michael@0	162	paddw xmm6,xmm2 ; xmm6=data1+data6=tmp1
michael@0	163	paddw xmm3,xmm5 ; xmm3=data0+data7=tmp0
michael@0	164
michael@0	165	movdqa xmm2, XMMWORD [wk(0)] ; xmm2=(42 52 62 72 43 53 63 73)
michael@0	166	movdqa xmm5, XMMWORD [wk(1)] ; xmm5=(44 54 64 74 45 55 65 75)
michael@0	167	movdqa XMMWORD [wk(0)], xmm1 ; wk(0)=tmp6
michael@0	168	movdqa XMMWORD [wk(1)], xmm0 ; wk(1)=tmp7
michael@0	169
michael@0	170	movdqa xmm1,xmm7 ; transpose coefficients(phase 3)
michael@0	171	punpcklqdq xmm7,xmm2 ; xmm7=(02 12 22 32 42 52 62 72)=data2
michael@0	172	punpckhqdq xmm1,xmm2 ; xmm1=(03 13 23 33 43 53 63 73)=data3
michael@0	173	movdqa xmm0,xmm4 ; transpose coefficients(phase 3)
michael@0	174	punpcklqdq xmm4,xmm5 ; xmm4=(04 14 24 34 44 54 64 74)=data4
michael@0	175	punpckhqdq xmm0,xmm5 ; xmm0=(05 15 25 35 45 55 65 75)=data5
michael@0	176
michael@0	177	movdqa xmm2,xmm1
michael@0	178	movdqa xmm5,xmm7
michael@0	179	paddw xmm1,xmm4 ; xmm1=data3+data4=tmp3
michael@0	180	paddw xmm7,xmm0 ; xmm7=data2+data5=tmp2
michael@0	181	psubw xmm2,xmm4 ; xmm2=data3-data4=tmp4
michael@0	182	psubw xmm5,xmm0 ; xmm5=data2-data5=tmp5
michael@0	183
michael@0	184	; -- Even part
michael@0	185
michael@0	186	movdqa xmm4,xmm3
michael@0	187	movdqa xmm0,xmm6
michael@0	188	psubw xmm3,xmm1 ; xmm3=tmp13
michael@0	189	psubw xmm6,xmm7 ; xmm6=tmp12
michael@0	190	paddw xmm4,xmm1 ; xmm4=tmp10
michael@0	191	paddw xmm0,xmm7 ; xmm0=tmp11
michael@0	192
michael@0	193	paddw xmm6,xmm3
michael@0	194	psllw xmm6,PRE_MULTIPLY_SCALE_BITS
michael@0	195	pmulhw xmm6,[rel PW_F0707] ; xmm6=z1
michael@0	196
michael@0	197	movdqa xmm1,xmm4
michael@0	198	movdqa xmm7,xmm3
michael@0	199	psubw xmm4,xmm0 ; xmm4=data4
michael@0	200	psubw xmm3,xmm6 ; xmm3=data6
michael@0	201	paddw xmm1,xmm0 ; xmm1=data0
michael@0	202	paddw xmm7,xmm6 ; xmm7=data2
michael@0	203
michael@0	204	movdqa xmm0, XMMWORD [wk(0)] ; xmm0=tmp6
michael@0	205	movdqa xmm6, XMMWORD [wk(1)] ; xmm6=tmp7
michael@0	206	movdqa XMMWORD [wk(0)], xmm4 ; wk(0)=data4
michael@0	207	movdqa XMMWORD [wk(1)], xmm3 ; wk(1)=data6
michael@0	208
michael@0	209	; -- Odd part
michael@0	210
michael@0	211	paddw xmm2,xmm5 ; xmm2=tmp10
michael@0	212	paddw xmm5,xmm0 ; xmm5=tmp11
michael@0	213	paddw xmm0,xmm6 ; xmm0=tmp12, xmm6=tmp7
michael@0	214
michael@0	215	psllw xmm2,PRE_MULTIPLY_SCALE_BITS
michael@0	216	psllw xmm0,PRE_MULTIPLY_SCALE_BITS
michael@0	217
michael@0	218	psllw xmm5,PRE_MULTIPLY_SCALE_BITS
michael@0	219	pmulhw xmm5,[rel PW_F0707] ; xmm5=z3
michael@0	220
michael@0	221	movdqa xmm4,xmm2 ; xmm4=tmp10
michael@0	222	psubw xmm2,xmm0
michael@0	223	pmulhw xmm2,[rel PW_F0382] ; xmm2=z5
michael@0	224	pmulhw xmm4,[rel PW_F0541] ; xmm4=MULTIPLY(tmp10,FIX_0_541196)
michael@0	225	pmulhw xmm0,[rel PW_F1306] ; xmm0=MULTIPLY(tmp12,FIX_1_306562)
michael@0	226	paddw xmm4,xmm2 ; xmm4=z2
michael@0	227	paddw xmm0,xmm2 ; xmm0=z4
michael@0	228
michael@0	229	movdqa xmm3,xmm6
michael@0	230	psubw xmm6,xmm5 ; xmm6=z13
michael@0	231	paddw xmm3,xmm5 ; xmm3=z11
michael@0	232
michael@0	233	movdqa xmm2,xmm6
michael@0	234	movdqa xmm5,xmm3
michael@0	235	psubw xmm6,xmm4 ; xmm6=data3
michael@0	236	psubw xmm3,xmm0 ; xmm3=data7
michael@0	237	paddw xmm2,xmm4 ; xmm2=data5
michael@0	238	paddw xmm5,xmm0 ; xmm5=data1
michael@0	239
michael@0	240	; ---- Pass 2: process columns.
michael@0	241
michael@0	242	; xmm1=(00 10 20 30 40 50 60 70), xmm7=(02 12 22 32 42 52 62 72)
michael@0	243	; xmm5=(01 11 21 31 41 51 61 71), xmm6=(03 13 23 33 43 53 63 73)
michael@0	244
michael@0	245	movdqa xmm4,xmm1 ; transpose coefficients(phase 1)
michael@0	246	punpcklwd xmm1,xmm5 ; xmm1=(00 01 10 11 20 21 30 31)
michael@0	247	punpckhwd xmm4,xmm5 ; xmm4=(40 41 50 51 60 61 70 71)
michael@0	248	movdqa xmm0,xmm7 ; transpose coefficients(phase 1)
michael@0	249	punpcklwd xmm7,xmm6 ; xmm7=(02 03 12 13 22 23 32 33)
michael@0	250	punpckhwd xmm0,xmm6 ; xmm0=(42 43 52 53 62 63 72 73)
michael@0	251
michael@0	252	movdqa xmm5, XMMWORD [wk(0)] ; xmm5=col4
michael@0	253	movdqa xmm6, XMMWORD [wk(1)] ; xmm6=col6
michael@0	254
michael@0	255	; xmm5=(04 14 24 34 44 54 64 74), xmm6=(06 16 26 36 46 56 66 76)
michael@0	256	; xmm2=(05 15 25 35 45 55 65 75), xmm3=(07 17 27 37 47 57 67 77)
michael@0	257
michael@0	258	movdqa XMMWORD [wk(0)], xmm7 ; wk(0)=(02 03 12 13 22 23 32 33)
michael@0	259	movdqa XMMWORD [wk(1)], xmm0 ; wk(1)=(42 43 52 53 62 63 72 73)
michael@0	260
michael@0	261	movdqa xmm7,xmm5 ; transpose coefficients(phase 1)
michael@0	262	punpcklwd xmm5,xmm2 ; xmm5=(04 05 14 15 24 25 34 35)
michael@0	263	punpckhwd xmm7,xmm2 ; xmm7=(44 45 54 55 64 65 74 75)
michael@0	264	movdqa xmm0,xmm6 ; transpose coefficients(phase 1)
michael@0	265	punpcklwd xmm6,xmm3 ; xmm6=(06 07 16 17 26 27 36 37)
michael@0	266	punpckhwd xmm0,xmm3 ; xmm0=(46 47 56 57 66 67 76 77)
michael@0	267
michael@0	268	movdqa xmm2,xmm5 ; transpose coefficients(phase 2)
michael@0	269	punpckldq xmm5,xmm6 ; xmm5=(04 05 06 07 14 15 16 17)
michael@0	270	punpckhdq xmm2,xmm6 ; xmm2=(24 25 26 27 34 35 36 37)
michael@0	271	movdqa xmm3,xmm7 ; transpose coefficients(phase 2)
michael@0	272	punpckldq xmm7,xmm0 ; xmm7=(44 45 46 47 54 55 56 57)
michael@0	273	punpckhdq xmm3,xmm0 ; xmm3=(64 65 66 67 74 75 76 77)
michael@0	274
michael@0	275	movdqa xmm6, XMMWORD [wk(0)] ; xmm6=(02 03 12 13 22 23 32 33)
michael@0	276	movdqa xmm0, XMMWORD [wk(1)] ; xmm0=(42 43 52 53 62 63 72 73)
michael@0	277	movdqa XMMWORD [wk(0)], xmm2 ; wk(0)=(24 25 26 27 34 35 36 37)
michael@0	278	movdqa XMMWORD [wk(1)], xmm7 ; wk(1)=(44 45 46 47 54 55 56 57)
michael@0	279
michael@0	280	movdqa xmm2,xmm1 ; transpose coefficients(phase 2)
michael@0	281	punpckldq xmm1,xmm6 ; xmm1=(00 01 02 03 10 11 12 13)
michael@0	282	punpckhdq xmm2,xmm6 ; xmm2=(20 21 22 23 30 31 32 33)
michael@0	283	movdqa xmm7,xmm4 ; transpose coefficients(phase 2)
michael@0	284	punpckldq xmm4,xmm0 ; xmm4=(40 41 42 43 50 51 52 53)
michael@0	285	punpckhdq xmm7,xmm0 ; xmm7=(60 61 62 63 70 71 72 73)
michael@0	286
michael@0	287	movdqa xmm6,xmm1 ; transpose coefficients(phase 3)
michael@0	288	punpcklqdq xmm1,xmm5 ; xmm1=(00 01 02 03 04 05 06 07)=data0
michael@0	289	punpckhqdq xmm6,xmm5 ; xmm6=(10 11 12 13 14 15 16 17)=data1
michael@0	290	movdqa xmm0,xmm7 ; transpose coefficients(phase 3)
michael@0	291	punpcklqdq xmm7,xmm3 ; xmm7=(60 61 62 63 64 65 66 67)=data6
michael@0	292	punpckhqdq xmm0,xmm3 ; xmm0=(70 71 72 73 74 75 76 77)=data7
michael@0	293
michael@0	294	movdqa xmm5,xmm6
michael@0	295	movdqa xmm3,xmm1
michael@0	296	psubw xmm6,xmm7 ; xmm6=data1-data6=tmp6
michael@0	297	psubw xmm1,xmm0 ; xmm1=data0-data7=tmp7
michael@0	298	paddw xmm5,xmm7 ; xmm5=data1+data6=tmp1
michael@0	299	paddw xmm3,xmm0 ; xmm3=data0+data7=tmp0
michael@0	300
michael@0	301	movdqa xmm7, XMMWORD [wk(0)] ; xmm7=(24 25 26 27 34 35 36 37)
michael@0	302	movdqa xmm0, XMMWORD [wk(1)] ; xmm0=(44 45 46 47 54 55 56 57)
michael@0	303	movdqa XMMWORD [wk(0)], xmm6 ; wk(0)=tmp6
michael@0	304	movdqa XMMWORD [wk(1)], xmm1 ; wk(1)=tmp7
michael@0	305
michael@0	306	movdqa xmm6,xmm2 ; transpose coefficients(phase 3)
michael@0	307	punpcklqdq xmm2,xmm7 ; xmm2=(20 21 22 23 24 25 26 27)=data2
michael@0	308	punpckhqdq xmm6,xmm7 ; xmm6=(30 31 32 33 34 35 36 37)=data3
michael@0	309	movdqa xmm1,xmm4 ; transpose coefficients(phase 3)
michael@0	310	punpcklqdq xmm4,xmm0 ; xmm4=(40 41 42 43 44 45 46 47)=data4
michael@0	311	punpckhqdq xmm1,xmm0 ; xmm1=(50 51 52 53 54 55 56 57)=data5
michael@0	312
michael@0	313	movdqa xmm7,xmm6
michael@0	314	movdqa xmm0,xmm2
michael@0	315	paddw xmm6,xmm4 ; xmm6=data3+data4=tmp3
michael@0	316	paddw xmm2,xmm1 ; xmm2=data2+data5=tmp2
michael@0	317	psubw xmm7,xmm4 ; xmm7=data3-data4=tmp4
michael@0	318	psubw xmm0,xmm1 ; xmm0=data2-data5=tmp5
michael@0	319
michael@0	320	; -- Even part
michael@0	321
michael@0	322	movdqa xmm4,xmm3
michael@0	323	movdqa xmm1,xmm5
michael@0	324	psubw xmm3,xmm6 ; xmm3=tmp13
michael@0	325	psubw xmm5,xmm2 ; xmm5=tmp12
michael@0	326	paddw xmm4,xmm6 ; xmm4=tmp10
michael@0	327	paddw xmm1,xmm2 ; xmm1=tmp11
michael@0	328
michael@0	329	paddw xmm5,xmm3
michael@0	330	psllw xmm5,PRE_MULTIPLY_SCALE_BITS
michael@0	331	pmulhw xmm5,[rel PW_F0707] ; xmm5=z1
michael@0	332
michael@0	333	movdqa xmm6,xmm4
michael@0	334	movdqa xmm2,xmm3
michael@0	335	psubw xmm4,xmm1 ; xmm4=data4
michael@0	336	psubw xmm3,xmm5 ; xmm3=data6
michael@0	337	paddw xmm6,xmm1 ; xmm6=data0
michael@0	338	paddw xmm2,xmm5 ; xmm2=data2
michael@0	339
michael@0	340	movdqa XMMWORD [XMMBLOCK(4,0,rdx,SIZEOF_DCTELEM)], xmm4
michael@0	341	movdqa XMMWORD [XMMBLOCK(6,0,rdx,SIZEOF_DCTELEM)], xmm3
michael@0	342	movdqa XMMWORD [XMMBLOCK(0,0,rdx,SIZEOF_DCTELEM)], xmm6
michael@0	343	movdqa XMMWORD [XMMBLOCK(2,0,rdx,SIZEOF_DCTELEM)], xmm2
michael@0	344
michael@0	345	; -- Odd part
michael@0	346
michael@0	347	movdqa xmm1, XMMWORD [wk(0)] ; xmm1=tmp6
michael@0	348	movdqa xmm5, XMMWORD [wk(1)] ; xmm5=tmp7
michael@0	349
michael@0	350	paddw xmm7,xmm0 ; xmm7=tmp10
michael@0	351	paddw xmm0,xmm1 ; xmm0=tmp11
michael@0	352	paddw xmm1,xmm5 ; xmm1=tmp12, xmm5=tmp7
michael@0	353
michael@0	354	psllw xmm7,PRE_MULTIPLY_SCALE_BITS
michael@0	355	psllw xmm1,PRE_MULTIPLY_SCALE_BITS
michael@0	356
michael@0	357	psllw xmm0,PRE_MULTIPLY_SCALE_BITS
michael@0	358	pmulhw xmm0,[rel PW_F0707] ; xmm0=z3
michael@0	359
michael@0	360	movdqa xmm4,xmm7 ; xmm4=tmp10
michael@0	361	psubw xmm7,xmm1
michael@0	362	pmulhw xmm7,[rel PW_F0382] ; xmm7=z5
michael@0	363	pmulhw xmm4,[rel PW_F0541] ; xmm4=MULTIPLY(tmp10,FIX_0_541196)
michael@0	364	pmulhw xmm1,[rel PW_F1306] ; xmm1=MULTIPLY(tmp12,FIX_1_306562)
michael@0	365	paddw xmm4,xmm7 ; xmm4=z2
michael@0	366	paddw xmm1,xmm7 ; xmm1=z4
michael@0	367
michael@0	368	movdqa xmm3,xmm5
michael@0	369	psubw xmm5,xmm0 ; xmm5=z13
michael@0	370	paddw xmm3,xmm0 ; xmm3=z11
michael@0	371
michael@0	372	movdqa xmm6,xmm5
michael@0	373	movdqa xmm2,xmm3
michael@0	374	psubw xmm5,xmm4 ; xmm5=data3
michael@0	375	psubw xmm3,xmm1 ; xmm3=data7
michael@0	376	paddw xmm6,xmm4 ; xmm6=data5
michael@0	377	paddw xmm2,xmm1 ; xmm2=data1
michael@0	378
michael@0	379	movdqa XMMWORD [XMMBLOCK(3,0,rdx,SIZEOF_DCTELEM)], xmm5
michael@0	380	movdqa XMMWORD [XMMBLOCK(7,0,rdx,SIZEOF_DCTELEM)], xmm3
michael@0	381	movdqa XMMWORD [XMMBLOCK(5,0,rdx,SIZEOF_DCTELEM)], xmm6
michael@0	382	movdqa XMMWORD [XMMBLOCK(1,0,rdx,SIZEOF_DCTELEM)], xmm2
michael@0	383
michael@0	384	uncollect_args
michael@0	385	mov rsp,rbp ; rsp <- aligned rbp
michael@0	386	pop rsp ; rsp <- original rbp
michael@0	387	pop rbp
michael@0	388	ret
michael@0	389
michael@0	390	; For some reason, the OS X linker does not honor the request to align the
michael@0	391	; segment unless we do this.
michael@0	392	align 16