michael@0: /********************************************************************
michael@0:  *                                                                  *
michael@0:  * THIS FILE IS PART OF THE OggTheora SOFTWARE CODEC SOURCE CODE.   *
michael@0:  * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
michael@0:  * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
michael@0:  * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
michael@0:  *                                                                  *
michael@0:  * THE Theora SOURCE CODE IS COPYRIGHT (C) 2002-2009                *
michael@0:  * by the Xiph.Org Foundation and contributors http://www.xiph.org/ *
michael@0:  *                                                                  *
michael@0:  ********************************************************************
michael@0: 
michael@0:   function:
michael@0:     last mod: $Id: mmxidct.c 16503 2009-08-22 18:14:02Z giles $
michael@0: 
michael@0:  ********************************************************************/
michael@0: 
michael@0: /*SSE2 acceleration of Theora's iDCT.*/
michael@0: #include "x86int.h"
michael@0: #include "sse2trans.h"
michael@0: #include "../dct.h"
michael@0: 
michael@0: #if defined(OC_X86_ASM)
michael@0: 
michael@0: /*A table of constants used by the MMX routines.*/
michael@0: const unsigned short __attribute__((aligned(16),used)) OC_IDCT_CONSTS[64]={
michael@0:         8,      8,      8,      8,      8,      8,      8,      8,
michael@0:   OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,
michael@0:   OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,
michael@0:   OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,
michael@0:   OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,
michael@0:   OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,
michael@0:   OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,
michael@0:   OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1
michael@0: };
michael@0: 
michael@0: 
michael@0: /*Performs the first three stages of the iDCT.
michael@0:   xmm2, xmm6, xmm3, and xmm5 must contain the corresponding rows of the input
michael@0:    (accessed in that order).
michael@0:   The remaining rows must be in _x at their corresponding locations.
michael@0:   On output, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
michael@0:    contain rows 4 through 7.*/
michael@0: #define OC_IDCT_8x8_ABC(_x) \
michael@0:   "#OC_IDCT_8x8_ABC\n\t" \
michael@0:   /*Stage 1:*/ \
michael@0:   /*2-3 rotation by 6pi/16. \
michael@0:     xmm4=xmm7=C6, xmm0=xmm1=C2, xmm2=X2, xmm6=X6.*/ \
michael@0:   "movdqa "OC_MEM_OFFS(0x20,c)",%%xmm1\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x60,c)",%%xmm4\n\t" \
michael@0:   "movdqa %%xmm1,%%xmm0\n\t" \
michael@0:   "pmulhw %%xmm2,%%xmm1\n\t" \
michael@0:   "movdqa %%xmm4,%%xmm7\n\t" \
michael@0:   "pmulhw %%xmm6,%%xmm0\n\t" \
michael@0:   "pmulhw %%xmm2,%%xmm7\n\t" \
michael@0:   "pmulhw %%xmm6,%%xmm4\n\t" \
michael@0:   "paddw %%xmm6,%%xmm0\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x30,c)",%%xmm6\n\t" \
michael@0:   "paddw %%xmm1,%%xmm2\n\t" \
michael@0:   "psubw %%xmm0,%%xmm7\n\t" \
michael@0:   "movdqa %%xmm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
michael@0:   "paddw %%xmm4,%%xmm2\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x50,c)",%%xmm4\n\t" \
michael@0:   "movdqa %%xmm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
michael@0:   /*5-6 rotation by 3pi/16. \
michael@0:     xmm4=xmm2=C5, xmm1=xmm6=C3, xmm3=X3, xmm5=X5.*/ \
michael@0:   "movdqa %%xmm4,%%xmm2\n\t" \
michael@0:   "movdqa %%xmm6,%%xmm1\n\t" \
michael@0:   "pmulhw %%xmm3,%%xmm4\n\t" \
michael@0:   "pmulhw %%xmm5,%%xmm1\n\t" \
michael@0:   "pmulhw %%xmm3,%%xmm6\n\t" \
michael@0:   "pmulhw %%xmm5,%%xmm2\n\t" \
michael@0:   "paddw %%xmm3,%%xmm4\n\t" \
michael@0:   "paddw %%xmm5,%%xmm3\n\t" \
michael@0:   "paddw %%xmm6,%%xmm3\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x70,_x)",%%xmm6\n\t" \
michael@0:   "paddw %%xmm5,%%xmm1\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x10,_x)",%%xmm5\n\t" \
michael@0:   "paddw %%xmm3,%%xmm2\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x70,c)",%%xmm3\n\t" \
michael@0:   "psubw %%xmm4,%%xmm1\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x10,c)",%%xmm4\n\t" \
michael@0:   /*4-7 rotation by 7pi/16. \
michael@0:     xmm4=xmm7=C1, xmm3=xmm0=C7, xmm5=X1, xmm6=X7.*/ \
michael@0:   "movdqa %%xmm3,%%xmm0\n\t" \
michael@0:   "movdqa %%xmm4,%%xmm7\n\t" \
michael@0:   "pmulhw %%xmm5,%%xmm3\n\t" \
michael@0:   "pmulhw %%xmm5,%%xmm7\n\t" \
michael@0:   "pmulhw %%xmm6,%%xmm4\n\t" \
michael@0:   "pmulhw %%xmm6,%%xmm0\n\t" \
michael@0:   "paddw %%xmm6,%%xmm4\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x40,_x)",%%xmm6\n\t" \
michael@0:   "paddw %%xmm5,%%xmm7\n\t" \
michael@0:   "psubw %%xmm4,%%xmm3\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x40,c)",%%xmm4\n\t" \
michael@0:   "paddw %%xmm7,%%xmm0\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x00,_x)",%%xmm7\n\t" \
michael@0:   /*0-1 butterfly. \
michael@0:     xmm4=xmm5=C4, xmm7=X0, xmm6=X4.*/ \
michael@0:   "paddw %%xmm7,%%xmm6\n\t" \
michael@0:   "movdqa %%xmm4,%%xmm5\n\t" \
michael@0:   "pmulhw %%xmm6,%%xmm4\n\t" \
michael@0:   "paddw %%xmm7,%%xmm7\n\t" \
michael@0:   "psubw %%xmm6,%%xmm7\n\t" \
michael@0:   "paddw %%xmm6,%%xmm4\n\t" \
michael@0:   /*Stage 2:*/ \
michael@0:   /*4-5 butterfly: xmm3=t[4], xmm1=t[5] \
michael@0:     7-6 butterfly: xmm2=t[6], xmm0=t[7]*/ \
michael@0:   "movdqa %%xmm3,%%xmm6\n\t" \
michael@0:   "paddw %%xmm1,%%xmm3\n\t" \
michael@0:   "psubw %%xmm1,%%xmm6\n\t" \
michael@0:   "movdqa %%xmm5,%%xmm1\n\t" \
michael@0:   "pmulhw %%xmm7,%%xmm5\n\t" \
michael@0:   "paddw %%xmm7,%%xmm5\n\t" \
michael@0:   "movdqa %%xmm0,%%xmm7\n\t" \
michael@0:   "paddw %%xmm2,%%xmm0\n\t" \
michael@0:   "psubw %%xmm2,%%xmm7\n\t" \
michael@0:   "movdqa %%xmm1,%%xmm2\n\t" \
michael@0:   "pmulhw %%xmm6,%%xmm1\n\t" \
michael@0:   "pmulhw %%xmm7,%%xmm2\n\t" \
michael@0:   "paddw %%xmm6,%%xmm1\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x00,buf)",%%xmm6\n\t" \
michael@0:   "paddw %%xmm7,%%xmm2\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x10,buf)",%%xmm7\n\t" \
michael@0:   /*Stage 3: \
michael@0:     6-5 butterfly: xmm1=t[5], xmm2=t[6] -> xmm1=t[6]+t[5], xmm2=t[6]-t[5] \
michael@0:     0-3 butterfly: xmm4=t[0], xmm7=t[3] -> xmm7=t[0]+t[3], xmm4=t[0]-t[3] \
michael@0:     1-2 butterfly: xmm5=t[1], xmm6=t[2] -> xmm6=t[1]+t[2], xmm5=t[1]-t[2]*/ \
michael@0:   "paddw %%xmm2,%%xmm1\n\t" \
michael@0:   "paddw %%xmm5,%%xmm6\n\t" \
michael@0:   "paddw %%xmm4,%%xmm7\n\t" \
michael@0:   "paddw %%xmm2,%%xmm2\n\t" \
michael@0:   "paddw %%xmm4,%%xmm4\n\t" \
michael@0:   "paddw %%xmm5,%%xmm5\n\t" \
michael@0:   "psubw %%xmm1,%%xmm2\n\t" \
michael@0:   "psubw %%xmm7,%%xmm4\n\t" \
michael@0:   "psubw %%xmm6,%%xmm5\n\t" \
michael@0: 
michael@0: /*Performs the last stage of the iDCT.
michael@0:   On input, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
michael@0:    contain rows 4 through 7.
michael@0:   On output, xmm0 through xmm7 contain the corresponding rows.*/
michael@0: #define OC_IDCT_8x8_D \
michael@0:   "#OC_IDCT_8x8_D\n\t" \
michael@0:   /*Stage 4: \
michael@0:     0-7 butterfly: xmm7=t[0], xmm0=t[7] -> xmm0=t[0]+t[7], xmm7=t[0]-t[7] \
michael@0:     1-6 butterfly: xmm6=t[1], xmm1=t[6] -> xmm1=t[1]+t[6], xmm6=t[1]-t[6] \
michael@0:     2-5 butterfly: xmm5=t[2], xmm2=t[5] -> xmm2=t[2]+t[5], xmm5=t[2]-t[5] \
michael@0:     3-4 butterfly: xmm4=t[3], xmm3=t[4] -> xmm3=t[3]+t[4], xmm4=t[3]-t[4]*/ \
michael@0:   "psubw %%xmm0,%%xmm7\n\t" \
michael@0:   "psubw %%xmm1,%%xmm6\n\t" \
michael@0:   "psubw %%xmm2,%%xmm5\n\t" \
michael@0:   "psubw %%xmm3,%%xmm4\n\t" \
michael@0:   "paddw %%xmm0,%%xmm0\n\t" \
michael@0:   "paddw %%xmm1,%%xmm1\n\t" \
michael@0:   "paddw %%xmm2,%%xmm2\n\t" \
michael@0:   "paddw %%xmm3,%%xmm3\n\t" \
michael@0:   "paddw %%xmm7,%%xmm0\n\t" \
michael@0:   "paddw %%xmm6,%%xmm1\n\t" \
michael@0:   "paddw %%xmm5,%%xmm2\n\t" \
michael@0:   "paddw %%xmm4,%%xmm3\n\t" \
michael@0: 
michael@0: /*Performs the last stage of the iDCT.
michael@0:   On input, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
michael@0:    contain rows 4 through 7.
michael@0:   On output, xmm0 through xmm7 contain the corresponding rows.*/
michael@0: #define OC_IDCT_8x8_D_STORE \
michael@0:   "#OC_IDCT_8x8_D_STORE\n\t" \
michael@0:   /*Stage 4: \
michael@0:     0-7 butterfly: xmm7=t[0], xmm0=t[7] -> xmm0=t[0]+t[7], xmm7=t[0]-t[7] \
michael@0:     1-6 butterfly: xmm6=t[1], xmm1=t[6] -> xmm1=t[1]+t[6], xmm6=t[1]-t[6] \
michael@0:     2-5 butterfly: xmm5=t[2], xmm2=t[5] -> xmm2=t[2]+t[5], xmm5=t[2]-t[5] \
michael@0:     3-4 butterfly: xmm4=t[3], xmm3=t[4] -> xmm3=t[3]+t[4], xmm4=t[3]-t[4]*/ \
michael@0:   "psubw %%xmm3,%%xmm4\n\t" \
michael@0:   "movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x00,c)",%%xmm4\n\t" \
michael@0:   "psubw %%xmm0,%%xmm7\n\t" \
michael@0:   "psubw %%xmm1,%%xmm6\n\t" \
michael@0:   "psubw %%xmm2,%%xmm5\n\t" \
michael@0:   "paddw %%xmm4,%%xmm7\n\t" \
michael@0:   "paddw %%xmm4,%%xmm6\n\t" \
michael@0:   "paddw %%xmm4,%%xmm5\n\t" \
michael@0:   "paddw "OC_MEM_OFFS(0x40,y)",%%xmm4\n\t" \
michael@0:   "paddw %%xmm0,%%xmm0\n\t" \
michael@0:   "paddw %%xmm1,%%xmm1\n\t" \
michael@0:   "paddw %%xmm2,%%xmm2\n\t" \
michael@0:   "paddw %%xmm3,%%xmm3\n\t" \
michael@0:   "paddw %%xmm7,%%xmm0\n\t" \
michael@0:   "paddw %%xmm6,%%xmm1\n\t" \
michael@0:   "psraw $4,%%xmm0\n\t" \
michael@0:   "paddw %%xmm5,%%xmm2\n\t" \
michael@0:   "movdqa %%xmm0,"OC_MEM_OFFS(0x00,y)"\n\t" \
michael@0:   "psraw $4,%%xmm1\n\t" \
michael@0:   "paddw %%xmm4,%%xmm3\n\t" \
michael@0:   "movdqa %%xmm1,"OC_MEM_OFFS(0x10,y)"\n\t" \
michael@0:   "psraw $4,%%xmm2\n\t" \
michael@0:   "movdqa %%xmm2,"OC_MEM_OFFS(0x20,y)"\n\t" \
michael@0:   "psraw $4,%%xmm3\n\t" \
michael@0:   "movdqa %%xmm3,"OC_MEM_OFFS(0x30,y)"\n\t" \
michael@0:   "psraw $4,%%xmm4\n\t" \
michael@0:   "movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t" \
michael@0:   "psraw $4,%%xmm5\n\t" \
michael@0:   "movdqa %%xmm5,"OC_MEM_OFFS(0x50,y)"\n\t" \
michael@0:   "psraw $4,%%xmm6\n\t" \
michael@0:   "movdqa %%xmm6,"OC_MEM_OFFS(0x60,y)"\n\t" \
michael@0:   "psraw $4,%%xmm7\n\t" \
michael@0:   "movdqa %%xmm7,"OC_MEM_OFFS(0x70,y)"\n\t" \
michael@0: 
michael@0: static void oc_idct8x8_slow_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64]){
michael@0:   OC_ALIGN16(ogg_int16_t buf[16]);
michael@0:   /*This routine accepts an 8x8 matrix pre-transposed.*/
michael@0:   __asm__ __volatile__(
michael@0:     /*Load rows 2, 3, 5, and 6 for the first stage of the iDCT.*/
michael@0:     "movdqa "OC_MEM_OFFS(0x20,x)",%%xmm2\n\t"
michael@0:     "movdqa "OC_MEM_OFFS(0x60,x)",%%xmm6\n\t"
michael@0:     "movdqa "OC_MEM_OFFS(0x30,x)",%%xmm3\n\t"
michael@0:     "movdqa "OC_MEM_OFFS(0x50,x)",%%xmm5\n\t"
michael@0:     OC_IDCT_8x8_ABC(x)
michael@0:     OC_IDCT_8x8_D
michael@0:     OC_TRANSPOSE_8x8
michael@0:     /*Clear out rows 0, 1, 4, and 7 for the first stage of the iDCT.*/
michael@0:     "movdqa %%xmm7,"OC_MEM_OFFS(0x70,y)"\n\t"
michael@0:     "movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t"
michael@0:     "movdqa %%xmm1,"OC_MEM_OFFS(0x10,y)"\n\t"
michael@0:     "movdqa %%xmm0,"OC_MEM_OFFS(0x00,y)"\n\t"
michael@0:     OC_IDCT_8x8_ABC(y)
michael@0:     OC_IDCT_8x8_D_STORE
michael@0:     :[buf]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,buf,16)),
michael@0:      [y]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_y,64))
michael@0:     :[x]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,_x,64)),
michael@0:      [c]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,OC_IDCT_CONSTS,128))
michael@0:   );
michael@0:   if(_x!=_y){
michael@0:     int i;
michael@0:     __asm__ __volatile__("pxor %%xmm0,%%xmm0\n\t"::);
michael@0:     /*Clear input data for next block (decoder only).*/
michael@0:     for(i=0;i<2;i++){
michael@0:       __asm__ __volatile__(
michael@0:         "movdqa %%xmm0,"OC_MEM_OFFS(0x00,x)"\n\t"
michael@0:         "movdqa %%xmm0,"OC_MEM_OFFS(0x10,x)"\n\t"
michael@0:         "movdqa %%xmm0,"OC_MEM_OFFS(0x20,x)"\n\t"
michael@0:         "movdqa %%xmm0,"OC_MEM_OFFS(0x30,x)"\n\t"
michael@0:         :[x]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_x+i*32,32))
michael@0:       );
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: /*For the first step of the 10-coefficient version of the 8x8 iDCT, we only
michael@0:    need to work with four columns at a time.
michael@0:   Doing this in MMX is faster on processors with a 64-bit data path.*/
michael@0: #define OC_IDCT_8x8_10_MMX \
michael@0:   "#OC_IDCT_8x8_10_MMX\n\t" \
michael@0:   /*Stage 1:*/ \
michael@0:   /*2-3 rotation by 6pi/16. \
michael@0:     mm7=C6, mm6=C2, mm2=X2, X6=0.*/ \
michael@0:   "movq "OC_MEM_OFFS(0x60,c)",%%mm7\n\t" \
michael@0:   "movq "OC_MEM_OFFS(0x20,c)",%%mm6\n\t" \
michael@0:   "pmulhw %%mm2,%%mm6\n\t" \
michael@0:   "pmulhw %%mm2,%%mm7\n\t" \
michael@0:   "movq "OC_MEM_OFFS(0x50,c)",%%mm5\n\t" \
michael@0:   "paddw %%mm6,%%mm2\n\t" \
michael@0:   "movq %%mm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
michael@0:   "movq "OC_MEM_OFFS(0x30,c)",%%mm2\n\t" \
michael@0:   "movq %%mm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
michael@0:   /*5-6 rotation by 3pi/16. \
michael@0:     mm5=C5, mm2=C3, mm3=X3, X5=0.*/ \
michael@0:   "pmulhw %%mm3,%%mm5\n\t" \
michael@0:   "pmulhw %%mm3,%%mm2\n\t" \
michael@0:   "movq "OC_MEM_OFFS(0x10,c)",%%mm7\n\t" \
michael@0:   "paddw %%mm3,%%mm5\n\t" \
michael@0:   "paddw %%mm3,%%mm2\n\t" \
michael@0:   "movq "OC_MEM_OFFS(0x70,c)",%%mm3\n\t" \
michael@0:   /*4-7 rotation by 7pi/16. \
michael@0:     mm7=C1, mm3=C7, mm1=X1, X7=0.*/ \
michael@0:   "pmulhw %%mm1,%%mm3\n\t" \
michael@0:   "pmulhw %%mm1,%%mm7\n\t" \
michael@0:   "movq "OC_MEM_OFFS(0x40,c)",%%mm4\n\t" \
michael@0:   "movq %%mm3,%%mm6\n\t" \
michael@0:   "paddw %%mm1,%%mm7\n\t" \
michael@0:   /*0-1 butterfly. \
michael@0:     mm4=C4, mm0=X0, X4=0.*/ \
michael@0:   /*Stage 2:*/ \
michael@0:   /*4-5 butterfly: mm3=t[4], mm5=t[5] \
michael@0:     7-6 butterfly: mm2=t[6], mm7=t[7]*/ \
michael@0:   "psubw %%mm5,%%mm3\n\t" \
michael@0:   "paddw %%mm5,%%mm6\n\t" \
michael@0:   "movq %%mm4,%%mm1\n\t" \
michael@0:   "pmulhw %%mm0,%%mm4\n\t" \
michael@0:   "paddw %%mm0,%%mm4\n\t" \
michael@0:   "movq %%mm7,%%mm0\n\t" \
michael@0:   "movq %%mm4,%%mm5\n\t" \
michael@0:   "paddw %%mm2,%%mm0\n\t" \
michael@0:   "psubw %%mm2,%%mm7\n\t" \
michael@0:   "movq %%mm1,%%mm2\n\t" \
michael@0:   "pmulhw %%mm6,%%mm1\n\t" \
michael@0:   "pmulhw %%mm7,%%mm2\n\t" \
michael@0:   "paddw %%mm6,%%mm1\n\t" \
michael@0:   "movq "OC_MEM_OFFS(0x00,buf)",%%mm6\n\t" \
michael@0:   "paddw %%mm7,%%mm2\n\t" \
michael@0:   "movq "OC_MEM_OFFS(0x10,buf)",%%mm7\n\t" \
michael@0:   /*Stage 3: \
michael@0:     6-5 butterfly: mm1=t[5], mm2=t[6] -> mm1=t[6]+t[5], mm2=t[6]-t[5] \
michael@0:     0-3 butterfly: mm4=t[0], mm7=t[3] -> mm7=t[0]+t[3], mm4=t[0]-t[3] \
michael@0:     1-2 butterfly: mm5=t[1], mm6=t[2] -> mm6=t[1]+t[2], mm5=t[1]-t[2]*/ \
michael@0:   "paddw %%mm2,%%mm1\n\t" \
michael@0:   "paddw %%mm5,%%mm6\n\t" \
michael@0:   "paddw %%mm4,%%mm7\n\t" \
michael@0:   "paddw %%mm2,%%mm2\n\t" \
michael@0:   "paddw %%mm4,%%mm4\n\t" \
michael@0:   "paddw %%mm5,%%mm5\n\t" \
michael@0:   "psubw %%mm1,%%mm2\n\t" \
michael@0:   "psubw %%mm7,%%mm4\n\t" \
michael@0:   "psubw %%mm6,%%mm5\n\t" \
michael@0:   /*Stage 4: \
michael@0:     0-7 butterfly: mm7=t[0], mm0=t[7] -> mm0=t[0]+t[7], mm7=t[0]-t[7] \
michael@0:     1-6 butterfly: mm6=t[1], mm1=t[6] -> mm1=t[1]+t[6], mm6=t[1]-t[6] \
michael@0:     2-5 butterfly: mm5=t[2], mm2=t[5] -> mm2=t[2]+t[5], mm5=t[2]-t[5] \
michael@0:     3-4 butterfly: mm4=t[3], mm3=t[4] -> mm3=t[3]+t[4], mm4=t[3]-t[4]*/ \
michael@0:   "psubw %%mm0,%%mm7\n\t" \
michael@0:   "psubw %%mm1,%%mm6\n\t" \
michael@0:   "psubw %%mm2,%%mm5\n\t" \
michael@0:   "psubw %%mm3,%%mm4\n\t" \
michael@0:   "paddw %%mm0,%%mm0\n\t" \
michael@0:   "paddw %%mm1,%%mm1\n\t" \
michael@0:   "paddw %%mm2,%%mm2\n\t" \
michael@0:   "paddw %%mm3,%%mm3\n\t" \
michael@0:   "paddw %%mm7,%%mm0\n\t" \
michael@0:   "paddw %%mm6,%%mm1\n\t" \
michael@0:   "paddw %%mm5,%%mm2\n\t" \
michael@0:   "paddw %%mm4,%%mm3\n\t" \
michael@0: 
michael@0: #define OC_IDCT_8x8_10_ABC \
michael@0:   "#OC_IDCT_8x8_10_ABC\n\t" \
michael@0:   /*Stage 1:*/ \
michael@0:   /*2-3 rotation by 6pi/16. \
michael@0:     xmm7=C6, xmm6=C2, xmm2=X2, X6=0.*/ \
michael@0:   "movdqa "OC_MEM_OFFS(0x60,c)",%%xmm7\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x20,c)",%%xmm6\n\t" \
michael@0:   "pmulhw %%xmm2,%%xmm6\n\t" \
michael@0:   "pmulhw %%xmm2,%%xmm7\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x50,c)",%%xmm5\n\t" \
michael@0:   "paddw %%xmm6,%%xmm2\n\t" \
michael@0:   "movdqa %%xmm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x30,c)",%%xmm2\n\t" \
michael@0:   "movdqa %%xmm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
michael@0:   /*5-6 rotation by 3pi/16. \
michael@0:     xmm5=C5, xmm2=C3, xmm3=X3, X5=0.*/ \
michael@0:   "pmulhw %%xmm3,%%xmm5\n\t" \
michael@0:   "pmulhw %%xmm3,%%xmm2\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x10,c)",%%xmm7\n\t" \
michael@0:   "paddw %%xmm3,%%xmm5\n\t" \
michael@0:   "paddw %%xmm3,%%xmm2\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x70,c)",%%xmm3\n\t" \
michael@0:   /*4-7 rotation by 7pi/16. \
michael@0:     xmm7=C1, xmm3=C7, xmm1=X1, X7=0.*/ \
michael@0:   "pmulhw %%xmm1,%%xmm3\n\t" \
michael@0:   "pmulhw %%xmm1,%%xmm7\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x40,c)",%%xmm4\n\t" \
michael@0:   "movdqa %%xmm3,%%xmm6\n\t" \
michael@0:   "paddw %%xmm1,%%xmm7\n\t" \
michael@0:   /*0-1 butterfly. \
michael@0:     xmm4=C4, xmm0=X0, X4=0.*/ \
michael@0:   /*Stage 2:*/ \
michael@0:   /*4-5 butterfly: xmm3=t[4], xmm5=t[5] \
michael@0:     7-6 butterfly: xmm2=t[6], xmm7=t[7]*/ \
michael@0:   "psubw %%xmm5,%%xmm3\n\t" \
michael@0:   "paddw %%xmm5,%%xmm6\n\t" \
michael@0:   "movdqa %%xmm4,%%xmm1\n\t" \
michael@0:   "pmulhw %%xmm0,%%xmm4\n\t" \
michael@0:   "paddw %%xmm0,%%xmm4\n\t" \
michael@0:   "movdqa %%xmm7,%%xmm0\n\t" \
michael@0:   "movdqa %%xmm4,%%xmm5\n\t" \
michael@0:   "paddw %%xmm2,%%xmm0\n\t" \
michael@0:   "psubw %%xmm2,%%xmm7\n\t" \
michael@0:   "movdqa %%xmm1,%%xmm2\n\t" \
michael@0:   "pmulhw %%xmm6,%%xmm1\n\t" \
michael@0:   "pmulhw %%xmm7,%%xmm2\n\t" \
michael@0:   "paddw %%xmm6,%%xmm1\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x00,buf)",%%xmm6\n\t" \
michael@0:   "paddw %%xmm7,%%xmm2\n\t" \
michael@0:   "movdqa "OC_MEM_OFFS(0x10,buf)",%%xmm7\n\t" \
michael@0:   /*Stage 3: \
michael@0:     6-5 butterfly: xmm1=t[5], xmm2=t[6] -> xmm1=t[6]+t[5], xmm2=t[6]-t[5] \
michael@0:     0-3 butterfly: xmm4=t[0], xmm7=t[3] -> xmm7=t[0]+t[3], xmm4=t[0]-t[3] \
michael@0:     1-2 butterfly: xmm5=t[1], xmm6=t[2] -> xmm6=t[1]+t[2], xmm5=t[1]-t[2]*/ \
michael@0:   "paddw %%xmm2,%%xmm1\n\t" \
michael@0:   "paddw %%xmm5,%%xmm6\n\t" \
michael@0:   "paddw %%xmm4,%%xmm7\n\t" \
michael@0:   "paddw %%xmm2,%%xmm2\n\t" \
michael@0:   "paddw %%xmm4,%%xmm4\n\t" \
michael@0:   "paddw %%xmm5,%%xmm5\n\t" \
michael@0:   "psubw %%xmm1,%%xmm2\n\t" \
michael@0:   "psubw %%xmm7,%%xmm4\n\t" \
michael@0:   "psubw %%xmm6,%%xmm5\n\t" \
michael@0: 
michael@0: static void oc_idct8x8_10_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64]){
michael@0:   OC_ALIGN16(ogg_int16_t buf[16]);
michael@0:   /*This routine accepts an 8x8 matrix pre-transposed.*/
michael@0:   __asm__ __volatile__(
michael@0:     "movq "OC_MEM_OFFS(0x20,x)",%%mm2\n\t"
michael@0:     "movq "OC_MEM_OFFS(0x30,x)",%%mm3\n\t"
michael@0:     "movq "OC_MEM_OFFS(0x10,x)",%%mm1\n\t"
michael@0:     "movq "OC_MEM_OFFS(0x00,x)",%%mm0\n\t"
michael@0:     OC_IDCT_8x8_10_MMX
michael@0:     OC_TRANSPOSE_8x4_MMX2SSE
michael@0:     OC_IDCT_8x8_10_ABC
michael@0:     OC_IDCT_8x8_D_STORE
michael@0:     :[buf]"=m"(OC_ARRAY_OPERAND(short,buf,16)),
michael@0:      [y]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_y,64))
michael@0:     :[x]"m"OC_CONST_ARRAY_OPERAND(ogg_int16_t,_x,64),
michael@0:      [c]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,OC_IDCT_CONSTS,128))
michael@0:   );
michael@0:   if(_x!=_y){
michael@0:     /*Clear input data for next block (decoder only).*/
michael@0:     __asm__ __volatile__(
michael@0:       "pxor %%mm0,%%mm0\n\t"
michael@0:       "movq %%mm0,"OC_MEM_OFFS(0x00,x)"\n\t"
michael@0:       "movq %%mm0,"OC_MEM_OFFS(0x10,x)"\n\t"
michael@0:       "movq %%mm0,"OC_MEM_OFFS(0x20,x)"\n\t"
michael@0:       "movq %%mm0,"OC_MEM_OFFS(0x30,x)"\n\t"
michael@0:       :[x]"+m"(OC_ARRAY_OPERAND(ogg_int16_t,_x,28))
michael@0:     );
michael@0:   }
michael@0: }
michael@0: 
michael@0: /*Performs an inverse 8x8 Type-II DCT transform.
michael@0:   The input is assumed to be scaled by a factor of 4 relative to orthonormal
michael@0:    version of the transform.*/
michael@0: void oc_idct8x8_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64],int _last_zzi){
michael@0:   /*_last_zzi is subtly different from an actual count of the number of
michael@0:      coefficients we decoded for this block.
michael@0:     It contains the value of zzi BEFORE the final token in the block was
michael@0:      decoded.
michael@0:     In most cases this is an EOB token (the continuation of an EOB run from a
michael@0:      previous block counts), and so this is the same as the coefficient count.
michael@0:     However, in the case that the last token was NOT an EOB token, but filled
michael@0:      the block up with exactly 64 coefficients, _last_zzi will be less than 64.
michael@0:     Provided the last token was not a pure zero run, the minimum value it can
michael@0:      be is 46, and so that doesn't affect any of the cases in this routine.
michael@0:     However, if the last token WAS a pure zero run of length 63, then _last_zzi
michael@0:      will be 1 while the number of coefficients decoded is 64.
michael@0:     Thus, we will trigger the following special case, where the real
michael@0:      coefficient count would not.
michael@0:     Note also that a zero run of length 64 will give _last_zzi a value of 0,
michael@0:      but we still process the DC coefficient, which might have a non-zero value
michael@0:      due to DC prediction.
michael@0:     Although convoluted, this is arguably the correct behavior: it allows us to
michael@0:      use a smaller transform when the block ends with a long zero run instead
michael@0:      of a normal EOB token.
michael@0:     It could be smarter... multiple separate zero runs at the end of a block
michael@0:      will fool it, but an encoder that generates these really deserves what it
michael@0:      gets.
michael@0:     Needless to say we inherited this approach from VP3.*/
michael@0:   /*Then perform the iDCT.*/
michael@0:   if(_last_zzi<=10)oc_idct8x8_10_sse2(_y,_x);
michael@0:   else oc_idct8x8_slow_sse2(_y,_x);
michael@0: }
michael@0: 
michael@0: #endif