michael@0: /********************************************************************
michael@0:  *                                                                  *
michael@0:  * THIS FILE IS PART OF THE OggTheora SOFTWARE CODEC SOURCE CODE.   *
michael@0:  * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
michael@0:  * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
michael@0:  * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
michael@0:  *                                                                  *
michael@0:  * THE Theora SOURCE CODE IS COPYRIGHT (C) 2002-2009                *
michael@0:  * by the Xiph.Org Foundation and contributors http://www.xiph.org/ *
michael@0:  *                                                                  *
michael@0:  ********************************************************************
michael@0: 
michael@0:   function:
michael@0:     last mod: $Id: sse2trans.h 15675 2009-02-06 09:43:27Z tterribe $
michael@0: 
michael@0:  ********************************************************************/
michael@0: 
michael@0: #if !defined(_x86_sse2trans_H)
michael@0: # define _x86_sse2trans_H (1)
michael@0: # include "x86int.h"
michael@0: 
michael@0: # if defined(OC_X86_64_ASM)
michael@0: /*On x86-64 we can transpose in-place without spilling registers.
michael@0:   By clever choices of the order to apply the butterflies and the order of
michael@0:    their outputs, we can take the rows in order and output the columns in order
michael@0:    without any extra operations and using just one temporary register.*/
michael@0: #  define OC_TRANSPOSE_8x8 \
michael@0:  "#OC_TRANSPOSE_8x8\n\t" \
michael@0:  "movdqa %%xmm4,%%xmm8\n\t" \
michael@0:  /*xmm4 = f3 e3 f2 e2 f1 e1 f0 e0*/ \
michael@0:  "punpcklwd %%xmm5,%%xmm4\n\t" \
michael@0:  /*xmm8 = f7 e7 f6 e6 f5 e5 f4 e4*/ \
michael@0:  "punpckhwd %%xmm5,%%xmm8\n\t" \
michael@0:  /*xmm5 is free.*/ \
michael@0:  "movdqa %%xmm0,%%xmm5\n\t" \
michael@0:  /*xmm0 = b3 a3 b2 a2 b1 a1 b0 a0*/ \
michael@0:  "punpcklwd %%xmm1,%%xmm0\n\t" \
michael@0:  /*xmm5 = b7 a7 b6 a6 b5 a5 b4 a4*/ \
michael@0:  "punpckhwd %%xmm1,%%xmm5\n\t" \
michael@0:  /*xmm1 is free.*/ \
michael@0:  "movdqa %%xmm6,%%xmm1\n\t" \
michael@0:  /*xmm6 = h3 g3 h2 g2 h1 g1 h0 g0*/ \
michael@0:  "punpcklwd %%xmm7,%%xmm6\n\t" \
michael@0:  /*xmm1 = h7 g7 h6 g6 h5 g5 h4 g4*/ \
michael@0:  "punpckhwd %%xmm7,%%xmm1\n\t" \
michael@0:  /*xmm7 is free.*/ \
michael@0:  "movdqa %%xmm2,%%xmm7\n\t" \
michael@0:  /*xmm2 = d7 c7 d6 c6 d5 c5 d4 c4*/ \
michael@0:  "punpckhwd %%xmm3,%%xmm2\n\t" \
michael@0:  /*xmm7 = d3 c3 d2 c2 d1 c1 d0 c0*/ \
michael@0:  "punpcklwd %%xmm3,%%xmm7\n\t" \
michael@0:  /*xmm3 is free.*/ \
michael@0:  "movdqa %%xmm0,%%xmm3\n\t" \
michael@0:  /*xmm0 = d1 c1 b1 a1 d0 c0 b0 a0*/ \
michael@0:  "punpckldq %%xmm7,%%xmm0\n\t" \
michael@0:  /*xmm3 = d3 c3 b3 a3 d2 c2 b2 a2*/ \
michael@0:  "punpckhdq %%xmm7,%%xmm3\n\t" \
michael@0:  /*xmm7 is free.*/ \
michael@0:  "movdqa %%xmm5,%%xmm7\n\t" \
michael@0:  /*xmm5 = d5 c5 b5 a5 d4 c4 b4 a4*/ \
michael@0:  "punpckldq %%xmm2,%%xmm5\n\t" \
michael@0:  /*xmm7 = d7 c7 b7 a7 d6 c6 b6 a6*/ \
michael@0:  "punpckhdq %%xmm2,%%xmm7\n\t" \
michael@0:  /*xmm2 is free.*/ \
michael@0:  "movdqa %%xmm4,%%xmm2\n\t" \
michael@0:  /*xmm4 = h3 g3 f3 e3 h2 g2 f2 e2*/ \
michael@0:  "punpckhdq %%xmm6,%%xmm4\n\t" \
michael@0:  /*xmm2 = h1 g1 f1 e1 h0 g0 f0 e0*/ \
michael@0:  "punpckldq %%xmm6,%%xmm2\n\t" \
michael@0:  /*xmm6 is free.*/ \
michael@0:  "movdqa %%xmm8,%%xmm6\n\t" \
michael@0:  /*xmm6 = h5 g5 f5 e5 h4 g4 f4 e4*/ \
michael@0:  "punpckldq %%xmm1,%%xmm6\n\t" \
michael@0:  /*xmm8 = h7 g7 f7 e7 h6 g6 f6 e6*/ \
michael@0:  "punpckhdq %%xmm1,%%xmm8\n\t" \
michael@0:  /*xmm1 is free.*/ \
michael@0:  "movdqa %%xmm0,%%xmm1\n\t" \
michael@0:  /*xmm0 = h0 g0 f0 e0 d0 c0 b0 a0*/ \
michael@0:  "punpcklqdq %%xmm2,%%xmm0\n\t" \
michael@0:  /*xmm1 = h1 g1 f1 e1 d1 c1 b1 a1*/ \
michael@0:  "punpckhqdq %%xmm2,%%xmm1\n\t" \
michael@0:  /*xmm2 is free.*/ \
michael@0:  "movdqa %%xmm3,%%xmm2\n\t" \
michael@0:  /*xmm3 = h3 g3 f3 e3 d3 c3 b3 a3*/ \
michael@0:  "punpckhqdq %%xmm4,%%xmm3\n\t" \
michael@0:  /*xmm2 = h2 g2 f2 e2 d2 c2 b2 a2*/ \
michael@0:  "punpcklqdq %%xmm4,%%xmm2\n\t" \
michael@0:  /*xmm4 is free.*/ \
michael@0:  "movdqa %%xmm5,%%xmm4\n\t" \
michael@0:  /*xmm5 = h5 g5 f5 e5 d5 c5 b5 a5*/ \
michael@0:  "punpckhqdq %%xmm6,%%xmm5\n\t" \
michael@0:  /*xmm4 = h4 g4 f4 e4 d4 c4 b4 a4*/ \
michael@0:  "punpcklqdq %%xmm6,%%xmm4\n\t" \
michael@0:  /*xmm6 is free.*/ \
michael@0:  "movdqa %%xmm7,%%xmm6\n\t" \
michael@0:  /*xmm7 = h7 g7 f7 e7 d7 c7 b7 a7*/ \
michael@0:  "punpckhqdq %%xmm8,%%xmm7\n\t" \
michael@0:  /*xmm6 = h6 g6 f6 e6 d6 c6 b6 a6*/ \
michael@0:  "punpcklqdq %%xmm8,%%xmm6\n\t" \
michael@0:  /*xmm8 is free.*/ \
michael@0: 
michael@0: # else
michael@0: /*Otherwise, we need to spill some values to %[buf] temporarily.
michael@0:   Again, the butterflies are carefully arranged to get the columns to come out
michael@0:    in order, minimizing register spills and maximizing the delay between a load
michael@0:    and when the value loaded is actually used.*/
michael@0: #  define OC_TRANSPOSE_8x8 \
michael@0:  "#OC_TRANSPOSE_8x8\n\t" \
michael@0:  /*buf[0] = a7 a6 a5 a4 a3 a2 a1 a0*/ \
michael@0:  "movdqa %%xmm0,"OC_MEM_OFFS(0x00,buf)"\n\t" \
michael@0:  /*xmm0 is free.*/ \
michael@0:  "movdqa %%xmm2,%%xmm0\n\t" \
michael@0:  /*xmm2 = d7 c7 d6 c6 d5 c5 d4 c4*/ \
michael@0:  "punpckhwd %%xmm3,%%xmm2\n\t" \
michael@0:  /*xmm0 = d3 c3 d2 c2 d1 c1 d0 c0*/ \
michael@0:  "punpcklwd %%xmm3,%%xmm0\n\t" \
michael@0:  /*xmm3 = a7 a6 a5 a4 a3 a2 a1 a0*/ \
michael@0:  "movdqa "OC_MEM_OFFS(0x00,buf)",%%xmm3\n\t" \
michael@0:  /*buf[1] = d7 c7 d6 c6 d5 c5 d4 c4*/ \
michael@0:  "movdqa %%xmm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
michael@0:  /*xmm2 is free.*/ \
michael@0:  "movdqa %%xmm6,%%xmm2\n\t" \
michael@0:  /*xmm6 = h3 g3 h2 g2 h1 g1 h0 g0*/ \
michael@0:  "punpcklwd %%xmm7,%%xmm6\n\t" \
michael@0:  /*xmm2 = h7 g7 h6 g6 h5 g5 h4 g4*/ \
michael@0:  "punpckhwd %%xmm7,%%xmm2\n\t" \
michael@0:  /*xmm7 is free.*/ \
michael@0:  "movdqa %%xmm4,%%xmm7\n\t" \
michael@0:  /*xmm4 = f3 e3 f2 e2 f1 e1 f0 e0*/ \
michael@0:  "punpcklwd %%xmm5,%%xmm4\n\t" \
michael@0:  /*xmm7 = f7 e7 f6 e6 f5 e5 f4 e4*/ \
michael@0:  "punpckhwd %%xmm5,%%xmm7\n\t" \
michael@0:  /*xmm5 is free.*/ \
michael@0:  "movdqa %%xmm3,%%xmm5\n\t" \
michael@0:  /*xmm3 = b3 a3 b2 a2 b1 a1 b0 a0*/ \
michael@0:  "punpcklwd %%xmm1,%%xmm3\n\t" \
michael@0:  /*xmm5 = b7 a7 b6 a6 b5 a5 b4 a4*/ \
michael@0:  "punpckhwd %%xmm1,%%xmm5\n\t" \
michael@0:  /*xmm1 is free.*/ \
michael@0:  "movdqa %%xmm7,%%xmm1\n\t" \
michael@0:  /*xmm7 = h5 g5 f5 e5 h4 g4 f4 e4*/ \
michael@0:  "punpckldq %%xmm2,%%xmm7\n\t" \
michael@0:  /*xmm1 = h7 g7 f7 e7 h6 g6 f6 e6*/ \
michael@0:  "punpckhdq %%xmm2,%%xmm1\n\t" \
michael@0:  /*xmm2 = d7 c7 d6 c6 d5 c5 d4 c4*/ \
michael@0:  "movdqa "OC_MEM_OFFS(0x10,buf)",%%xmm2\n\t" \
michael@0:  /*buf[0] = h7 g7 f7 e7 h6 g6 f6 e6*/ \
michael@0:  "movdqa %%xmm1,"OC_MEM_OFFS(0x00,buf)"\n\t" \
michael@0:  /*xmm1 is free.*/ \
michael@0:  "movdqa %%xmm3,%%xmm1\n\t" \
michael@0:  /*xmm3 = d3 c3 b3 a3 d2 c2 b2 a2*/ \
michael@0:  "punpckhdq %%xmm0,%%xmm3\n\t" \
michael@0:  /*xmm1 = d1 c1 b1 a1 d0 c0 b0 a0*/ \
michael@0:  "punpckldq %%xmm0,%%xmm1\n\t" \
michael@0:  /*xmm0 is free.*/ \
michael@0:  "movdqa %%xmm4,%%xmm0\n\t" \
michael@0:  /*xmm4 = h3 g3 f3 e3 h2 g2 f2 e2*/ \
michael@0:  "punpckhdq %%xmm6,%%xmm4\n\t" \
michael@0:  /*xmm0 = h1 g1 f1 e1 h0 g0 f0 e0*/ \
michael@0:  "punpckldq %%xmm6,%%xmm0\n\t" \
michael@0:  /*xmm6 is free.*/ \
michael@0:  "movdqa %%xmm5,%%xmm6\n\t" \
michael@0:  /*xmm5 = d5 c5 b5 a5 d4 c4 b4 a4*/ \
michael@0:  "punpckldq %%xmm2,%%xmm5\n\t" \
michael@0:  /*xmm6 = d7 c7 b7 a7 d6 c6 b6 a6*/ \
michael@0:  "punpckhdq %%xmm2,%%xmm6\n\t" \
michael@0:  /*xmm2 is free.*/ \
michael@0:  "movdqa %%xmm1,%%xmm2\n\t" \
michael@0:  /*xmm1 = h1 g1 f1 e1 d1 c1 b1 a1*/ \
michael@0:  "punpckhqdq %%xmm0,%%xmm1\n\t" \
michael@0:  /*xmm2 = h0 g0 f0 e0 d0 c0 b0 a0*/ \
michael@0:  "punpcklqdq %%xmm0,%%xmm2\n\t" \
michael@0:  /*xmm0 = h7 g7 f7 e7 h6 g6 f6 e6*/ \
michael@0:  "movdqa "OC_MEM_OFFS(0x00,buf)",%%xmm0\n\t" \
michael@0:  /*buf[1] = h0 g0 f0 e0 d0 c0 b0 a0*/ \
michael@0:  "movdqa %%xmm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
michael@0:  /*xmm2 is free.*/ \
michael@0:  "movdqa %%xmm3,%%xmm2\n\t" \
michael@0:  /*xmm3 = h3 g3 f3 e3 d3 c3 b3 a3*/ \
michael@0:  "punpckhqdq %%xmm4,%%xmm3\n\t" \
michael@0:  /*xmm2 = h2 g2 f2 e2 d2 c2 b2 a2*/ \
michael@0:  "punpcklqdq %%xmm4,%%xmm2\n\t" \
michael@0:  /*xmm4 is free.*/ \
michael@0:  "movdqa %%xmm5,%%xmm4\n\t" \
michael@0:  /*xmm5 = h5 g5 f5 e5 d5 c5 b5 a5*/ \
michael@0:  "punpckhqdq %%xmm7,%%xmm5\n\t" \
michael@0:  /*xmm4 = h4 g4 f4 e4 d4 c4 b4 a4*/ \
michael@0:  "punpcklqdq %%xmm7,%%xmm4\n\t" \
michael@0:  /*xmm7 is free.*/ \
michael@0:  "movdqa %%xmm6,%%xmm7\n\t" \
michael@0:  /*xmm6 = h6 g6 f6 e6 d6 c6 b6 a6*/ \
michael@0:  "punpcklqdq %%xmm0,%%xmm6\n\t" \
michael@0:  /*xmm7 = h7 g7 f7 e7 d7 c7 b7 a7*/ \
michael@0:  "punpckhqdq %%xmm0,%%xmm7\n\t" \
michael@0:  /*xmm0 = h0 g0 f0 e0 d0 c0 b0 a0*/ \
michael@0:  "movdqa "OC_MEM_OFFS(0x10,buf)",%%xmm0\n\t" \
michael@0: 
michael@0: # endif
michael@0: 
michael@0: /*Transpose 4 values in each of 8 MMX registers into 8 values in the first
michael@0:    four SSE registers.
michael@0:   No need to be clever here; we have plenty of room.*/
michael@0: #  define OC_TRANSPOSE_8x4_MMX2SSE \
michael@0:  "#OC_TRANSPOSE_8x4_MMX2SSE\n\t" \
michael@0:  "movq2dq %%mm0,%%xmm0\n\t" \
michael@0:  "movq2dq %%mm1,%%xmm1\n\t" \
michael@0:  /*xmmA = b3 a3 b2 a2 b1 a1 b0 a0*/ \
michael@0:  "punpcklwd %%xmm1,%%xmm0\n\t" \
michael@0:  "movq2dq %%mm2,%%xmm3\n\t" \
michael@0:  "movq2dq %%mm3,%%xmm2\n\t" \
michael@0:  /*xmmC = d3 c3 d2 c2 d1 c1 d0 c0*/ \
michael@0:  "punpcklwd %%xmm2,%%xmm3\n\t" \
michael@0:  "movq2dq %%mm4,%%xmm4\n\t" \
michael@0:  "movq2dq %%mm5,%%xmm5\n\t" \
michael@0:  /*xmmE = f3 e3 f2 e2 f1 e1 f0 e0*/ \
michael@0:  "punpcklwd %%xmm5,%%xmm4\n\t" \
michael@0:  "movq2dq %%mm6,%%xmm7\n\t" \
michael@0:  "movq2dq %%mm7,%%xmm6\n\t" \
michael@0:  /*xmmG = h3 g3 h2 g2 h1 g1 h0 g0*/ \
michael@0:  "punpcklwd %%xmm6,%%xmm7\n\t" \
michael@0:  "movdqa %%xmm0,%%xmm2\n\t" \
michael@0:  /*xmm0 = d1 c1 b1 a1 d0 c0 b0 a0*/ \
michael@0:  "punpckldq %%xmm3,%%xmm0\n\t" \
michael@0:  /*xmm2 = d3 c3 b3 a3 d2 c2 b2 a2*/ \
michael@0:  "punpckhdq %%xmm3,%%xmm2\n\t" \
michael@0:  "movdqa %%xmm4,%%xmm5\n\t" \
michael@0:  /*xmm4 = h1 g1 f1 e1 h0 g0 f0 e0*/ \
michael@0:  "punpckldq %%xmm7,%%xmm4\n\t" \
michael@0:  /*xmm3 = h3 g3 f3 e3 h2 g2 f2 e2*/ \
michael@0:  "punpckhdq %%xmm7,%%xmm5\n\t" \
michael@0:  "movdqa %%xmm0,%%xmm1\n\t" \
michael@0:  /*xmm0 = h0 g0 f0 e0 d0 c0 b0 a0*/ \
michael@0:  "punpcklqdq %%xmm4,%%xmm0\n\t" \
michael@0:  /*xmm1 = h1 g1 f1 e1 d1 c1 b1 a1*/ \
michael@0:  "punpckhqdq %%xmm4,%%xmm1\n\t" \
michael@0:  "movdqa %%xmm2,%%xmm3\n\t" \
michael@0:  /*xmm2 = h2 g2 f2 e2 d2 c2 b2 a2*/ \
michael@0:  "punpcklqdq %%xmm5,%%xmm2\n\t" \
michael@0:  /*xmm3 = h3 g3 f3 e3 d3 c3 b3 a3*/ \
michael@0:  "punpckhqdq %%xmm5,%%xmm3\n\t" \
michael@0: 
michael@0: #endif