michael@0: /********************************************************************
michael@0:  *                                                                  *
michael@0:  * THIS FILE IS PART OF THE OggTheora SOFTWARE CODEC SOURCE CODE.   *
michael@0:  * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
michael@0:  * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
michael@0:  * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
michael@0:  *                                                                  *
michael@0:  * THE Theora SOURCE CODE IS COPYRIGHT (C) 2002-2009                *
michael@0:  * by the Xiph.Org Foundation and contributors http://www.xiph.org/ *
michael@0:  *                                                                  *
michael@0:  ********************************************************************
michael@0: 
michael@0:   function:
michael@0:     last mod: $Id: mmxfrag.c 17446 2010-09-23 20:06:20Z tterribe $
michael@0: 
michael@0:  ********************************************************************/
michael@0: 
michael@0: /*MMX acceleration of fragment reconstruction for motion compensation.
michael@0:   Originally written by Rudolf Marek.
michael@0:   Additional optimization by Nils Pipenbrinck.
michael@0:   Note: Loops are unrolled for best performance.
michael@0:   The iteration each instruction belongs to is marked in the comments as #i.*/
michael@0: #include <stddef.h>
michael@0: #include "x86int.h"
michael@0: 
michael@0: #if defined(OC_X86_ASM)
michael@0: 
michael@0: /*Copies an 8x8 block of pixels from _src to _dst, assuming _ystride bytes
michael@0:    between rows.*/
michael@0: # define OC_FRAG_COPY_MMX(_dst,_src,_ystride) \
michael@0:   do{ \
michael@0:     const unsigned char *src; \
michael@0:     unsigned char       *dst; \
michael@0:     src=(_src); \
michael@0:     dst=(_dst); \
michael@0:     __asm  mov SRC,src \
michael@0:     __asm  mov DST,dst \
michael@0:     __asm  mov YSTRIDE,_ystride \
michael@0:     /*src+0*ystride*/ \
michael@0:     __asm  movq mm0,[SRC] \
michael@0:     /*src+1*ystride*/ \
michael@0:     __asm  movq mm1,[SRC+YSTRIDE] \
michael@0:     /*ystride3=ystride*3*/ \
michael@0:     __asm  lea YSTRIDE3,[YSTRIDE+YSTRIDE*2] \
michael@0:     /*src+2*ystride*/ \
michael@0:     __asm  movq mm2,[SRC+YSTRIDE*2] \
michael@0:     /*src+3*ystride*/ \
michael@0:     __asm  movq mm3,[SRC+YSTRIDE3] \
michael@0:     /*dst+0*ystride*/ \
michael@0:     __asm  movq [DST],mm0 \
michael@0:     /*dst+1*ystride*/ \
michael@0:     __asm  movq [DST+YSTRIDE],mm1 \
michael@0:     /*Pointer to next 4.*/ \
michael@0:     __asm  lea SRC,[SRC+YSTRIDE*4] \
michael@0:     /*dst+2*ystride*/ \
michael@0:     __asm  movq [DST+YSTRIDE*2],mm2 \
michael@0:     /*dst+3*ystride*/ \
michael@0:     __asm  movq [DST+YSTRIDE3],mm3 \
michael@0:     /*Pointer to next 4.*/ \
michael@0:     __asm  lea DST,[DST+YSTRIDE*4] \
michael@0:     /*src+0*ystride*/ \
michael@0:     __asm  movq mm0,[SRC] \
michael@0:     /*src+1*ystride*/ \
michael@0:     __asm  movq mm1,[SRC+YSTRIDE] \
michael@0:     /*src+2*ystride*/ \
michael@0:     __asm  movq mm2,[SRC+YSTRIDE*2] \
michael@0:     /*src+3*ystride*/ \
michael@0:     __asm  movq mm3,[SRC+YSTRIDE3] \
michael@0:     /*dst+0*ystride*/ \
michael@0:     __asm  movq [DST],mm0 \
michael@0:     /*dst+1*ystride*/ \
michael@0:     __asm  movq [DST+YSTRIDE],mm1 \
michael@0:     /*dst+2*ystride*/ \
michael@0:     __asm  movq [DST+YSTRIDE*2],mm2 \
michael@0:     /*dst+3*ystride*/ \
michael@0:     __asm  movq [DST+YSTRIDE3],mm3 \
michael@0:   } \
michael@0:   while(0)
michael@0: 
michael@0: /*Copies an 8x8 block of pixels from _src to _dst, assuming _ystride bytes
michael@0:    between rows.*/
michael@0: void oc_frag_copy_mmx(unsigned char *_dst,
michael@0:  const unsigned char *_src,int _ystride){
michael@0: #define SRC edx
michael@0: #define DST eax
michael@0: #define YSTRIDE ecx
michael@0: #define YSTRIDE3 esi
michael@0:   OC_FRAG_COPY_MMX(_dst,_src,_ystride);
michael@0: #undef SRC
michael@0: #undef DST
michael@0: #undef YSTRIDE
michael@0: #undef YSTRIDE3
michael@0: }
michael@0: 
michael@0: /*Copies the fragments specified by the lists of fragment indices from one
michael@0:    frame to another.
michael@0:   _dst_frame:     The reference frame to copy to.
michael@0:   _src_frame:     The reference frame to copy from.
michael@0:   _ystride:       The row stride of the reference frames.
michael@0:   _fragis:        A pointer to a list of fragment indices.
michael@0:   _nfragis:       The number of fragment indices to copy.
michael@0:   _frag_buf_offs: The offsets of fragments in the reference frames.*/
michael@0: void oc_frag_copy_list_mmx(unsigned char *_dst_frame,
michael@0:  const unsigned char *_src_frame,int _ystride,
michael@0:  const ptrdiff_t *_fragis,ptrdiff_t _nfragis,const ptrdiff_t *_frag_buf_offs){
michael@0:   ptrdiff_t fragii;
michael@0:   for(fragii=0;fragii<_nfragis;fragii++){
michael@0:     ptrdiff_t frag_buf_off;
michael@0:     frag_buf_off=_frag_buf_offs[_fragis[fragii]];
michael@0: #define SRC edx
michael@0: #define DST eax
michael@0: #define YSTRIDE ecx
michael@0: #define YSTRIDE3 edi
michael@0:     OC_FRAG_COPY_MMX(_dst_frame+frag_buf_off,
michael@0:      _src_frame+frag_buf_off,_ystride);
michael@0: #undef SRC
michael@0: #undef DST
michael@0: #undef YSTRIDE
michael@0: #undef YSTRIDE3
michael@0:   }
michael@0: }
michael@0: 
michael@0: void oc_frag_recon_intra_mmx(unsigned char *_dst,int _ystride,
michael@0:  const ogg_int16_t *_residue){
michael@0:   __asm{
michael@0: #define DST edx
michael@0: #define DST4 esi
michael@0: #define YSTRIDE eax
michael@0: #define YSTRIDE3 edi
michael@0: #define RESIDUE ecx
michael@0:     mov DST,_dst
michael@0:     mov YSTRIDE,_ystride
michael@0:     mov RESIDUE,_residue
michael@0:     lea DST4,[DST+YSTRIDE*4]
michael@0:     lea YSTRIDE3,[YSTRIDE+YSTRIDE*2]
michael@0:     /*Set mm0 to 0xFFFFFFFFFFFFFFFF.*/
michael@0:     pcmpeqw mm0,mm0
michael@0:     /*#0 Load low residue.*/
michael@0:     movq mm1,[0*8+RESIDUE]
michael@0:     /*#0 Load high residue.*/
michael@0:     movq mm2,[1*8+RESIDUE]
michael@0:     /*Set mm0 to 0x8000800080008000.*/
michael@0:     psllw mm0,15
michael@0:     /*#1 Load low residue.*/
michael@0:     movq mm3,[2*8+RESIDUE]
michael@0:     /*#1 Load high residue.*/
michael@0:     movq mm4,[3*8+RESIDUE]
michael@0:     /*Set mm0 to 0x0080008000800080.*/
michael@0:     psrlw mm0,8
michael@0:     /*#2 Load low residue.*/
michael@0:     movq mm5,[4*8+RESIDUE]
michael@0:     /*#2 Load high residue.*/
michael@0:     movq mm6,[5*8+RESIDUE]
michael@0:     /*#0 Bias low  residue.*/
michael@0:     paddsw mm1,mm0
michael@0:     /*#0 Bias high residue.*/
michael@0:     paddsw mm2,mm0
michael@0:     /*#0 Pack to byte.*/
michael@0:     packuswb mm1,mm2
michael@0:     /*#1 Bias low  residue.*/
michael@0:     paddsw mm3,mm0
michael@0:     /*#1 Bias high residue.*/
michael@0:     paddsw mm4,mm0
michael@0:     /*#1 Pack to byte.*/
michael@0:     packuswb mm3,mm4
michael@0:     /*#2 Bias low  residue.*/
michael@0:     paddsw mm5,mm0
michael@0:     /*#2 Bias high residue.*/
michael@0:     paddsw mm6,mm0
michael@0:     /*#2 Pack to byte.*/
michael@0:     packuswb mm5,mm6
michael@0:     /*#0 Write row.*/
michael@0:     movq [DST],mm1
michael@0:     /*#1 Write row.*/
michael@0:     movq [DST+YSTRIDE],mm3
michael@0:     /*#2 Write row.*/
michael@0:     movq [DST+YSTRIDE*2],mm5
michael@0:     /*#3 Load low residue.*/
michael@0:     movq mm1,[6*8+RESIDUE]
michael@0:     /*#3 Load high residue.*/
michael@0:     movq mm2,[7*8+RESIDUE]
michael@0:     /*#4 Load high residue.*/
michael@0:     movq mm3,[8*8+RESIDUE]
michael@0:     /*#4 Load high residue.*/
michael@0:     movq mm4,[9*8+RESIDUE]
michael@0:     /*#5 Load high residue.*/
michael@0:     movq mm5,[10*8+RESIDUE]
michael@0:     /*#5 Load high residue.*/
michael@0:     movq mm6,[11*8+RESIDUE]
michael@0:     /*#3 Bias low  residue.*/
michael@0:     paddsw mm1,mm0
michael@0:     /*#3 Bias high residue.*/
michael@0:     paddsw mm2,mm0
michael@0:     /*#3 Pack to byte.*/
michael@0:     packuswb mm1,mm2
michael@0:     /*#4 Bias low  residue.*/
michael@0:     paddsw mm3,mm0
michael@0:     /*#4 Bias high residue.*/
michael@0:     paddsw mm4,mm0
michael@0:     /*#4 Pack to byte.*/
michael@0:     packuswb mm3,mm4
michael@0:     /*#5 Bias low  residue.*/
michael@0:     paddsw mm5,mm0
michael@0:     /*#5 Bias high residue.*/
michael@0:     paddsw mm6,mm0
michael@0:     /*#5 Pack to byte.*/
michael@0:     packuswb mm5,mm6
michael@0:     /*#3 Write row.*/
michael@0:     movq [DST+YSTRIDE3],mm1
michael@0:     /*#4 Write row.*/
michael@0:     movq [DST4],mm3
michael@0:     /*#5 Write row.*/
michael@0:     movq [DST4+YSTRIDE],mm5
michael@0:     /*#6 Load low residue.*/
michael@0:     movq mm1,[12*8+RESIDUE]
michael@0:     /*#6 Load high residue.*/
michael@0:     movq mm2,[13*8+RESIDUE]
michael@0:     /*#7 Load low residue.*/
michael@0:     movq mm3,[14*8+RESIDUE]
michael@0:     /*#7 Load high residue.*/
michael@0:     movq mm4,[15*8+RESIDUE]
michael@0:     /*#6 Bias low  residue.*/
michael@0:     paddsw mm1,mm0
michael@0:     /*#6 Bias high residue.*/
michael@0:     paddsw mm2,mm0
michael@0:     /*#6 Pack to byte.*/
michael@0:     packuswb mm1,mm2
michael@0:     /*#7 Bias low  residue.*/
michael@0:     paddsw mm3,mm0
michael@0:     /*#7 Bias high residue.*/
michael@0:     paddsw mm4,mm0
michael@0:     /*#7 Pack to byte.*/
michael@0:     packuswb mm3,mm4
michael@0:     /*#6 Write row.*/
michael@0:     movq [DST4+YSTRIDE*2],mm1
michael@0:     /*#7 Write row.*/
michael@0:     movq [DST4+YSTRIDE3],mm3
michael@0: #undef DST
michael@0: #undef DST4
michael@0: #undef YSTRIDE
michael@0: #undef YSTRIDE3
michael@0: #undef RESIDUE
michael@0:   }
michael@0: }
michael@0: 
michael@0: void oc_frag_recon_inter_mmx(unsigned char *_dst,const unsigned char *_src,
michael@0:  int _ystride,const ogg_int16_t *_residue){
michael@0:   int i;
michael@0:   /*Zero mm0.*/
michael@0:   __asm pxor mm0,mm0;
michael@0:   for(i=4;i-->0;){
michael@0:     __asm{
michael@0: #define DST edx
michael@0: #define SRC ecx
michael@0: #define YSTRIDE edi
michael@0: #define RESIDUE eax
michael@0:       mov DST,_dst
michael@0:       mov SRC,_src
michael@0:       mov YSTRIDE,_ystride
michael@0:       mov RESIDUE,_residue
michael@0:       /*#0 Load source.*/
michael@0:       movq mm3,[SRC]
michael@0:       /*#1 Load source.*/
michael@0:       movq mm7,[SRC+YSTRIDE]
michael@0:       /*#0 Get copy of src.*/
michael@0:       movq mm4,mm3
michael@0:       /*#0 Expand high source.*/
michael@0:       punpckhbw mm4,mm0
michael@0:       /*#0 Expand low  source.*/
michael@0:       punpcklbw mm3,mm0
michael@0:       /*#0 Add residue high.*/
michael@0:       paddsw mm4,[8+RESIDUE]
michael@0:       /*#1 Get copy of src.*/
michael@0:       movq mm2,mm7
michael@0:       /*#0 Add residue low.*/
michael@0:       paddsw  mm3,[RESIDUE]
michael@0:       /*#1 Expand high source.*/
michael@0:       punpckhbw mm2,mm0
michael@0:       /*#0 Pack final row pixels.*/
michael@0:       packuswb mm3,mm4
michael@0:       /*#1 Expand low  source.*/
michael@0:       punpcklbw mm7,mm0
michael@0:       /*#1 Add residue low.*/
michael@0:       paddsw mm7,[16+RESIDUE]
michael@0:       /*#1 Add residue high.*/
michael@0:       paddsw mm2,[24+RESIDUE]
michael@0:       /*Advance residue.*/
michael@0:       lea RESIDUE,[32+RESIDUE]
michael@0:       /*#1 Pack final row pixels.*/
michael@0:       packuswb mm7,mm2
michael@0:       /*Advance src.*/
michael@0:       lea SRC,[SRC+YSTRIDE*2]
michael@0:       /*#0 Write row.*/
michael@0:       movq [DST],mm3
michael@0:       /*#1 Write row.*/
michael@0:       movq [DST+YSTRIDE],mm7
michael@0:       /*Advance dst.*/
michael@0:       lea DST,[DST+YSTRIDE*2]
michael@0:       mov _residue,RESIDUE
michael@0:       mov _dst,DST
michael@0:       mov _src,SRC
michael@0: #undef DST
michael@0: #undef SRC
michael@0: #undef YSTRIDE
michael@0: #undef RESIDUE
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: void oc_frag_recon_inter2_mmx(unsigned char *_dst,const unsigned char *_src1,
michael@0:  const unsigned char *_src2,int _ystride,const ogg_int16_t *_residue){
michael@0:   int i;
michael@0:   /*Zero mm7.*/
michael@0:   __asm pxor mm7,mm7;
michael@0:   for(i=4;i-->0;){
michael@0:     __asm{
michael@0: #define SRC1 ecx
michael@0: #define SRC2 edi
michael@0: #define YSTRIDE esi
michael@0: #define RESIDUE edx
michael@0: #define DST eax
michael@0:       mov YSTRIDE,_ystride
michael@0:       mov DST,_dst
michael@0:       mov RESIDUE,_residue
michael@0:       mov SRC1,_src1
michael@0:       mov SRC2,_src2
michael@0:       /*#0 Load src1.*/
michael@0:       movq mm0,[SRC1]
michael@0:       /*#0 Load src2.*/
michael@0:       movq mm2,[SRC2]
michael@0:       /*#0 Copy src1.*/
michael@0:       movq mm1,mm0
michael@0:       /*#0 Copy src2.*/
michael@0:       movq mm3,mm2
michael@0:       /*#1 Load src1.*/
michael@0:       movq mm4,[SRC1+YSTRIDE]
michael@0:       /*#0 Unpack lower src1.*/
michael@0:       punpcklbw mm0,mm7
michael@0:       /*#1 Load src2.*/
michael@0:       movq mm5,[SRC2+YSTRIDE]
michael@0:       /*#0 Unpack higher src1.*/
michael@0:       punpckhbw mm1,mm7
michael@0:       /*#0 Unpack lower src2.*/
michael@0:       punpcklbw mm2,mm7
michael@0:       /*#0 Unpack higher src2.*/
michael@0:       punpckhbw mm3,mm7
michael@0:       /*Advance src1 ptr.*/
michael@0:       lea SRC1,[SRC1+YSTRIDE*2]
michael@0:       /*Advance src2 ptr.*/
michael@0:       lea SRC2,[SRC2+YSTRIDE*2]
michael@0:       /*#0 Lower src1+src2.*/
michael@0:       paddsw mm0,mm2
michael@0:       /*#0 Higher src1+src2.*/
michael@0:       paddsw mm1,mm3
michael@0:       /*#1 Copy src1.*/
michael@0:       movq mm2,mm4
michael@0:       /*#0 Build lo average.*/
michael@0:       psraw mm0,1
michael@0:       /*#1 Copy src2.*/
michael@0:       movq mm3,mm5
michael@0:       /*#1 Unpack lower src1.*/
michael@0:       punpcklbw mm4,mm7
michael@0:       /*#0 Build hi average.*/
michael@0:       psraw mm1,1
michael@0:       /*#1 Unpack higher src1.*/
michael@0:       punpckhbw mm2,mm7
michael@0:       /*#0 low+=residue.*/
michael@0:       paddsw mm0,[RESIDUE]
michael@0:       /*#1 Unpack lower src2.*/
michael@0:       punpcklbw mm5,mm7
michael@0:       /*#0 high+=residue.*/
michael@0:       paddsw mm1,[8+RESIDUE]
michael@0:       /*#1 Unpack higher src2.*/
michael@0:       punpckhbw mm3,mm7
michael@0:       /*#1 Lower src1+src2.*/
michael@0:       paddsw mm5,mm4
michael@0:       /*#0 Pack and saturate.*/
michael@0:       packuswb mm0,mm1
michael@0:       /*#1 Higher src1+src2.*/
michael@0:       paddsw mm3,mm2
michael@0:       /*#0 Write row.*/
michael@0:       movq [DST],mm0
michael@0:       /*#1 Build lo average.*/
michael@0:       psraw mm5,1
michael@0:       /*#1 Build hi average.*/
michael@0:       psraw mm3,1
michael@0:       /*#1 low+=residue.*/
michael@0:       paddsw mm5,[16+RESIDUE]
michael@0:       /*#1 high+=residue.*/
michael@0:       paddsw mm3,[24+RESIDUE]
michael@0:       /*#1 Pack and saturate.*/
michael@0:       packuswb  mm5,mm3
michael@0:       /*#1 Write row ptr.*/
michael@0:       movq [DST+YSTRIDE],mm5
michael@0:       /*Advance residue ptr.*/
michael@0:       add RESIDUE,32
michael@0:       /*Advance dest ptr.*/
michael@0:       lea DST,[DST+YSTRIDE*2]
michael@0:       mov _dst,DST
michael@0:       mov _residue,RESIDUE
michael@0:       mov _src1,SRC1
michael@0:       mov _src2,SRC2
michael@0: #undef SRC1
michael@0: #undef SRC2
michael@0: #undef YSTRIDE
michael@0: #undef RESIDUE
michael@0: #undef DST
michael@0:     }
michael@0:   }
michael@0: }
michael@0: 
michael@0: void oc_restore_fpu_mmx(void){
michael@0:   __asm emms;
michael@0: }
michael@0: 
michael@0: #endif