The Tor Browser / file comparison
comparison: media/libtheora/lib/x86/mmxloop.h
media/libtheora/lib/x86/mmxloop.h
- branch
- TOR_BUG_9701
- changeset 15
- b8a032363ba2
equal
deleted
inserted
replaced
| -1:000000000000 | 0:a5ca6f5e2f17 |
|---|---|
| 1 #if !defined(_x86_mmxloop_H) | |
| 2 # define _x86_mmxloop_H (1) | |
| 3 # include <stddef.h> | |
| 4 # include "x86int.h" | |
| 5 | |
| 6 #if defined(OC_X86_ASM) | |
| 7 | |
| 8 /*On entry, mm0={a0,...,a7}, mm1={b0,...,b7}, mm2={c0,...,c7}, mm3={d0,...d7}. | |
| 9 On exit, mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)} and | |
| 10 mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}; mm0 and mm3 are clobbered.*/ | |
| 11 #define OC_LOOP_FILTER8_MMX \ | |
| 12 "#OC_LOOP_FILTER8_MMX\n\t" \ | |
| 13 /*mm7=0*/ \ | |
| 14 "pxor %%mm7,%%mm7\n\t" \ | |
| 15 /*mm6:mm0={a0,...,a7}*/ \ | |
| 16 "movq %%mm0,%%mm6\n\t" \ | |
| 17 "punpcklbw %%mm7,%%mm0\n\t" \ | |
| 18 "punpckhbw %%mm7,%%mm6\n\t" \ | |
| 19 /*mm3:mm5={d0,...,d7}*/ \ | |
| 20 "movq %%mm3,%%mm5\n\t" \ | |
| 21 "punpcklbw %%mm7,%%mm3\n\t" \ | |
| 22 "punpckhbw %%mm7,%%mm5\n\t" \ | |
| 23 /*mm6:mm0={a0-d0,...,a7-d7}*/ \ | |
| 24 "psubw %%mm3,%%mm0\n\t" \ | |
| 25 "psubw %%mm5,%%mm6\n\t" \ | |
| 26 /*mm3:mm1={b0,...,b7}*/ \ | |
| 27 "movq %%mm1,%%mm3\n\t" \ | |
| 28 "punpcklbw %%mm7,%%mm1\n\t" \ | |
| 29 "movq %%mm2,%%mm4\n\t" \ | |
| 30 "punpckhbw %%mm7,%%mm3\n\t" \ | |
| 31 /*mm5:mm4={c0,...,c7}*/ \ | |
| 32 "movq %%mm2,%%mm5\n\t" \ | |
| 33 "punpcklbw %%mm7,%%mm4\n\t" \ | |
| 34 "punpckhbw %%mm7,%%mm5\n\t" \ | |
| 35 /*mm7={3}x4 \ | |
| 36 mm5:mm4={c0-b0,...,c7-b7}*/ \ | |
| 37 "pcmpeqw %%mm7,%%mm7\n\t" \ | |
| 38 "psubw %%mm1,%%mm4\n\t" \ | |
| 39 "psrlw $14,%%mm7\n\t" \ | |
| 40 "psubw %%mm3,%%mm5\n\t" \ | |
| 41 /*Scale by 3.*/ \ | |
| 42 "pmullw %%mm7,%%mm4\n\t" \ | |
| 43 "pmullw %%mm7,%%mm5\n\t" \ | |
| 44 /*mm7={4}x4 \ | |
| 45 mm5:mm4=f={a0-d0+3*(c0-b0),...,a7-d7+3*(c7-b7)}*/ \ | |
| 46 "psrlw $1,%%mm7\n\t" \ | |
| 47 "paddw %%mm0,%%mm4\n\t" \ | |
| 48 "psllw $2,%%mm7\n\t" \ | |
| 49 "movq (%[ll]),%%mm0\n\t" \ | |
| 50 "paddw %%mm6,%%mm5\n\t" \ | |
| 51 /*R_i has the range [-127,128], so we compute -R_i instead. \ | |
| 52 mm4=-R_i=-(f+4>>3)=0xFF^(f-4>>3)*/ \ | |
| 53 "psubw %%mm7,%%mm4\n\t" \ | |
| 54 "psubw %%mm7,%%mm5\n\t" \ | |
| 55 "psraw $3,%%mm4\n\t" \ | |
| 56 "psraw $3,%%mm5\n\t" \ | |
| 57 "pcmpeqb %%mm7,%%mm7\n\t" \ | |
| 58 "packsswb %%mm5,%%mm4\n\t" \ | |
| 59 "pxor %%mm6,%%mm6\n\t" \ | |
| 60 "pxor %%mm7,%%mm4\n\t" \ | |
| 61 "packuswb %%mm3,%%mm1\n\t" \ | |
| 62 /*Now compute lflim of -mm4 cf. Section 7.10 of the sepc.*/ \ | |
| 63 /*There's no unsigned byte+signed byte with unsigned saturation op code, so \ | |
| 64 we have to split things by sign (the other option is to work in 16 bits, \ | |
| 65 but working in 8 bits gives much better parallelism). \ | |
| 66 We compute abs(R_i), but save a mask of which terms were negative in mm6. \ | |
| 67 Then we compute mm4=abs(lflim(R_i,L))=min(abs(R_i),max(2*L-abs(R_i),0)). \ | |
| 68 Finally, we split mm4 into positive and negative pieces using the mask in \ | |
| 69 mm6, and add and subtract them as appropriate.*/ \ | |
| 70 /*mm4=abs(-R_i)*/ \ | |
| 71 /*mm7=255-2*L*/ \ | |
| 72 "pcmpgtb %%mm4,%%mm6\n\t" \ | |
| 73 "psubb %%mm0,%%mm7\n\t" \ | |
| 74 "pxor %%mm6,%%mm4\n\t" \ | |
| 75 "psubb %%mm0,%%mm7\n\t" \ | |
| 76 "psubb %%mm6,%%mm4\n\t" \ | |
| 77 /*mm7=255-max(2*L-abs(R_i),0)*/ \ | |
| 78 "paddusb %%mm4,%%mm7\n\t" \ | |
| 79 /*mm4=min(abs(R_i),max(2*L-abs(R_i),0))*/ \ | |
| 80 "paddusb %%mm7,%%mm4\n\t" \ | |
| 81 "psubusb %%mm7,%%mm4\n\t" \ | |
| 82 /*Now split mm4 by the original sign of -R_i.*/ \ | |
| 83 "movq %%mm4,%%mm5\n\t" \ | |
| 84 "pand %%mm6,%%mm4\n\t" \ | |
| 85 "pandn %%mm5,%%mm6\n\t" \ | |
| 86 /*mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)}*/ \ | |
| 87 /*mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}*/ \ | |
| 88 "paddusb %%mm4,%%mm1\n\t" \ | |
| 89 "psubusb %%mm4,%%mm2\n\t" \ | |
| 90 "psubusb %%mm6,%%mm1\n\t" \ | |
| 91 "paddusb %%mm6,%%mm2\n\t" \ | |
| 92 | |
| 93 /*On entry, mm0={a0,...,a7}, mm1={b0,...,b7}, mm2={c0,...,c7}, mm3={d0,...d7}. | |
| 94 On exit, mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)} and | |
| 95 mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}. | |
| 96 All other MMX registers are clobbered.*/ | |
| 97 #define OC_LOOP_FILTER8_MMXEXT \ | |
| 98 "#OC_LOOP_FILTER8_MMXEXT\n\t" \ | |
| 99 /*R_i=(a_i-3*b_i+3*c_i-d_i+4>>3) has the range [-127,128], so we compute \ | |
| 100 -R_i=(-a_i+3*b_i-3*c_i+d_i+3>>3) instead.*/ \ | |
| 101 /*This first part is based on the transformation \ | |
| 102 f = -(3*(c-b)+a-d+4>>3) \ | |
| 103 = -(3*(c+255-b)+(a+255-d)+4-1020>>3) \ | |
| 104 = -(3*(c+~b)+(a+~d)-1016>>3) \ | |
| 105 = 127-(3*(c+~b)+(a+~d)>>3) \ | |
| 106 = 128+~(3*(c+~b)+(a+~d)>>3) (mod 256). \ | |
| 107 Although pavgb(a,b) = (a+b+1>>1) (biased up), we rely heavily on the \ | |
| 108 fact that ~pavgb(~a,~b) = (a+b>>1) (biased down). \ | |
| 109 Using this, the last expression above can be computed in 8 bits of working \ | |
| 110 precision via: \ | |
| 111 u = ~pavgb(~b,c); \ | |
| 112 v = pavgb(b,~c); \ | |
| 113 This mask is 0 or 0xFF, and controls whether t is biased up or down: \ | |
| 114 m = u-v; \ | |
| 115 t = m^pavgb(m^~a,m^d); \ | |
| 116 f = 128+pavgb(pavgb(t,u),v); \ | |
| 117 This required some careful analysis to ensure that carries are propagated \ | |
| 118 correctly in all cases, but has been checked exhaustively.*/ \ | |
| 119 /*input (a, b, c, d, ., ., ., .)*/ \ | |
| 120 /*ff=0xFF; \ | |
| 121 u=b; \ | |
| 122 v=c; \ | |
| 123 ll=255-2*L;*/ \ | |
| 124 "pcmpeqb %%mm7,%%mm7\n\t" \ | |
| 125 "movq %%mm1,%%mm4\n\t" \ | |
| 126 "movq %%mm2,%%mm5\n\t" \ | |
| 127 "movq (%[ll]),%%mm6\n\t" \ | |
| 128 /*allocated u, v, ll, ff: (a, b, c, d, u, v, ll, ff)*/ \ | |
| 129 /*u^=ff; \ | |
| 130 v^=ff;*/ \ | |
| 131 "pxor %%mm7,%%mm4\n\t" \ | |
| 132 "pxor %%mm7,%%mm5\n\t" \ | |
| 133 /*allocated ll: (a, b, c, d, u, v, ll, ff)*/ \ | |
| 134 /*u=pavgb(u,c); \ | |
| 135 v=pavgb(v,b);*/ \ | |
| 136 "pavgb %%mm2,%%mm4\n\t" \ | |
| 137 "pavgb %%mm1,%%mm5\n\t" \ | |
| 138 /*u^=ff; \ | |
| 139 a^=ff;*/ \ | |
| 140 "pxor %%mm7,%%mm4\n\t" \ | |
| 141 "pxor %%mm7,%%mm0\n\t" \ | |
| 142 /*m=u-v;*/ \ | |
| 143 "psubb %%mm5,%%mm4\n\t" \ | |
| 144 /*freed u, allocated m: (a, b, c, d, m, v, ll, ff)*/ \ | |
| 145 /*a^=m; \ | |
| 146 d^=m;*/ \ | |
| 147 "pxor %%mm4,%%mm0\n\t" \ | |
| 148 "pxor %%mm4,%%mm3\n\t" \ | |
| 149 /*t=pavgb(a,d);*/ \ | |
| 150 "pavgb %%mm3,%%mm0\n\t" \ | |
| 151 "psllw $7,%%mm7\n\t" \ | |
| 152 /*freed a, d, ff, allocated t, of: (t, b, c, ., m, v, ll, of)*/ \ | |
| 153 /*t^=m; \ | |
| 154 u=m+v;*/ \ | |
| 155 "pxor %%mm4,%%mm0\n\t" \ | |
| 156 "paddb %%mm5,%%mm4\n\t" \ | |
| 157 /*freed t, m, allocated f, u: (f, b, c, ., u, v, ll, of)*/ \ | |
| 158 /*f=pavgb(f,u); \ | |
| 159 of=128;*/ \ | |
| 160 "pavgb %%mm4,%%mm0\n\t" \ | |
| 161 "packsswb %%mm7,%%mm7\n\t" \ | |
| 162 /*freed u, ff, allocated ll: (f, b, c, ., ll, v, ll, of)*/ \ | |
| 163 /*f=pavgb(f,v);*/ \ | |
| 164 "pavgb %%mm5,%%mm0\n\t" \ | |
| 165 "movq %%mm7,%%mm3\n\t" \ | |
| 166 "movq %%mm6,%%mm4\n\t" \ | |
| 167 /*freed v, allocated of: (f, b, c, of, ll, ., ll, of)*/ \ | |
| 168 /*Now compute lflim of R_i=-(128+mm0) cf. Section 7.10 of the sepc.*/ \ | |
| 169 /*There's no unsigned byte+signed byte with unsigned saturation op code, so \ | |
| 170 we have to split things by sign (the other option is to work in 16 bits, \ | |
| 171 but staying in 8 bits gives much better parallelism).*/ \ | |
| 172 /*Instead of adding the offset of 128 in mm3, we use it to split mm0. \ | |
| 173 This is the same number of instructions as computing a mask and splitting \ | |
| 174 after the lflim computation, but has shorter dependency chains.*/ \ | |
| 175 /*mm0=R_i<0?-R_i:0 (denoted abs(R_i<0))\ | |
| 176 mm3=R_i>0?R_i:0* (denoted abs(R_i>0))*/ \ | |
| 177 "psubusb %%mm0,%%mm3\n\t" \ | |
| 178 "psubusb %%mm7,%%mm0\n\t" \ | |
| 179 /*mm6=255-max(2*L-abs(R_i<0),0) \ | |
| 180 mm4=255-max(2*L-abs(R_i>0),0)*/ \ | |
| 181 "paddusb %%mm3,%%mm4\n\t" \ | |
| 182 "paddusb %%mm0,%%mm6\n\t" \ | |
| 183 /*mm0=min(abs(R_i<0),max(2*L-abs(R_i<0),0)) \ | |
| 184 mm3=min(abs(R_i>0),max(2*L-abs(R_i>0),0))*/ \ | |
| 185 "paddusb %%mm4,%%mm3\n\t" \ | |
| 186 "paddusb %%mm6,%%mm0\n\t" \ | |
| 187 "psubusb %%mm4,%%mm3\n\t" \ | |
| 188 "psubusb %%mm6,%%mm0\n\t" \ | |
| 189 /*mm1={b0+lflim(R_0,L),...,b7+lflim(R_7,L)}*/ \ | |
| 190 /*mm2={c0-lflim(R_0,L),...,c7-lflim(R_7,L)}*/ \ | |
| 191 "paddusb %%mm3,%%mm1\n\t" \ | |
| 192 "psubusb %%mm3,%%mm2\n\t" \ | |
| 193 "psubusb %%mm0,%%mm1\n\t" \ | |
| 194 "paddusb %%mm0,%%mm2\n\t" \ | |
| 195 | |
| 196 #define OC_LOOP_FILTER_V(_filter,_pix,_ystride,_ll) \ | |
| 197 do{ \ | |
| 198 ptrdiff_t ystride3__; \ | |
| 199 __asm__ __volatile__( \ | |
| 200 /*mm0={a0,...,a7}*/ \ | |
| 201 "movq (%[pix]),%%mm0\n\t" \ | |
| 202 /*ystride3=_ystride*3*/ \ | |
| 203 "lea (%[ystride],%[ystride],2),%[ystride3]\n\t" \ | |
| 204 /*mm3={d0,...,d7}*/ \ | |
| 205 "movq (%[pix],%[ystride3]),%%mm3\n\t" \ | |
| 206 /*mm1={b0,...,b7}*/ \ | |
| 207 "movq (%[pix],%[ystride]),%%mm1\n\t" \ | |
| 208 /*mm2={c0,...,c7}*/ \ | |
| 209 "movq (%[pix],%[ystride],2),%%mm2\n\t" \ | |
| 210 _filter \ | |
| 211 /*Write it back out.*/ \ | |
| 212 "movq %%mm1,(%[pix],%[ystride])\n\t" \ | |
| 213 "movq %%mm2,(%[pix],%[ystride],2)\n\t" \ | |
| 214 :[ystride3]"=&r"(ystride3__) \ | |
| 215 :[pix]"r"(_pix-_ystride*2),[ystride]"r"((ptrdiff_t)(_ystride)), \ | |
| 216 [ll]"r"(_ll) \ | |
| 217 :"memory" \ | |
| 218 ); \ | |
| 219 } \ | |
| 220 while(0) | |
| 221 | |
| 222 #define OC_LOOP_FILTER_H(_filter,_pix,_ystride,_ll) \ | |
| 223 do{ \ | |
| 224 unsigned char *pix__; \ | |
| 225 ptrdiff_t ystride3__; \ | |
| 226 ptrdiff_t d__; \ | |
| 227 pix__=(_pix)-2; \ | |
| 228 __asm__ __volatile__( \ | |
| 229 /*x x x x d0 c0 b0 a0*/ \ | |
| 230 "movd (%[pix]),%%mm0\n\t" \ | |
| 231 /*x x x x d1 c1 b1 a1*/ \ | |
| 232 "movd (%[pix],%[ystride]),%%mm1\n\t" \ | |
| 233 /*ystride3=_ystride*3*/ \ | |
| 234 "lea (%[ystride],%[ystride],2),%[ystride3]\n\t" \ | |
| 235 /*x x x x d2 c2 b2 a2*/ \ | |
| 236 "movd (%[pix],%[ystride],2),%%mm2\n\t" \ | |
| 237 /*x x x x d3 c3 b3 a3*/ \ | |
| 238 "lea (%[pix],%[ystride],4),%[d]\n\t" \ | |
| 239 "movd (%[pix],%[ystride3]),%%mm3\n\t" \ | |
| 240 /*x x x x d4 c4 b4 a4*/ \ | |
| 241 "movd (%[d]),%%mm4\n\t" \ | |
| 242 /*x x x x d5 c5 b5 a5*/ \ | |
| 243 "movd (%[d],%[ystride]),%%mm5\n\t" \ | |
| 244 /*x x x x d6 c6 b6 a6*/ \ | |
| 245 "movd (%[d],%[ystride],2),%%mm6\n\t" \ | |
| 246 /*x x x x d7 c7 b7 a7*/ \ | |
| 247 "movd (%[d],%[ystride3]),%%mm7\n\t" \ | |
| 248 /*mm0=d1 d0 c1 c0 b1 b0 a1 a0*/ \ | |
| 249 "punpcklbw %%mm1,%%mm0\n\t" \ | |
| 250 /*mm2=d3 d2 c3 c2 b3 b2 a3 a2*/ \ | |
| 251 "punpcklbw %%mm3,%%mm2\n\t" \ | |
| 252 /*mm3=d1 d0 c1 c0 b1 b0 a1 a0*/ \ | |
| 253 "movq %%mm0,%%mm3\n\t" \ | |
| 254 /*mm0=b3 b2 b1 b0 a3 a2 a1 a0*/ \ | |
| 255 "punpcklwd %%mm2,%%mm0\n\t" \ | |
| 256 /*mm3=d3 d2 d1 d0 c3 c2 c1 c0*/ \ | |
| 257 "punpckhwd %%mm2,%%mm3\n\t" \ | |
| 258 /*mm1=b3 b2 b1 b0 a3 a2 a1 a0*/ \ | |
| 259 "movq %%mm0,%%mm1\n\t" \ | |
| 260 /*mm4=d5 d4 c5 c4 b5 b4 a5 a4*/ \ | |
| 261 "punpcklbw %%mm5,%%mm4\n\t" \ | |
| 262 /*mm6=d7 d6 c7 c6 b7 b6 a7 a6*/ \ | |
| 263 "punpcklbw %%mm7,%%mm6\n\t" \ | |
| 264 /*mm5=d5 d4 c5 c4 b5 b4 a5 a4*/ \ | |
| 265 "movq %%mm4,%%mm5\n\t" \ | |
| 266 /*mm4=b7 b6 b5 b4 a7 a6 a5 a4*/ \ | |
| 267 "punpcklwd %%mm6,%%mm4\n\t" \ | |
| 268 /*mm5=d7 d6 d5 d4 c7 c6 c5 c4*/ \ | |
| 269 "punpckhwd %%mm6,%%mm5\n\t" \ | |
| 270 /*mm2=d3 d2 d1 d0 c3 c2 c1 c0*/ \ | |
| 271 "movq %%mm3,%%mm2\n\t" \ | |
| 272 /*mm0=a7 a6 a5 a4 a3 a2 a1 a0*/ \ | |
| 273 "punpckldq %%mm4,%%mm0\n\t" \ | |
| 274 /*mm1=b7 b6 b5 b4 b3 b2 b1 b0*/ \ | |
| 275 "punpckhdq %%mm4,%%mm1\n\t" \ | |
| 276 /*mm2=c7 c6 c5 c4 c3 c2 c1 c0*/ \ | |
| 277 "punpckldq %%mm5,%%mm2\n\t" \ | |
| 278 /*mm3=d7 d6 d5 d4 d3 d2 d1 d0*/ \ | |
| 279 "punpckhdq %%mm5,%%mm3\n\t" \ | |
| 280 _filter \ | |
| 281 /*mm2={b0+R_0'',...,b7+R_7''}*/ \ | |
| 282 "movq %%mm1,%%mm0\n\t" \ | |
| 283 /*mm1={b0+R_0'',c0-R_0'',...,b3+R_3'',c3-R_3''}*/ \ | |
| 284 "punpcklbw %%mm2,%%mm1\n\t" \ | |
| 285 /*mm2={b4+R_4'',c4-R_4'',...,b7+R_7'',c7-R_7''}*/ \ | |
| 286 "punpckhbw %%mm2,%%mm0\n\t" \ | |
| 287 /*[d]=c1 b1 c0 b0*/ \ | |
| 288 "movd %%mm1,%[d]\n\t" \ | |
| 289 "movw %w[d],1(%[pix])\n\t" \ | |
| 290 "psrlq $32,%%mm1\n\t" \ | |
| 291 "shr $16,%[d]\n\t" \ | |
| 292 "movw %w[d],1(%[pix],%[ystride])\n\t" \ | |
| 293 /*[d]=c3 b3 c2 b2*/ \ | |
| 294 "movd %%mm1,%[d]\n\t" \ | |
| 295 "movw %w[d],1(%[pix],%[ystride],2)\n\t" \ | |
| 296 "shr $16,%[d]\n\t" \ | |
| 297 "movw %w[d],1(%[pix],%[ystride3])\n\t" \ | |
| 298 "lea (%[pix],%[ystride],4),%[pix]\n\t" \ | |
| 299 /*[d]=c5 b5 c4 b4*/ \ | |
| 300 "movd %%mm0,%[d]\n\t" \ | |
| 301 "movw %w[d],1(%[pix])\n\t" \ | |
| 302 "psrlq $32,%%mm0\n\t" \ | |
| 303 "shr $16,%[d]\n\t" \ | |
| 304 "movw %w[d],1(%[pix],%[ystride])\n\t" \ | |
| 305 /*[d]=c7 b7 c6 b6*/ \ | |
| 306 "movd %%mm0,%[d]\n\t" \ | |
| 307 "movw %w[d],1(%[pix],%[ystride],2)\n\t" \ | |
| 308 "shr $16,%[d]\n\t" \ | |
| 309 "movw %w[d],1(%[pix],%[ystride3])\n\t" \ | |
| 310 :[pix]"+r"(pix__),[ystride3]"=&r"(ystride3__),[d]"=&r"(d__) \ | |
| 311 :[ystride]"r"((ptrdiff_t)(_ystride)),[ll]"r"(_ll) \ | |
| 312 :"memory" \ | |
| 313 ); \ | |
| 314 } \ | |
| 315 while(0) | |
| 316 | |
| 317 # endif | |
| 318 #endif |
