The Tor Browser / file comparison

comparison: media/libvpx/vp9/encoder/x86/vp9_dct_sse2.c

media/libvpx/vp9/encoder/x86/vp9_dct_sse2.c

branch
TOR_BUG_9701
changeset 10
ac0c01689b40
equal deleted inserted replaced
-1:000000000000 0:9366912b84d1
1 /*
2 * Copyright (c) 2012 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <emmintrin.h> // SSE2
12 #include "vp9/common/vp9_idct.h" // for cospi constants
13 #include "vpx_ports/mem.h"
14
15 void vp9_fdct4x4_sse2(const int16_t *input, int16_t *output, int stride) {
16 // The 2D transform is done with two passes which are actually pretty
17 // similar. In the first one, we transform the columns and transpose
18 // the results. In the second one, we transform the rows. To achieve that,
19 // as the first pass results are transposed, we tranpose the columns (that
20 // is the transposed rows) and transpose the results (so that it goes back
21 // in normal/row positions).
22 int pass;
23 // Constants
24 // When we use them, in one case, they are all the same. In all others
25 // it's a pair of them that we need to repeat four times. This is done
26 // by constructing the 32 bit constant corresponding to that pair.
27 const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
28 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
29 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
30 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
31 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
32 const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1);
33 const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0);
34 const __m128i kOne = _mm_set1_epi16(1);
35 __m128i in0, in1, in2, in3;
36 // Load inputs.
37 {
38 in0 = _mm_loadl_epi64((const __m128i *)(input + 0 * stride));
39 in1 = _mm_loadl_epi64((const __m128i *)(input + 1 * stride));
40 in2 = _mm_loadl_epi64((const __m128i *)(input + 2 * stride));
41 in3 = _mm_loadl_epi64((const __m128i *)(input + 3 * stride));
42 // x = x << 4
43 in0 = _mm_slli_epi16(in0, 4);
44 in1 = _mm_slli_epi16(in1, 4);
45 in2 = _mm_slli_epi16(in2, 4);
46 in3 = _mm_slli_epi16(in3, 4);
47 // if (i == 0 && input[0]) input[0] += 1;
48 {
49 // The mask will only contain wether the first value is zero, all
50 // other comparison will fail as something shifted by 4 (above << 4)
51 // can never be equal to one. To increment in the non-zero case, we
52 // add the mask and one for the first element:
53 // - if zero, mask = -1, v = v - 1 + 1 = v
54 // - if non-zero, mask = 0, v = v + 0 + 1 = v + 1
55 __m128i mask = _mm_cmpeq_epi16(in0, k__nonzero_bias_a);
56 in0 = _mm_add_epi16(in0, mask);
57 in0 = _mm_add_epi16(in0, k__nonzero_bias_b);
58 }
59 }
60 // Do the two transform/transpose passes
61 for (pass = 0; pass < 2; ++pass) {
62 // Transform 1/2: Add/substract
63 const __m128i r0 = _mm_add_epi16(in0, in3);
64 const __m128i r1 = _mm_add_epi16(in1, in2);
65 const __m128i r2 = _mm_sub_epi16(in1, in2);
66 const __m128i r3 = _mm_sub_epi16(in0, in3);
67 // Transform 1/2: Interleave to do the multiply by constants which gets us
68 // into 32 bits.
69 const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
70 const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
71 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
72 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
73 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
74 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
75 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
76 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
77 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
78 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
79 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
80 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
81 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
82 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
83 // Combine and transpose
84 const __m128i res0 = _mm_packs_epi32(w0, w2);
85 const __m128i res1 = _mm_packs_epi32(w4, w6);
86 // 00 01 02 03 20 21 22 23
87 // 10 11 12 13 30 31 32 33
88 const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1);
89 const __m128i tr0_1 = _mm_unpackhi_epi16(res0, res1);
90 // 00 10 01 11 02 12 03 13
91 // 20 30 21 31 22 32 23 33
92 in0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
93 in2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
94 // 00 10 20 30 01 11 21 31 in0 contains 0 followed by 1
95 // 02 12 22 32 03 13 23 33 in2 contains 2 followed by 3
96 if (0 == pass) {
97 // Extract values in the high part for second pass as transform code
98 // only uses the first four values.
99 in1 = _mm_unpackhi_epi64(in0, in0);
100 in3 = _mm_unpackhi_epi64(in2, in2);
101 } else {
102 // Post-condition output and store it (v + 1) >> 2, taking advantage
103 // of the fact 1/3 are stored just after 0/2.
104 __m128i out01 = _mm_add_epi16(in0, kOne);
105 __m128i out23 = _mm_add_epi16(in2, kOne);
106 out01 = _mm_srai_epi16(out01, 2);
107 out23 = _mm_srai_epi16(out23, 2);
108 _mm_storeu_si128((__m128i *)(output + 0 * 4), out01);
109 _mm_storeu_si128((__m128i *)(output + 2 * 4), out23);
110 }
111 }
112 }
113
114 static INLINE void load_buffer_4x4(const int16_t *input, __m128i *in,
115 int stride) {
116 const __m128i k__nonzero_bias_a = _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1);
117 const __m128i k__nonzero_bias_b = _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0);
118 __m128i mask;
119
120 in[0] = _mm_loadl_epi64((const __m128i *)(input + 0 * stride));
121 in[1] = _mm_loadl_epi64((const __m128i *)(input + 1 * stride));
122 in[2] = _mm_loadl_epi64((const __m128i *)(input + 2 * stride));
123 in[3] = _mm_loadl_epi64((const __m128i *)(input + 3 * stride));
124
125 in[0] = _mm_slli_epi16(in[0], 4);
126 in[1] = _mm_slli_epi16(in[1], 4);
127 in[2] = _mm_slli_epi16(in[2], 4);
128 in[3] = _mm_slli_epi16(in[3], 4);
129
130 mask = _mm_cmpeq_epi16(in[0], k__nonzero_bias_a);
131 in[0] = _mm_add_epi16(in[0], mask);
132 in[0] = _mm_add_epi16(in[0], k__nonzero_bias_b);
133 }
134
135 static INLINE void write_buffer_4x4(int16_t *output, __m128i *res) {
136 const __m128i kOne = _mm_set1_epi16(1);
137 __m128i in01 = _mm_unpacklo_epi64(res[0], res[1]);
138 __m128i in23 = _mm_unpacklo_epi64(res[2], res[3]);
139 __m128i out01 = _mm_add_epi16(in01, kOne);
140 __m128i out23 = _mm_add_epi16(in23, kOne);
141 out01 = _mm_srai_epi16(out01, 2);
142 out23 = _mm_srai_epi16(out23, 2);
143 _mm_store_si128((__m128i *)(output + 0 * 8), out01);
144 _mm_store_si128((__m128i *)(output + 1 * 8), out23);
145 }
146
147 static INLINE void transpose_4x4(__m128i *res) {
148 // Combine and transpose
149 // 00 01 02 03 20 21 22 23
150 // 10 11 12 13 30 31 32 33
151 const __m128i tr0_0 = _mm_unpacklo_epi16(res[0], res[1]);
152 const __m128i tr0_1 = _mm_unpackhi_epi16(res[0], res[1]);
153
154 // 00 10 01 11 02 12 03 13
155 // 20 30 21 31 22 32 23 33
156 res[0] = _mm_unpacklo_epi32(tr0_0, tr0_1);
157 res[2] = _mm_unpackhi_epi32(tr0_0, tr0_1);
158
159 // 00 10 20 30 01 11 21 31
160 // 02 12 22 32 03 13 23 33
161 // only use the first 4 16-bit integers
162 res[1] = _mm_unpackhi_epi64(res[0], res[0]);
163 res[3] = _mm_unpackhi_epi64(res[2], res[2]);
164 }
165
166 void fdct4_1d_sse2(__m128i *in) {
167 const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
168 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
169 const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64);
170 const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64);
171 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
172
173 __m128i u[4], v[4];
174 u[0]=_mm_unpacklo_epi16(in[0], in[1]);
175 u[1]=_mm_unpacklo_epi16(in[3], in[2]);
176
177 v[0] = _mm_add_epi16(u[0], u[1]);
178 v[1] = _mm_sub_epi16(u[0], u[1]);
179
180 u[0] = _mm_madd_epi16(v[0], k__cospi_p16_p16); // 0
181 u[1] = _mm_madd_epi16(v[0], k__cospi_p16_m16); // 2
182 u[2] = _mm_madd_epi16(v[1], k__cospi_p08_p24); // 1
183 u[3] = _mm_madd_epi16(v[1], k__cospi_p24_m08); // 3
184
185 v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
186 v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
187 v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
188 v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING);
189 u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS);
190 u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS);
191 u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS);
192 u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS);
193
194 in[0] = _mm_packs_epi32(u[0], u[1]);
195 in[1] = _mm_packs_epi32(u[2], u[3]);
196 transpose_4x4(in);
197 }
198
199 void fadst4_1d_sse2(__m128i *in) {
200 const __m128i k__sinpi_p01_p02 = pair_set_epi16(sinpi_1_9, sinpi_2_9);
201 const __m128i k__sinpi_p04_m01 = pair_set_epi16(sinpi_4_9, -sinpi_1_9);
202 const __m128i k__sinpi_p03_p04 = pair_set_epi16(sinpi_3_9, sinpi_4_9);
203 const __m128i k__sinpi_m03_p02 = pair_set_epi16(-sinpi_3_9, sinpi_2_9);
204 const __m128i k__sinpi_p03_p03 = _mm_set1_epi16(sinpi_3_9);
205 const __m128i kZero = _mm_set1_epi16(0);
206 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
207 __m128i u[8], v[8];
208 __m128i in7 = _mm_add_epi16(in[0], in[1]);
209
210 u[0] = _mm_unpacklo_epi16(in[0], in[1]);
211 u[1] = _mm_unpacklo_epi16(in[2], in[3]);
212 u[2] = _mm_unpacklo_epi16(in7, kZero);
213 u[3] = _mm_unpacklo_epi16(in[2], kZero);
214 u[4] = _mm_unpacklo_epi16(in[3], kZero);
215
216 v[0] = _mm_madd_epi16(u[0], k__sinpi_p01_p02); // s0 + s2
217 v[1] = _mm_madd_epi16(u[1], k__sinpi_p03_p04); // s4 + s5
218 v[2] = _mm_madd_epi16(u[2], k__sinpi_p03_p03); // x1
219 v[3] = _mm_madd_epi16(u[0], k__sinpi_p04_m01); // s1 - s3
220 v[4] = _mm_madd_epi16(u[1], k__sinpi_m03_p02); // -s4 + s6
221 v[5] = _mm_madd_epi16(u[3], k__sinpi_p03_p03); // s4
222 v[6] = _mm_madd_epi16(u[4], k__sinpi_p03_p03);
223
224 u[0] = _mm_add_epi32(v[0], v[1]);
225 u[1] = _mm_sub_epi32(v[2], v[6]);
226 u[2] = _mm_add_epi32(v[3], v[4]);
227 u[3] = _mm_sub_epi32(u[2], u[0]);
228 u[4] = _mm_slli_epi32(v[5], 2);
229 u[5] = _mm_sub_epi32(u[4], v[5]);
230 u[6] = _mm_add_epi32(u[3], u[5]);
231
232 v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
233 v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
234 v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
235 v[3] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING);
236
237 u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS);
238 u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS);
239 u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS);
240 u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS);
241
242 in[0] = _mm_packs_epi32(u[0], u[2]);
243 in[1] = _mm_packs_epi32(u[1], u[3]);
244 transpose_4x4(in);
245 }
246
247 void vp9_short_fht4x4_sse2(const int16_t *input, int16_t *output,
248 int stride, int tx_type) {
249 __m128i in[4];
250 load_buffer_4x4(input, in, stride);
251 switch (tx_type) {
252 case 0: // DCT_DCT
253 fdct4_1d_sse2(in);
254 fdct4_1d_sse2(in);
255 break;
256 case 1: // ADST_DCT
257 fadst4_1d_sse2(in);
258 fdct4_1d_sse2(in);
259 break;
260 case 2: // DCT_ADST
261 fdct4_1d_sse2(in);
262 fadst4_1d_sse2(in);
263 break;
264 case 3: // ADST_ADST
265 fadst4_1d_sse2(in);
266 fadst4_1d_sse2(in);
267 break;
268 default:
269 assert(0);
270 break;
271 }
272 write_buffer_4x4(output, in);
273 }
274
275 void vp9_fdct8x8_sse2(const int16_t *input, int16_t *output, int stride) {
276 int pass;
277 // Constants
278 // When we use them, in one case, they are all the same. In all others
279 // it's a pair of them that we need to repeat four times. This is done
280 // by constructing the 32 bit constant corresponding to that pair.
281 const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
282 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
283 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
284 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
285 const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
286 const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
287 const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
288 const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
289 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
290 // Load input
291 __m128i in0 = _mm_load_si128((const __m128i *)(input + 0 * stride));
292 __m128i in1 = _mm_load_si128((const __m128i *)(input + 1 * stride));
293 __m128i in2 = _mm_load_si128((const __m128i *)(input + 2 * stride));
294 __m128i in3 = _mm_load_si128((const __m128i *)(input + 3 * stride));
295 __m128i in4 = _mm_load_si128((const __m128i *)(input + 4 * stride));
296 __m128i in5 = _mm_load_si128((const __m128i *)(input + 5 * stride));
297 __m128i in6 = _mm_load_si128((const __m128i *)(input + 6 * stride));
298 __m128i in7 = _mm_load_si128((const __m128i *)(input + 7 * stride));
299 // Pre-condition input (shift by two)
300 in0 = _mm_slli_epi16(in0, 2);
301 in1 = _mm_slli_epi16(in1, 2);
302 in2 = _mm_slli_epi16(in2, 2);
303 in3 = _mm_slli_epi16(in3, 2);
304 in4 = _mm_slli_epi16(in4, 2);
305 in5 = _mm_slli_epi16(in5, 2);
306 in6 = _mm_slli_epi16(in6, 2);
307 in7 = _mm_slli_epi16(in7, 2);
308
309 // We do two passes, first the columns, then the rows. The results of the
310 // first pass are transposed so that the same column code can be reused. The
311 // results of the second pass are also transposed so that the rows (processed
312 // as columns) are put back in row positions.
313 for (pass = 0; pass < 2; pass++) {
314 // To store results of each pass before the transpose.
315 __m128i res0, res1, res2, res3, res4, res5, res6, res7;
316 // Add/substract
317 const __m128i q0 = _mm_add_epi16(in0, in7);
318 const __m128i q1 = _mm_add_epi16(in1, in6);
319 const __m128i q2 = _mm_add_epi16(in2, in5);
320 const __m128i q3 = _mm_add_epi16(in3, in4);
321 const __m128i q4 = _mm_sub_epi16(in3, in4);
322 const __m128i q5 = _mm_sub_epi16(in2, in5);
323 const __m128i q6 = _mm_sub_epi16(in1, in6);
324 const __m128i q7 = _mm_sub_epi16(in0, in7);
325 // Work on first four results
326 {
327 // Add/substract
328 const __m128i r0 = _mm_add_epi16(q0, q3);
329 const __m128i r1 = _mm_add_epi16(q1, q2);
330 const __m128i r2 = _mm_sub_epi16(q1, q2);
331 const __m128i r3 = _mm_sub_epi16(q0, q3);
332 // Interleave to do the multiply by constants which gets us into 32bits
333 const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
334 const __m128i t1 = _mm_unpackhi_epi16(r0, r1);
335 const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
336 const __m128i t3 = _mm_unpackhi_epi16(r2, r3);
337 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
338 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16);
339 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
340 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16);
341 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
342 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08);
343 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
344 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24);
345 // dct_const_round_shift
346 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
347 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
348 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
349 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
350 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
351 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
352 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
353 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
354 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
355 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
356 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
357 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
358 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
359 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
360 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
361 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
362 // Combine
363 res0 = _mm_packs_epi32(w0, w1);
364 res4 = _mm_packs_epi32(w2, w3);
365 res2 = _mm_packs_epi32(w4, w5);
366 res6 = _mm_packs_epi32(w6, w7);
367 }
368 // Work on next four results
369 {
370 // Interleave to do the multiply by constants which gets us into 32bits
371 const __m128i d0 = _mm_unpacklo_epi16(q6, q5);
372 const __m128i d1 = _mm_unpackhi_epi16(q6, q5);
373 const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16);
374 const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16);
375 const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16);
376 const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16);
377 // dct_const_round_shift
378 const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING);
379 const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING);
380 const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING);
381 const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING);
382 const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS);
383 const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS);
384 const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS);
385 const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS);
386 // Combine
387 const __m128i r0 = _mm_packs_epi32(s0, s1);
388 const __m128i r1 = _mm_packs_epi32(s2, s3);
389 // Add/substract
390 const __m128i x0 = _mm_add_epi16(q4, r0);
391 const __m128i x1 = _mm_sub_epi16(q4, r0);
392 const __m128i x2 = _mm_sub_epi16(q7, r1);
393 const __m128i x3 = _mm_add_epi16(q7, r1);
394 // Interleave to do the multiply by constants which gets us into 32bits
395 const __m128i t0 = _mm_unpacklo_epi16(x0, x3);
396 const __m128i t1 = _mm_unpackhi_epi16(x0, x3);
397 const __m128i t2 = _mm_unpacklo_epi16(x1, x2);
398 const __m128i t3 = _mm_unpackhi_epi16(x1, x2);
399 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04);
400 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04);
401 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28);
402 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28);
403 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20);
404 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20);
405 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12);
406 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12);
407 // dct_const_round_shift
408 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
409 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
410 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
411 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
412 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
413 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
414 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
415 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
416 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
417 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
418 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
419 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
420 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
421 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
422 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
423 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
424 // Combine
425 res1 = _mm_packs_epi32(w0, w1);
426 res7 = _mm_packs_epi32(w2, w3);
427 res5 = _mm_packs_epi32(w4, w5);
428 res3 = _mm_packs_epi32(w6, w7);
429 }
430 // Transpose the 8x8.
431 {
432 // 00 01 02 03 04 05 06 07
433 // 10 11 12 13 14 15 16 17
434 // 20 21 22 23 24 25 26 27
435 // 30 31 32 33 34 35 36 37
436 // 40 41 42 43 44 45 46 47
437 // 50 51 52 53 54 55 56 57
438 // 60 61 62 63 64 65 66 67
439 // 70 71 72 73 74 75 76 77
440 const __m128i tr0_0 = _mm_unpacklo_epi16(res0, res1);
441 const __m128i tr0_1 = _mm_unpacklo_epi16(res2, res3);
442 const __m128i tr0_2 = _mm_unpackhi_epi16(res0, res1);
443 const __m128i tr0_3 = _mm_unpackhi_epi16(res2, res3);
444 const __m128i tr0_4 = _mm_unpacklo_epi16(res4, res5);
445 const __m128i tr0_5 = _mm_unpacklo_epi16(res6, res7);
446 const __m128i tr0_6 = _mm_unpackhi_epi16(res4, res5);
447 const __m128i tr0_7 = _mm_unpackhi_epi16(res6, res7);
448 // 00 10 01 11 02 12 03 13
449 // 20 30 21 31 22 32 23 33
450 // 04 14 05 15 06 16 07 17
451 // 24 34 25 35 26 36 27 37
452 // 40 50 41 51 42 52 43 53
453 // 60 70 61 71 62 72 63 73
454 // 54 54 55 55 56 56 57 57
455 // 64 74 65 75 66 76 67 77
456 const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
457 const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3);
458 const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
459 const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3);
460 const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5);
461 const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
462 const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5);
463 const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
464 // 00 10 20 30 01 11 21 31
465 // 40 50 60 70 41 51 61 71
466 // 02 12 22 32 03 13 23 33
467 // 42 52 62 72 43 53 63 73
468 // 04 14 24 34 05 15 21 36
469 // 44 54 64 74 45 55 61 76
470 // 06 16 26 36 07 17 27 37
471 // 46 56 66 76 47 57 67 77
472 in0 = _mm_unpacklo_epi64(tr1_0, tr1_4);
473 in1 = _mm_unpackhi_epi64(tr1_0, tr1_4);
474 in2 = _mm_unpacklo_epi64(tr1_2, tr1_6);
475 in3 = _mm_unpackhi_epi64(tr1_2, tr1_6);
476 in4 = _mm_unpacklo_epi64(tr1_1, tr1_5);
477 in5 = _mm_unpackhi_epi64(tr1_1, tr1_5);
478 in6 = _mm_unpacklo_epi64(tr1_3, tr1_7);
479 in7 = _mm_unpackhi_epi64(tr1_3, tr1_7);
480 // 00 10 20 30 40 50 60 70
481 // 01 11 21 31 41 51 61 71
482 // 02 12 22 32 42 52 62 72
483 // 03 13 23 33 43 53 63 73
484 // 04 14 24 34 44 54 64 74
485 // 05 15 25 35 45 55 65 75
486 // 06 16 26 36 46 56 66 76
487 // 07 17 27 37 47 57 67 77
488 }
489 }
490 // Post-condition output and store it
491 {
492 // Post-condition (division by two)
493 // division of two 16 bits signed numbers using shifts
494 // n / 2 = (n - (n >> 15)) >> 1
495 const __m128i sign_in0 = _mm_srai_epi16(in0, 15);
496 const __m128i sign_in1 = _mm_srai_epi16(in1, 15);
497 const __m128i sign_in2 = _mm_srai_epi16(in2, 15);
498 const __m128i sign_in3 = _mm_srai_epi16(in3, 15);
499 const __m128i sign_in4 = _mm_srai_epi16(in4, 15);
500 const __m128i sign_in5 = _mm_srai_epi16(in5, 15);
501 const __m128i sign_in6 = _mm_srai_epi16(in6, 15);
502 const __m128i sign_in7 = _mm_srai_epi16(in7, 15);
503 in0 = _mm_sub_epi16(in0, sign_in0);
504 in1 = _mm_sub_epi16(in1, sign_in1);
505 in2 = _mm_sub_epi16(in2, sign_in2);
506 in3 = _mm_sub_epi16(in3, sign_in3);
507 in4 = _mm_sub_epi16(in4, sign_in4);
508 in5 = _mm_sub_epi16(in5, sign_in5);
509 in6 = _mm_sub_epi16(in6, sign_in6);
510 in7 = _mm_sub_epi16(in7, sign_in7);
511 in0 = _mm_srai_epi16(in0, 1);
512 in1 = _mm_srai_epi16(in1, 1);
513 in2 = _mm_srai_epi16(in2, 1);
514 in3 = _mm_srai_epi16(in3, 1);
515 in4 = _mm_srai_epi16(in4, 1);
516 in5 = _mm_srai_epi16(in5, 1);
517 in6 = _mm_srai_epi16(in6, 1);
518 in7 = _mm_srai_epi16(in7, 1);
519 // store results
520 _mm_store_si128((__m128i *)(output + 0 * 8), in0);
521 _mm_store_si128((__m128i *)(output + 1 * 8), in1);
522 _mm_store_si128((__m128i *)(output + 2 * 8), in2);
523 _mm_store_si128((__m128i *)(output + 3 * 8), in3);
524 _mm_store_si128((__m128i *)(output + 4 * 8), in4);
525 _mm_store_si128((__m128i *)(output + 5 * 8), in5);
526 _mm_store_si128((__m128i *)(output + 6 * 8), in6);
527 _mm_store_si128((__m128i *)(output + 7 * 8), in7);
528 }
529 }
530
531 // load 8x8 array
532 static INLINE void load_buffer_8x8(const int16_t *input, __m128i *in,
533 int stride) {
534 in[0] = _mm_load_si128((const __m128i *)(input + 0 * stride));
535 in[1] = _mm_load_si128((const __m128i *)(input + 1 * stride));
536 in[2] = _mm_load_si128((const __m128i *)(input + 2 * stride));
537 in[3] = _mm_load_si128((const __m128i *)(input + 3 * stride));
538 in[4] = _mm_load_si128((const __m128i *)(input + 4 * stride));
539 in[5] = _mm_load_si128((const __m128i *)(input + 5 * stride));
540 in[6] = _mm_load_si128((const __m128i *)(input + 6 * stride));
541 in[7] = _mm_load_si128((const __m128i *)(input + 7 * stride));
542
543 in[0] = _mm_slli_epi16(in[0], 2);
544 in[1] = _mm_slli_epi16(in[1], 2);
545 in[2] = _mm_slli_epi16(in[2], 2);
546 in[3] = _mm_slli_epi16(in[3], 2);
547 in[4] = _mm_slli_epi16(in[4], 2);
548 in[5] = _mm_slli_epi16(in[5], 2);
549 in[6] = _mm_slli_epi16(in[6], 2);
550 in[7] = _mm_slli_epi16(in[7], 2);
551 }
552
553 // right shift and rounding
554 static INLINE void right_shift_8x8(__m128i *res, int const bit) {
555 const __m128i kOne = _mm_set1_epi16(1);
556 const int bit_m02 = bit - 2;
557 __m128i sign0 = _mm_srai_epi16(res[0], 15);
558 __m128i sign1 = _mm_srai_epi16(res[1], 15);
559 __m128i sign2 = _mm_srai_epi16(res[2], 15);
560 __m128i sign3 = _mm_srai_epi16(res[3], 15);
561 __m128i sign4 = _mm_srai_epi16(res[4], 15);
562 __m128i sign5 = _mm_srai_epi16(res[5], 15);
563 __m128i sign6 = _mm_srai_epi16(res[6], 15);
564 __m128i sign7 = _mm_srai_epi16(res[7], 15);
565
566 if (bit_m02 >= 0) {
567 __m128i k_const_rounding = _mm_slli_epi16(kOne, bit_m02);
568 res[0] = _mm_add_epi16(res[0], k_const_rounding);
569 res[1] = _mm_add_epi16(res[1], k_const_rounding);
570 res[2] = _mm_add_epi16(res[2], k_const_rounding);
571 res[3] = _mm_add_epi16(res[3], k_const_rounding);
572 res[4] = _mm_add_epi16(res[4], k_const_rounding);
573 res[5] = _mm_add_epi16(res[5], k_const_rounding);
574 res[6] = _mm_add_epi16(res[6], k_const_rounding);
575 res[7] = _mm_add_epi16(res[7], k_const_rounding);
576 }
577
578 res[0] = _mm_sub_epi16(res[0], sign0);
579 res[1] = _mm_sub_epi16(res[1], sign1);
580 res[2] = _mm_sub_epi16(res[2], sign2);
581 res[3] = _mm_sub_epi16(res[3], sign3);
582 res[4] = _mm_sub_epi16(res[4], sign4);
583 res[5] = _mm_sub_epi16(res[5], sign5);
584 res[6] = _mm_sub_epi16(res[6], sign6);
585 res[7] = _mm_sub_epi16(res[7], sign7);
586
587 res[0] = _mm_srai_epi16(res[0], bit);
588 res[1] = _mm_srai_epi16(res[1], bit);
589 res[2] = _mm_srai_epi16(res[2], bit);
590 res[3] = _mm_srai_epi16(res[3], bit);
591 res[4] = _mm_srai_epi16(res[4], bit);
592 res[5] = _mm_srai_epi16(res[5], bit);
593 res[6] = _mm_srai_epi16(res[6], bit);
594 res[7] = _mm_srai_epi16(res[7], bit);
595 }
596
597 // write 8x8 array
598 static INLINE void write_buffer_8x8(int16_t *output, __m128i *res, int stride) {
599 _mm_store_si128((__m128i *)(output + 0 * stride), res[0]);
600 _mm_store_si128((__m128i *)(output + 1 * stride), res[1]);
601 _mm_store_si128((__m128i *)(output + 2 * stride), res[2]);
602 _mm_store_si128((__m128i *)(output + 3 * stride), res[3]);
603 _mm_store_si128((__m128i *)(output + 4 * stride), res[4]);
604 _mm_store_si128((__m128i *)(output + 5 * stride), res[5]);
605 _mm_store_si128((__m128i *)(output + 6 * stride), res[6]);
606 _mm_store_si128((__m128i *)(output + 7 * stride), res[7]);
607 }
608
609 // perform in-place transpose
610 static INLINE void array_transpose_8x8(__m128i *in, __m128i *res) {
611 const __m128i tr0_0 = _mm_unpacklo_epi16(in[0], in[1]);
612 const __m128i tr0_1 = _mm_unpacklo_epi16(in[2], in[3]);
613 const __m128i tr0_2 = _mm_unpackhi_epi16(in[0], in[1]);
614 const __m128i tr0_3 = _mm_unpackhi_epi16(in[2], in[3]);
615 const __m128i tr0_4 = _mm_unpacklo_epi16(in[4], in[5]);
616 const __m128i tr0_5 = _mm_unpacklo_epi16(in[6], in[7]);
617 const __m128i tr0_6 = _mm_unpackhi_epi16(in[4], in[5]);
618 const __m128i tr0_7 = _mm_unpackhi_epi16(in[6], in[7]);
619 // 00 10 01 11 02 12 03 13
620 // 20 30 21 31 22 32 23 33
621 // 04 14 05 15 06 16 07 17
622 // 24 34 25 35 26 36 27 37
623 // 40 50 41 51 42 52 43 53
624 // 60 70 61 71 62 72 63 73
625 // 44 54 45 55 46 56 47 57
626 // 64 74 65 75 66 76 67 77
627 const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
628 const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_4, tr0_5);
629 const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
630 const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_4, tr0_5);
631 const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_2, tr0_3);
632 const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
633 const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_2, tr0_3);
634 const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
635 // 00 10 20 30 01 11 21 31
636 // 40 50 60 70 41 51 61 71
637 // 02 12 22 32 03 13 23 33
638 // 42 52 62 72 43 53 63 73
639 // 04 14 24 34 05 15 25 35
640 // 44 54 64 74 45 55 65 75
641 // 06 16 26 36 07 17 27 37
642 // 46 56 66 76 47 57 67 77
643 res[0] = _mm_unpacklo_epi64(tr1_0, tr1_1);
644 res[1] = _mm_unpackhi_epi64(tr1_0, tr1_1);
645 res[2] = _mm_unpacklo_epi64(tr1_2, tr1_3);
646 res[3] = _mm_unpackhi_epi64(tr1_2, tr1_3);
647 res[4] = _mm_unpacklo_epi64(tr1_4, tr1_5);
648 res[5] = _mm_unpackhi_epi64(tr1_4, tr1_5);
649 res[6] = _mm_unpacklo_epi64(tr1_6, tr1_7);
650 res[7] = _mm_unpackhi_epi64(tr1_6, tr1_7);
651 // 00 10 20 30 40 50 60 70
652 // 01 11 21 31 41 51 61 71
653 // 02 12 22 32 42 52 62 72
654 // 03 13 23 33 43 53 63 73
655 // 04 14 24 34 44 54 64 74
656 // 05 15 25 35 45 55 65 75
657 // 06 16 26 36 46 56 66 76
658 // 07 17 27 37 47 57 67 77
659 }
660
661 void fdct8_1d_sse2(__m128i *in) {
662 // constants
663 const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
664 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
665 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
666 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
667 const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
668 const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
669 const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
670 const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
671 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
672 __m128i u0, u1, u2, u3, u4, u5, u6, u7;
673 __m128i v0, v1, v2, v3, v4, v5, v6, v7;
674 __m128i s0, s1, s2, s3, s4, s5, s6, s7;
675
676 // stage 1
677 s0 = _mm_add_epi16(in[0], in[7]);
678 s1 = _mm_add_epi16(in[1], in[6]);
679 s2 = _mm_add_epi16(in[2], in[5]);
680 s3 = _mm_add_epi16(in[3], in[4]);
681 s4 = _mm_sub_epi16(in[3], in[4]);
682 s5 = _mm_sub_epi16(in[2], in[5]);
683 s6 = _mm_sub_epi16(in[1], in[6]);
684 s7 = _mm_sub_epi16(in[0], in[7]);
685
686 u0 = _mm_add_epi16(s0, s3);
687 u1 = _mm_add_epi16(s1, s2);
688 u2 = _mm_sub_epi16(s1, s2);
689 u3 = _mm_sub_epi16(s0, s3);
690 // interleave and perform butterfly multiplication/addition
691 v0 = _mm_unpacklo_epi16(u0, u1);
692 v1 = _mm_unpackhi_epi16(u0, u1);
693 v2 = _mm_unpacklo_epi16(u2, u3);
694 v3 = _mm_unpackhi_epi16(u2, u3);
695
696 u0 = _mm_madd_epi16(v0, k__cospi_p16_p16);
697 u1 = _mm_madd_epi16(v1, k__cospi_p16_p16);
698 u2 = _mm_madd_epi16(v0, k__cospi_p16_m16);
699 u3 = _mm_madd_epi16(v1, k__cospi_p16_m16);
700 u4 = _mm_madd_epi16(v2, k__cospi_p24_p08);
701 u5 = _mm_madd_epi16(v3, k__cospi_p24_p08);
702 u6 = _mm_madd_epi16(v2, k__cospi_m08_p24);
703 u7 = _mm_madd_epi16(v3, k__cospi_m08_p24);
704
705 // shift and rounding
706 v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
707 v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
708 v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
709 v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
710 v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
711 v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
712 v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
713 v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
714
715 u0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
716 u1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
717 u2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
718 u3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
719 u4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
720 u5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
721 u6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
722 u7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
723
724 in[0] = _mm_packs_epi32(u0, u1);
725 in[2] = _mm_packs_epi32(u4, u5);
726 in[4] = _mm_packs_epi32(u2, u3);
727 in[6] = _mm_packs_epi32(u6, u7);
728
729 // stage 2
730 // interleave and perform butterfly multiplication/addition
731 u0 = _mm_unpacklo_epi16(s6, s5);
732 u1 = _mm_unpackhi_epi16(s6, s5);
733 v0 = _mm_madd_epi16(u0, k__cospi_p16_m16);
734 v1 = _mm_madd_epi16(u1, k__cospi_p16_m16);
735 v2 = _mm_madd_epi16(u0, k__cospi_p16_p16);
736 v3 = _mm_madd_epi16(u1, k__cospi_p16_p16);
737
738 // shift and rounding
739 u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING);
740 u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING);
741 u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING);
742 u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING);
743
744 v0 = _mm_srai_epi32(u0, DCT_CONST_BITS);
745 v1 = _mm_srai_epi32(u1, DCT_CONST_BITS);
746 v2 = _mm_srai_epi32(u2, DCT_CONST_BITS);
747 v3 = _mm_srai_epi32(u3, DCT_CONST_BITS);
748
749 u0 = _mm_packs_epi32(v0, v1);
750 u1 = _mm_packs_epi32(v2, v3);
751
752 // stage 3
753 s0 = _mm_add_epi16(s4, u0);
754 s1 = _mm_sub_epi16(s4, u0);
755 s2 = _mm_sub_epi16(s7, u1);
756 s3 = _mm_add_epi16(s7, u1);
757
758 // stage 4
759 u0 = _mm_unpacklo_epi16(s0, s3);
760 u1 = _mm_unpackhi_epi16(s0, s3);
761 u2 = _mm_unpacklo_epi16(s1, s2);
762 u3 = _mm_unpackhi_epi16(s1, s2);
763
764 v0 = _mm_madd_epi16(u0, k__cospi_p28_p04);
765 v1 = _mm_madd_epi16(u1, k__cospi_p28_p04);
766 v2 = _mm_madd_epi16(u2, k__cospi_p12_p20);
767 v3 = _mm_madd_epi16(u3, k__cospi_p12_p20);
768 v4 = _mm_madd_epi16(u2, k__cospi_m20_p12);
769 v5 = _mm_madd_epi16(u3, k__cospi_m20_p12);
770 v6 = _mm_madd_epi16(u0, k__cospi_m04_p28);
771 v7 = _mm_madd_epi16(u1, k__cospi_m04_p28);
772
773 // shift and rounding
774 u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING);
775 u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING);
776 u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING);
777 u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING);
778 u4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING);
779 u5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING);
780 u6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING);
781 u7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING);
782
783 v0 = _mm_srai_epi32(u0, DCT_CONST_BITS);
784 v1 = _mm_srai_epi32(u1, DCT_CONST_BITS);
785 v2 = _mm_srai_epi32(u2, DCT_CONST_BITS);
786 v3 = _mm_srai_epi32(u3, DCT_CONST_BITS);
787 v4 = _mm_srai_epi32(u4, DCT_CONST_BITS);
788 v5 = _mm_srai_epi32(u5, DCT_CONST_BITS);
789 v6 = _mm_srai_epi32(u6, DCT_CONST_BITS);
790 v7 = _mm_srai_epi32(u7, DCT_CONST_BITS);
791
792 in[1] = _mm_packs_epi32(v0, v1);
793 in[3] = _mm_packs_epi32(v4, v5);
794 in[5] = _mm_packs_epi32(v2, v3);
795 in[7] = _mm_packs_epi32(v6, v7);
796
797 // transpose
798 array_transpose_8x8(in, in);
799 }
800
801 void fadst8_1d_sse2(__m128i *in) {
802 // Constants
803 const __m128i k__cospi_p02_p30 = pair_set_epi16(cospi_2_64, cospi_30_64);
804 const __m128i k__cospi_p30_m02 = pair_set_epi16(cospi_30_64, -cospi_2_64);
805 const __m128i k__cospi_p10_p22 = pair_set_epi16(cospi_10_64, cospi_22_64);
806 const __m128i k__cospi_p22_m10 = pair_set_epi16(cospi_22_64, -cospi_10_64);
807 const __m128i k__cospi_p18_p14 = pair_set_epi16(cospi_18_64, cospi_14_64);
808 const __m128i k__cospi_p14_m18 = pair_set_epi16(cospi_14_64, -cospi_18_64);
809 const __m128i k__cospi_p26_p06 = pair_set_epi16(cospi_26_64, cospi_6_64);
810 const __m128i k__cospi_p06_m26 = pair_set_epi16(cospi_6_64, -cospi_26_64);
811 const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64);
812 const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64);
813 const __m128i k__cospi_m24_p08 = pair_set_epi16(-cospi_24_64, cospi_8_64);
814 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
815 const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
816 const __m128i k__const_0 = _mm_set1_epi16(0);
817 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
818
819 __m128i u0, u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, u14, u15;
820 __m128i v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15;
821 __m128i w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11, w12, w13, w14, w15;
822 __m128i s0, s1, s2, s3, s4, s5, s6, s7;
823 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
824
825 // properly aligned for butterfly input
826 in0 = in[7];
827 in1 = in[0];
828 in2 = in[5];
829 in3 = in[2];
830 in4 = in[3];
831 in5 = in[4];
832 in6 = in[1];
833 in7 = in[6];
834
835 // column transformation
836 // stage 1
837 // interleave and multiply/add into 32-bit integer
838 s0 = _mm_unpacklo_epi16(in0, in1);
839 s1 = _mm_unpackhi_epi16(in0, in1);
840 s2 = _mm_unpacklo_epi16(in2, in3);
841 s3 = _mm_unpackhi_epi16(in2, in3);
842 s4 = _mm_unpacklo_epi16(in4, in5);
843 s5 = _mm_unpackhi_epi16(in4, in5);
844 s6 = _mm_unpacklo_epi16(in6, in7);
845 s7 = _mm_unpackhi_epi16(in6, in7);
846
847 u0 = _mm_madd_epi16(s0, k__cospi_p02_p30);
848 u1 = _mm_madd_epi16(s1, k__cospi_p02_p30);
849 u2 = _mm_madd_epi16(s0, k__cospi_p30_m02);
850 u3 = _mm_madd_epi16(s1, k__cospi_p30_m02);
851 u4 = _mm_madd_epi16(s2, k__cospi_p10_p22);
852 u5 = _mm_madd_epi16(s3, k__cospi_p10_p22);
853 u6 = _mm_madd_epi16(s2, k__cospi_p22_m10);
854 u7 = _mm_madd_epi16(s3, k__cospi_p22_m10);
855 u8 = _mm_madd_epi16(s4, k__cospi_p18_p14);
856 u9 = _mm_madd_epi16(s5, k__cospi_p18_p14);
857 u10 = _mm_madd_epi16(s4, k__cospi_p14_m18);
858 u11 = _mm_madd_epi16(s5, k__cospi_p14_m18);
859 u12 = _mm_madd_epi16(s6, k__cospi_p26_p06);
860 u13 = _mm_madd_epi16(s7, k__cospi_p26_p06);
861 u14 = _mm_madd_epi16(s6, k__cospi_p06_m26);
862 u15 = _mm_madd_epi16(s7, k__cospi_p06_m26);
863
864 // addition
865 w0 = _mm_add_epi32(u0, u8);
866 w1 = _mm_add_epi32(u1, u9);
867 w2 = _mm_add_epi32(u2, u10);
868 w3 = _mm_add_epi32(u3, u11);
869 w4 = _mm_add_epi32(u4, u12);
870 w5 = _mm_add_epi32(u5, u13);
871 w6 = _mm_add_epi32(u6, u14);
872 w7 = _mm_add_epi32(u7, u15);
873 w8 = _mm_sub_epi32(u0, u8);
874 w9 = _mm_sub_epi32(u1, u9);
875 w10 = _mm_sub_epi32(u2, u10);
876 w11 = _mm_sub_epi32(u3, u11);
877 w12 = _mm_sub_epi32(u4, u12);
878 w13 = _mm_sub_epi32(u5, u13);
879 w14 = _mm_sub_epi32(u6, u14);
880 w15 = _mm_sub_epi32(u7, u15);
881
882 // shift and rounding
883 v0 = _mm_add_epi32(w0, k__DCT_CONST_ROUNDING);
884 v1 = _mm_add_epi32(w1, k__DCT_CONST_ROUNDING);
885 v2 = _mm_add_epi32(w2, k__DCT_CONST_ROUNDING);
886 v3 = _mm_add_epi32(w3, k__DCT_CONST_ROUNDING);
887 v4 = _mm_add_epi32(w4, k__DCT_CONST_ROUNDING);
888 v5 = _mm_add_epi32(w5, k__DCT_CONST_ROUNDING);
889 v6 = _mm_add_epi32(w6, k__DCT_CONST_ROUNDING);
890 v7 = _mm_add_epi32(w7, k__DCT_CONST_ROUNDING);
891 v8 = _mm_add_epi32(w8, k__DCT_CONST_ROUNDING);
892 v9 = _mm_add_epi32(w9, k__DCT_CONST_ROUNDING);
893 v10 = _mm_add_epi32(w10, k__DCT_CONST_ROUNDING);
894 v11 = _mm_add_epi32(w11, k__DCT_CONST_ROUNDING);
895 v12 = _mm_add_epi32(w12, k__DCT_CONST_ROUNDING);
896 v13 = _mm_add_epi32(w13, k__DCT_CONST_ROUNDING);
897 v14 = _mm_add_epi32(w14, k__DCT_CONST_ROUNDING);
898 v15 = _mm_add_epi32(w15, k__DCT_CONST_ROUNDING);
899
900 u0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
901 u1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
902 u2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
903 u3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
904 u4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
905 u5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
906 u6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
907 u7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
908 u8 = _mm_srai_epi32(v8, DCT_CONST_BITS);
909 u9 = _mm_srai_epi32(v9, DCT_CONST_BITS);
910 u10 = _mm_srai_epi32(v10, DCT_CONST_BITS);
911 u11 = _mm_srai_epi32(v11, DCT_CONST_BITS);
912 u12 = _mm_srai_epi32(v12, DCT_CONST_BITS);
913 u13 = _mm_srai_epi32(v13, DCT_CONST_BITS);
914 u14 = _mm_srai_epi32(v14, DCT_CONST_BITS);
915 u15 = _mm_srai_epi32(v15, DCT_CONST_BITS);
916
917 // back to 16-bit and pack 8 integers into __m128i
918 in[0] = _mm_packs_epi32(u0, u1);
919 in[1] = _mm_packs_epi32(u2, u3);
920 in[2] = _mm_packs_epi32(u4, u5);
921 in[3] = _mm_packs_epi32(u6, u7);
922 in[4] = _mm_packs_epi32(u8, u9);
923 in[5] = _mm_packs_epi32(u10, u11);
924 in[6] = _mm_packs_epi32(u12, u13);
925 in[7] = _mm_packs_epi32(u14, u15);
926
927 // stage 2
928 s0 = _mm_add_epi16(in[0], in[2]);
929 s1 = _mm_add_epi16(in[1], in[3]);
930 s2 = _mm_sub_epi16(in[0], in[2]);
931 s3 = _mm_sub_epi16(in[1], in[3]);
932 u0 = _mm_unpacklo_epi16(in[4], in[5]);
933 u1 = _mm_unpackhi_epi16(in[4], in[5]);
934 u2 = _mm_unpacklo_epi16(in[6], in[7]);
935 u3 = _mm_unpackhi_epi16(in[6], in[7]);
936
937 v0 = _mm_madd_epi16(u0, k__cospi_p08_p24);
938 v1 = _mm_madd_epi16(u1, k__cospi_p08_p24);
939 v2 = _mm_madd_epi16(u0, k__cospi_p24_m08);
940 v3 = _mm_madd_epi16(u1, k__cospi_p24_m08);
941 v4 = _mm_madd_epi16(u2, k__cospi_m24_p08);
942 v5 = _mm_madd_epi16(u3, k__cospi_m24_p08);
943 v6 = _mm_madd_epi16(u2, k__cospi_p08_p24);
944 v7 = _mm_madd_epi16(u3, k__cospi_p08_p24);
945
946 w0 = _mm_add_epi32(v0, v4);
947 w1 = _mm_add_epi32(v1, v5);
948 w2 = _mm_add_epi32(v2, v6);
949 w3 = _mm_add_epi32(v3, v7);
950 w4 = _mm_sub_epi32(v0, v4);
951 w5 = _mm_sub_epi32(v1, v5);
952 w6 = _mm_sub_epi32(v2, v6);
953 w7 = _mm_sub_epi32(v3, v7);
954
955 v0 = _mm_add_epi32(w0, k__DCT_CONST_ROUNDING);
956 v1 = _mm_add_epi32(w1, k__DCT_CONST_ROUNDING);
957 v2 = _mm_add_epi32(w2, k__DCT_CONST_ROUNDING);
958 v3 = _mm_add_epi32(w3, k__DCT_CONST_ROUNDING);
959 v4 = _mm_add_epi32(w4, k__DCT_CONST_ROUNDING);
960 v5 = _mm_add_epi32(w5, k__DCT_CONST_ROUNDING);
961 v6 = _mm_add_epi32(w6, k__DCT_CONST_ROUNDING);
962 v7 = _mm_add_epi32(w7, k__DCT_CONST_ROUNDING);
963
964 u0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
965 u1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
966 u2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
967 u3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
968 u4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
969 u5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
970 u6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
971 u7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
972
973 // back to 16-bit intergers
974 s4 = _mm_packs_epi32(u0, u1);
975 s5 = _mm_packs_epi32(u2, u3);
976 s6 = _mm_packs_epi32(u4, u5);
977 s7 = _mm_packs_epi32(u6, u7);
978
979 // stage 3
980 u0 = _mm_unpacklo_epi16(s2, s3);
981 u1 = _mm_unpackhi_epi16(s2, s3);
982 u2 = _mm_unpacklo_epi16(s6, s7);
983 u3 = _mm_unpackhi_epi16(s6, s7);
984
985 v0 = _mm_madd_epi16(u0, k__cospi_p16_p16);
986 v1 = _mm_madd_epi16(u1, k__cospi_p16_p16);
987 v2 = _mm_madd_epi16(u0, k__cospi_p16_m16);
988 v3 = _mm_madd_epi16(u1, k__cospi_p16_m16);
989 v4 = _mm_madd_epi16(u2, k__cospi_p16_p16);
990 v5 = _mm_madd_epi16(u3, k__cospi_p16_p16);
991 v6 = _mm_madd_epi16(u2, k__cospi_p16_m16);
992 v7 = _mm_madd_epi16(u3, k__cospi_p16_m16);
993
994 u0 = _mm_add_epi32(v0, k__DCT_CONST_ROUNDING);
995 u1 = _mm_add_epi32(v1, k__DCT_CONST_ROUNDING);
996 u2 = _mm_add_epi32(v2, k__DCT_CONST_ROUNDING);
997 u3 = _mm_add_epi32(v3, k__DCT_CONST_ROUNDING);
998 u4 = _mm_add_epi32(v4, k__DCT_CONST_ROUNDING);
999 u5 = _mm_add_epi32(v5, k__DCT_CONST_ROUNDING);
1000 u6 = _mm_add_epi32(v6, k__DCT_CONST_ROUNDING);
1001 u7 = _mm_add_epi32(v7, k__DCT_CONST_ROUNDING);
1002
1003 v0 = _mm_srai_epi32(u0, DCT_CONST_BITS);
1004 v1 = _mm_srai_epi32(u1, DCT_CONST_BITS);
1005 v2 = _mm_srai_epi32(u2, DCT_CONST_BITS);
1006 v3 = _mm_srai_epi32(u3, DCT_CONST_BITS);
1007 v4 = _mm_srai_epi32(u4, DCT_CONST_BITS);
1008 v5 = _mm_srai_epi32(u5, DCT_CONST_BITS);
1009 v6 = _mm_srai_epi32(u6, DCT_CONST_BITS);
1010 v7 = _mm_srai_epi32(u7, DCT_CONST_BITS);
1011
1012 s2 = _mm_packs_epi32(v0, v1);
1013 s3 = _mm_packs_epi32(v2, v3);
1014 s6 = _mm_packs_epi32(v4, v5);
1015 s7 = _mm_packs_epi32(v6, v7);
1016
1017 // FIXME(jingning): do subtract using bit inversion?
1018 in[0] = s0;
1019 in[1] = _mm_sub_epi16(k__const_0, s4);
1020 in[2] = s6;
1021 in[3] = _mm_sub_epi16(k__const_0, s2);
1022 in[4] = s3;
1023 in[5] = _mm_sub_epi16(k__const_0, s7);
1024 in[6] = s5;
1025 in[7] = _mm_sub_epi16(k__const_0, s1);
1026
1027 // transpose
1028 array_transpose_8x8(in, in);
1029 }
1030
1031 void vp9_short_fht8x8_sse2(const int16_t *input, int16_t *output,
1032 int stride, int tx_type) {
1033 __m128i in[8];
1034 load_buffer_8x8(input, in, stride);
1035 switch (tx_type) {
1036 case 0: // DCT_DCT
1037 fdct8_1d_sse2(in);
1038 fdct8_1d_sse2(in);
1039 break;
1040 case 1: // ADST_DCT
1041 fadst8_1d_sse2(in);
1042 fdct8_1d_sse2(in);
1043 break;
1044 case 2: // DCT_ADST
1045 fdct8_1d_sse2(in);
1046 fadst8_1d_sse2(in);
1047 break;
1048 case 3: // ADST_ADST
1049 fadst8_1d_sse2(in);
1050 fadst8_1d_sse2(in);
1051 break;
1052 default:
1053 assert(0);
1054 break;
1055 }
1056 right_shift_8x8(in, 1);
1057 write_buffer_8x8(output, in, 8);
1058 }
1059
1060 void vp9_fdct16x16_sse2(const int16_t *input, int16_t *output, int stride) {
1061 // The 2D transform is done with two passes which are actually pretty
1062 // similar. In the first one, we transform the columns and transpose
1063 // the results. In the second one, we transform the rows. To achieve that,
1064 // as the first pass results are transposed, we tranpose the columns (that
1065 // is the transposed rows) and transpose the results (so that it goes back
1066 // in normal/row positions).
1067 int pass;
1068 // We need an intermediate buffer between passes.
1069 DECLARE_ALIGNED_ARRAY(16, int16_t, intermediate, 256);
1070 const int16_t *in = input;
1071 int16_t *out = intermediate;
1072 // Constants
1073 // When we use them, in one case, they are all the same. In all others
1074 // it's a pair of them that we need to repeat four times. This is done
1075 // by constructing the 32 bit constant corresponding to that pair.
1076 const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
1077 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
1078 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
1079 const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
1080 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
1081 const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
1082 const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
1083 const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
1084 const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
1085 const __m128i k__cospi_p30_p02 = pair_set_epi16(cospi_30_64, cospi_2_64);
1086 const __m128i k__cospi_p14_p18 = pair_set_epi16(cospi_14_64, cospi_18_64);
1087 const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64);
1088 const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64);
1089 const __m128i k__cospi_p22_p10 = pair_set_epi16(cospi_22_64, cospi_10_64);
1090 const __m128i k__cospi_p06_p26 = pair_set_epi16(cospi_6_64, cospi_26_64);
1091 const __m128i k__cospi_m10_p22 = pair_set_epi16(-cospi_10_64, cospi_22_64);
1092 const __m128i k__cospi_m26_p06 = pair_set_epi16(-cospi_26_64, cospi_6_64);
1093 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
1094 const __m128i kOne = _mm_set1_epi16(1);
1095 // Do the two transform/transpose passes
1096 for (pass = 0; pass < 2; ++pass) {
1097 // We process eight columns (transposed rows in second pass) at a time.
1098 int column_start;
1099 for (column_start = 0; column_start < 16; column_start += 8) {
1100 __m128i in00, in01, in02, in03, in04, in05, in06, in07;
1101 __m128i in08, in09, in10, in11, in12, in13, in14, in15;
1102 __m128i input0, input1, input2, input3, input4, input5, input6, input7;
1103 __m128i step1_0, step1_1, step1_2, step1_3;
1104 __m128i step1_4, step1_5, step1_6, step1_7;
1105 __m128i step2_1, step2_2, step2_3, step2_4, step2_5, step2_6;
1106 __m128i step3_0, step3_1, step3_2, step3_3;
1107 __m128i step3_4, step3_5, step3_6, step3_7;
1108 __m128i res00, res01, res02, res03, res04, res05, res06, res07;
1109 __m128i res08, res09, res10, res11, res12, res13, res14, res15;
1110 // Load and pre-condition input.
1111 if (0 == pass) {
1112 in00 = _mm_load_si128((const __m128i *)(in + 0 * stride));
1113 in01 = _mm_load_si128((const __m128i *)(in + 1 * stride));
1114 in02 = _mm_load_si128((const __m128i *)(in + 2 * stride));
1115 in03 = _mm_load_si128((const __m128i *)(in + 3 * stride));
1116 in04 = _mm_load_si128((const __m128i *)(in + 4 * stride));
1117 in05 = _mm_load_si128((const __m128i *)(in + 5 * stride));
1118 in06 = _mm_load_si128((const __m128i *)(in + 6 * stride));
1119 in07 = _mm_load_si128((const __m128i *)(in + 7 * stride));
1120 in08 = _mm_load_si128((const __m128i *)(in + 8 * stride));
1121 in09 = _mm_load_si128((const __m128i *)(in + 9 * stride));
1122 in10 = _mm_load_si128((const __m128i *)(in + 10 * stride));
1123 in11 = _mm_load_si128((const __m128i *)(in + 11 * stride));
1124 in12 = _mm_load_si128((const __m128i *)(in + 12 * stride));
1125 in13 = _mm_load_si128((const __m128i *)(in + 13 * stride));
1126 in14 = _mm_load_si128((const __m128i *)(in + 14 * stride));
1127 in15 = _mm_load_si128((const __m128i *)(in + 15 * stride));
1128 // x = x << 2
1129 in00 = _mm_slli_epi16(in00, 2);
1130 in01 = _mm_slli_epi16(in01, 2);
1131 in02 = _mm_slli_epi16(in02, 2);
1132 in03 = _mm_slli_epi16(in03, 2);
1133 in04 = _mm_slli_epi16(in04, 2);
1134 in05 = _mm_slli_epi16(in05, 2);
1135 in06 = _mm_slli_epi16(in06, 2);
1136 in07 = _mm_slli_epi16(in07, 2);
1137 in08 = _mm_slli_epi16(in08, 2);
1138 in09 = _mm_slli_epi16(in09, 2);
1139 in10 = _mm_slli_epi16(in10, 2);
1140 in11 = _mm_slli_epi16(in11, 2);
1141 in12 = _mm_slli_epi16(in12, 2);
1142 in13 = _mm_slli_epi16(in13, 2);
1143 in14 = _mm_slli_epi16(in14, 2);
1144 in15 = _mm_slli_epi16(in15, 2);
1145 } else {
1146 in00 = _mm_load_si128((const __m128i *)(in + 0 * 16));
1147 in01 = _mm_load_si128((const __m128i *)(in + 1 * 16));
1148 in02 = _mm_load_si128((const __m128i *)(in + 2 * 16));
1149 in03 = _mm_load_si128((const __m128i *)(in + 3 * 16));
1150 in04 = _mm_load_si128((const __m128i *)(in + 4 * 16));
1151 in05 = _mm_load_si128((const __m128i *)(in + 5 * 16));
1152 in06 = _mm_load_si128((const __m128i *)(in + 6 * 16));
1153 in07 = _mm_load_si128((const __m128i *)(in + 7 * 16));
1154 in08 = _mm_load_si128((const __m128i *)(in + 8 * 16));
1155 in09 = _mm_load_si128((const __m128i *)(in + 9 * 16));
1156 in10 = _mm_load_si128((const __m128i *)(in + 10 * 16));
1157 in11 = _mm_load_si128((const __m128i *)(in + 11 * 16));
1158 in12 = _mm_load_si128((const __m128i *)(in + 12 * 16));
1159 in13 = _mm_load_si128((const __m128i *)(in + 13 * 16));
1160 in14 = _mm_load_si128((const __m128i *)(in + 14 * 16));
1161 in15 = _mm_load_si128((const __m128i *)(in + 15 * 16));
1162 // x = (x + 1) >> 2
1163 in00 = _mm_add_epi16(in00, kOne);
1164 in01 = _mm_add_epi16(in01, kOne);
1165 in02 = _mm_add_epi16(in02, kOne);
1166 in03 = _mm_add_epi16(in03, kOne);
1167 in04 = _mm_add_epi16(in04, kOne);
1168 in05 = _mm_add_epi16(in05, kOne);
1169 in06 = _mm_add_epi16(in06, kOne);
1170 in07 = _mm_add_epi16(in07, kOne);
1171 in08 = _mm_add_epi16(in08, kOne);
1172 in09 = _mm_add_epi16(in09, kOne);
1173 in10 = _mm_add_epi16(in10, kOne);
1174 in11 = _mm_add_epi16(in11, kOne);
1175 in12 = _mm_add_epi16(in12, kOne);
1176 in13 = _mm_add_epi16(in13, kOne);
1177 in14 = _mm_add_epi16(in14, kOne);
1178 in15 = _mm_add_epi16(in15, kOne);
1179 in00 = _mm_srai_epi16(in00, 2);
1180 in01 = _mm_srai_epi16(in01, 2);
1181 in02 = _mm_srai_epi16(in02, 2);
1182 in03 = _mm_srai_epi16(in03, 2);
1183 in04 = _mm_srai_epi16(in04, 2);
1184 in05 = _mm_srai_epi16(in05, 2);
1185 in06 = _mm_srai_epi16(in06, 2);
1186 in07 = _mm_srai_epi16(in07, 2);
1187 in08 = _mm_srai_epi16(in08, 2);
1188 in09 = _mm_srai_epi16(in09, 2);
1189 in10 = _mm_srai_epi16(in10, 2);
1190 in11 = _mm_srai_epi16(in11, 2);
1191 in12 = _mm_srai_epi16(in12, 2);
1192 in13 = _mm_srai_epi16(in13, 2);
1193 in14 = _mm_srai_epi16(in14, 2);
1194 in15 = _mm_srai_epi16(in15, 2);
1195 }
1196 in += 8;
1197 // Calculate input for the first 8 results.
1198 {
1199 input0 = _mm_add_epi16(in00, in15);
1200 input1 = _mm_add_epi16(in01, in14);
1201 input2 = _mm_add_epi16(in02, in13);
1202 input3 = _mm_add_epi16(in03, in12);
1203 input4 = _mm_add_epi16(in04, in11);
1204 input5 = _mm_add_epi16(in05, in10);
1205 input6 = _mm_add_epi16(in06, in09);
1206 input7 = _mm_add_epi16(in07, in08);
1207 }
1208 // Calculate input for the next 8 results.
1209 {
1210 step1_0 = _mm_sub_epi16(in07, in08);
1211 step1_1 = _mm_sub_epi16(in06, in09);
1212 step1_2 = _mm_sub_epi16(in05, in10);
1213 step1_3 = _mm_sub_epi16(in04, in11);
1214 step1_4 = _mm_sub_epi16(in03, in12);
1215 step1_5 = _mm_sub_epi16(in02, in13);
1216 step1_6 = _mm_sub_epi16(in01, in14);
1217 step1_7 = _mm_sub_epi16(in00, in15);
1218 }
1219 // Work on the first eight values; fdct8_1d(input, even_results);
1220 {
1221 // Add/substract
1222 const __m128i q0 = _mm_add_epi16(input0, input7);
1223 const __m128i q1 = _mm_add_epi16(input1, input6);
1224 const __m128i q2 = _mm_add_epi16(input2, input5);
1225 const __m128i q3 = _mm_add_epi16(input3, input4);
1226 const __m128i q4 = _mm_sub_epi16(input3, input4);
1227 const __m128i q5 = _mm_sub_epi16(input2, input5);
1228 const __m128i q6 = _mm_sub_epi16(input1, input6);
1229 const __m128i q7 = _mm_sub_epi16(input0, input7);
1230 // Work on first four results
1231 {
1232 // Add/substract
1233 const __m128i r0 = _mm_add_epi16(q0, q3);
1234 const __m128i r1 = _mm_add_epi16(q1, q2);
1235 const __m128i r2 = _mm_sub_epi16(q1, q2);
1236 const __m128i r3 = _mm_sub_epi16(q0, q3);
1237 // Interleave to do the multiply by constants which gets us
1238 // into 32 bits.
1239 const __m128i t0 = _mm_unpacklo_epi16(r0, r1);
1240 const __m128i t1 = _mm_unpackhi_epi16(r0, r1);
1241 const __m128i t2 = _mm_unpacklo_epi16(r2, r3);
1242 const __m128i t3 = _mm_unpackhi_epi16(r2, r3);
1243 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
1244 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16);
1245 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_p16_m16);
1246 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_p16_m16);
1247 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p24_p08);
1248 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p24_p08);
1249 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m08_p24);
1250 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m08_p24);
1251 // dct_const_round_shift
1252 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1253 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1254 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1255 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1256 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
1257 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
1258 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
1259 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
1260 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1261 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1262 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1263 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1264 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
1265 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
1266 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
1267 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
1268 // Combine
1269 res00 = _mm_packs_epi32(w0, w1);
1270 res08 = _mm_packs_epi32(w2, w3);
1271 res04 = _mm_packs_epi32(w4, w5);
1272 res12 = _mm_packs_epi32(w6, w7);
1273 }
1274 // Work on next four results
1275 {
1276 // Interleave to do the multiply by constants which gets us
1277 // into 32 bits.
1278 const __m128i d0 = _mm_unpacklo_epi16(q6, q5);
1279 const __m128i d1 = _mm_unpackhi_epi16(q6, q5);
1280 const __m128i e0 = _mm_madd_epi16(d0, k__cospi_p16_m16);
1281 const __m128i e1 = _mm_madd_epi16(d1, k__cospi_p16_m16);
1282 const __m128i e2 = _mm_madd_epi16(d0, k__cospi_p16_p16);
1283 const __m128i e3 = _mm_madd_epi16(d1, k__cospi_p16_p16);
1284 // dct_const_round_shift
1285 const __m128i f0 = _mm_add_epi32(e0, k__DCT_CONST_ROUNDING);
1286 const __m128i f1 = _mm_add_epi32(e1, k__DCT_CONST_ROUNDING);
1287 const __m128i f2 = _mm_add_epi32(e2, k__DCT_CONST_ROUNDING);
1288 const __m128i f3 = _mm_add_epi32(e3, k__DCT_CONST_ROUNDING);
1289 const __m128i s0 = _mm_srai_epi32(f0, DCT_CONST_BITS);
1290 const __m128i s1 = _mm_srai_epi32(f1, DCT_CONST_BITS);
1291 const __m128i s2 = _mm_srai_epi32(f2, DCT_CONST_BITS);
1292 const __m128i s3 = _mm_srai_epi32(f3, DCT_CONST_BITS);
1293 // Combine
1294 const __m128i r0 = _mm_packs_epi32(s0, s1);
1295 const __m128i r1 = _mm_packs_epi32(s2, s3);
1296 // Add/substract
1297 const __m128i x0 = _mm_add_epi16(q4, r0);
1298 const __m128i x1 = _mm_sub_epi16(q4, r0);
1299 const __m128i x2 = _mm_sub_epi16(q7, r1);
1300 const __m128i x3 = _mm_add_epi16(q7, r1);
1301 // Interleave to do the multiply by constants which gets us
1302 // into 32 bits.
1303 const __m128i t0 = _mm_unpacklo_epi16(x0, x3);
1304 const __m128i t1 = _mm_unpackhi_epi16(x0, x3);
1305 const __m128i t2 = _mm_unpacklo_epi16(x1, x2);
1306 const __m128i t3 = _mm_unpackhi_epi16(x1, x2);
1307 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p28_p04);
1308 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p28_p04);
1309 const __m128i u2 = _mm_madd_epi16(t0, k__cospi_m04_p28);
1310 const __m128i u3 = _mm_madd_epi16(t1, k__cospi_m04_p28);
1311 const __m128i u4 = _mm_madd_epi16(t2, k__cospi_p12_p20);
1312 const __m128i u5 = _mm_madd_epi16(t3, k__cospi_p12_p20);
1313 const __m128i u6 = _mm_madd_epi16(t2, k__cospi_m20_p12);
1314 const __m128i u7 = _mm_madd_epi16(t3, k__cospi_m20_p12);
1315 // dct_const_round_shift
1316 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1317 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1318 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1319 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1320 const __m128i v4 = _mm_add_epi32(u4, k__DCT_CONST_ROUNDING);
1321 const __m128i v5 = _mm_add_epi32(u5, k__DCT_CONST_ROUNDING);
1322 const __m128i v6 = _mm_add_epi32(u6, k__DCT_CONST_ROUNDING);
1323 const __m128i v7 = _mm_add_epi32(u7, k__DCT_CONST_ROUNDING);
1324 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1325 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1326 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1327 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1328 const __m128i w4 = _mm_srai_epi32(v4, DCT_CONST_BITS);
1329 const __m128i w5 = _mm_srai_epi32(v5, DCT_CONST_BITS);
1330 const __m128i w6 = _mm_srai_epi32(v6, DCT_CONST_BITS);
1331 const __m128i w7 = _mm_srai_epi32(v7, DCT_CONST_BITS);
1332 // Combine
1333 res02 = _mm_packs_epi32(w0, w1);
1334 res14 = _mm_packs_epi32(w2, w3);
1335 res10 = _mm_packs_epi32(w4, w5);
1336 res06 = _mm_packs_epi32(w6, w7);
1337 }
1338 }
1339 // Work on the next eight values; step1 -> odd_results
1340 {
1341 // step 2
1342 {
1343 const __m128i t0 = _mm_unpacklo_epi16(step1_5, step1_2);
1344 const __m128i t1 = _mm_unpackhi_epi16(step1_5, step1_2);
1345 const __m128i t2 = _mm_unpacklo_epi16(step1_4, step1_3);
1346 const __m128i t3 = _mm_unpackhi_epi16(step1_4, step1_3);
1347 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_m16);
1348 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_m16);
1349 const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p16_m16);
1350 const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p16_m16);
1351 // dct_const_round_shift
1352 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1353 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1354 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1355 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1356 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1357 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1358 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1359 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1360 // Combine
1361 step2_2 = _mm_packs_epi32(w0, w1);
1362 step2_3 = _mm_packs_epi32(w2, w3);
1363 }
1364 {
1365 const __m128i t0 = _mm_unpacklo_epi16(step1_5, step1_2);
1366 const __m128i t1 = _mm_unpackhi_epi16(step1_5, step1_2);
1367 const __m128i t2 = _mm_unpacklo_epi16(step1_4, step1_3);
1368 const __m128i t3 = _mm_unpackhi_epi16(step1_4, step1_3);
1369 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p16_p16);
1370 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p16_p16);
1371 const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p16_p16);
1372 const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p16_p16);
1373 // dct_const_round_shift
1374 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1375 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1376 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1377 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1378 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1379 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1380 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1381 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1382 // Combine
1383 step2_5 = _mm_packs_epi32(w0, w1);
1384 step2_4 = _mm_packs_epi32(w2, w3);
1385 }
1386 // step 3
1387 {
1388 step3_0 = _mm_add_epi16(step1_0, step2_3);
1389 step3_1 = _mm_add_epi16(step1_1, step2_2);
1390 step3_2 = _mm_sub_epi16(step1_1, step2_2);
1391 step3_3 = _mm_sub_epi16(step1_0, step2_3);
1392 step3_4 = _mm_sub_epi16(step1_7, step2_4);
1393 step3_5 = _mm_sub_epi16(step1_6, step2_5);
1394 step3_6 = _mm_add_epi16(step1_6, step2_5);
1395 step3_7 = _mm_add_epi16(step1_7, step2_4);
1396 }
1397 // step 4
1398 {
1399 const __m128i t0 = _mm_unpacklo_epi16(step3_1, step3_6);
1400 const __m128i t1 = _mm_unpackhi_epi16(step3_1, step3_6);
1401 const __m128i t2 = _mm_unpacklo_epi16(step3_2, step3_5);
1402 const __m128i t3 = _mm_unpackhi_epi16(step3_2, step3_5);
1403 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_m08_p24);
1404 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_m08_p24);
1405 const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m24_m08);
1406 const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m24_m08);
1407 // dct_const_round_shift
1408 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1409 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1410 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1411 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1412 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1413 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1414 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1415 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1416 // Combine
1417 step2_1 = _mm_packs_epi32(w0, w1);
1418 step2_2 = _mm_packs_epi32(w2, w3);
1419 }
1420 {
1421 const __m128i t0 = _mm_unpacklo_epi16(step3_1, step3_6);
1422 const __m128i t1 = _mm_unpackhi_epi16(step3_1, step3_6);
1423 const __m128i t2 = _mm_unpacklo_epi16(step3_2, step3_5);
1424 const __m128i t3 = _mm_unpackhi_epi16(step3_2, step3_5);
1425 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p24_p08);
1426 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p24_p08);
1427 const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m08_p24);
1428 const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m08_p24);
1429 // dct_const_round_shift
1430 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1431 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1432 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1433 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1434 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1435 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1436 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1437 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1438 // Combine
1439 step2_6 = _mm_packs_epi32(w0, w1);
1440 step2_5 = _mm_packs_epi32(w2, w3);
1441 }
1442 // step 5
1443 {
1444 step1_0 = _mm_add_epi16(step3_0, step2_1);
1445 step1_1 = _mm_sub_epi16(step3_0, step2_1);
1446 step1_2 = _mm_sub_epi16(step3_3, step2_2);
1447 step1_3 = _mm_add_epi16(step3_3, step2_2);
1448 step1_4 = _mm_add_epi16(step3_4, step2_5);
1449 step1_5 = _mm_sub_epi16(step3_4, step2_5);
1450 step1_6 = _mm_sub_epi16(step3_7, step2_6);
1451 step1_7 = _mm_add_epi16(step3_7, step2_6);
1452 }
1453 // step 6
1454 {
1455 const __m128i t0 = _mm_unpacklo_epi16(step1_0, step1_7);
1456 const __m128i t1 = _mm_unpackhi_epi16(step1_0, step1_7);
1457 const __m128i t2 = _mm_unpacklo_epi16(step1_1, step1_6);
1458 const __m128i t3 = _mm_unpackhi_epi16(step1_1, step1_6);
1459 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p30_p02);
1460 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p30_p02);
1461 const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p14_p18);
1462 const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p14_p18);
1463 // dct_const_round_shift
1464 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1465 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1466 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1467 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1468 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1469 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1470 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1471 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1472 // Combine
1473 res01 = _mm_packs_epi32(w0, w1);
1474 res09 = _mm_packs_epi32(w2, w3);
1475 }
1476 {
1477 const __m128i t0 = _mm_unpacklo_epi16(step1_2, step1_5);
1478 const __m128i t1 = _mm_unpackhi_epi16(step1_2, step1_5);
1479 const __m128i t2 = _mm_unpacklo_epi16(step1_3, step1_4);
1480 const __m128i t3 = _mm_unpackhi_epi16(step1_3, step1_4);
1481 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_p22_p10);
1482 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_p22_p10);
1483 const __m128i u2 = _mm_madd_epi16(t2, k__cospi_p06_p26);
1484 const __m128i u3 = _mm_madd_epi16(t3, k__cospi_p06_p26);
1485 // dct_const_round_shift
1486 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1487 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1488 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1489 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1490 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1491 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1492 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1493 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1494 // Combine
1495 res05 = _mm_packs_epi32(w0, w1);
1496 res13 = _mm_packs_epi32(w2, w3);
1497 }
1498 {
1499 const __m128i t0 = _mm_unpacklo_epi16(step1_2, step1_5);
1500 const __m128i t1 = _mm_unpackhi_epi16(step1_2, step1_5);
1501 const __m128i t2 = _mm_unpacklo_epi16(step1_3, step1_4);
1502 const __m128i t3 = _mm_unpackhi_epi16(step1_3, step1_4);
1503 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_m10_p22);
1504 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_m10_p22);
1505 const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m26_p06);
1506 const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m26_p06);
1507 // dct_const_round_shift
1508 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1509 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1510 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1511 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1512 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1513 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1514 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1515 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1516 // Combine
1517 res11 = _mm_packs_epi32(w0, w1);
1518 res03 = _mm_packs_epi32(w2, w3);
1519 }
1520 {
1521 const __m128i t0 = _mm_unpacklo_epi16(step1_0, step1_7);
1522 const __m128i t1 = _mm_unpackhi_epi16(step1_0, step1_7);
1523 const __m128i t2 = _mm_unpacklo_epi16(step1_1, step1_6);
1524 const __m128i t3 = _mm_unpackhi_epi16(step1_1, step1_6);
1525 const __m128i u0 = _mm_madd_epi16(t0, k__cospi_m02_p30);
1526 const __m128i u1 = _mm_madd_epi16(t1, k__cospi_m02_p30);
1527 const __m128i u2 = _mm_madd_epi16(t2, k__cospi_m18_p14);
1528 const __m128i u3 = _mm_madd_epi16(t3, k__cospi_m18_p14);
1529 // dct_const_round_shift
1530 const __m128i v0 = _mm_add_epi32(u0, k__DCT_CONST_ROUNDING);
1531 const __m128i v1 = _mm_add_epi32(u1, k__DCT_CONST_ROUNDING);
1532 const __m128i v2 = _mm_add_epi32(u2, k__DCT_CONST_ROUNDING);
1533 const __m128i v3 = _mm_add_epi32(u3, k__DCT_CONST_ROUNDING);
1534 const __m128i w0 = _mm_srai_epi32(v0, DCT_CONST_BITS);
1535 const __m128i w1 = _mm_srai_epi32(v1, DCT_CONST_BITS);
1536 const __m128i w2 = _mm_srai_epi32(v2, DCT_CONST_BITS);
1537 const __m128i w3 = _mm_srai_epi32(v3, DCT_CONST_BITS);
1538 // Combine
1539 res15 = _mm_packs_epi32(w0, w1);
1540 res07 = _mm_packs_epi32(w2, w3);
1541 }
1542 }
1543 // Transpose the results, do it as two 8x8 transposes.
1544 {
1545 // 00 01 02 03 04 05 06 07
1546 // 10 11 12 13 14 15 16 17
1547 // 20 21 22 23 24 25 26 27
1548 // 30 31 32 33 34 35 36 37
1549 // 40 41 42 43 44 45 46 47
1550 // 50 51 52 53 54 55 56 57
1551 // 60 61 62 63 64 65 66 67
1552 // 70 71 72 73 74 75 76 77
1553 const __m128i tr0_0 = _mm_unpacklo_epi16(res00, res01);
1554 const __m128i tr0_1 = _mm_unpacklo_epi16(res02, res03);
1555 const __m128i tr0_2 = _mm_unpackhi_epi16(res00, res01);
1556 const __m128i tr0_3 = _mm_unpackhi_epi16(res02, res03);
1557 const __m128i tr0_4 = _mm_unpacklo_epi16(res04, res05);
1558 const __m128i tr0_5 = _mm_unpacklo_epi16(res06, res07);
1559 const __m128i tr0_6 = _mm_unpackhi_epi16(res04, res05);
1560 const __m128i tr0_7 = _mm_unpackhi_epi16(res06, res07);
1561 // 00 10 01 11 02 12 03 13
1562 // 20 30 21 31 22 32 23 33
1563 // 04 14 05 15 06 16 07 17
1564 // 24 34 25 35 26 36 27 37
1565 // 40 50 41 51 42 52 43 53
1566 // 60 70 61 71 62 72 63 73
1567 // 54 54 55 55 56 56 57 57
1568 // 64 74 65 75 66 76 67 77
1569 const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
1570 const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3);
1571 const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
1572 const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3);
1573 const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5);
1574 const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
1575 const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5);
1576 const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
1577 // 00 10 20 30 01 11 21 31
1578 // 40 50 60 70 41 51 61 71
1579 // 02 12 22 32 03 13 23 33
1580 // 42 52 62 72 43 53 63 73
1581 // 04 14 24 34 05 15 21 36
1582 // 44 54 64 74 45 55 61 76
1583 // 06 16 26 36 07 17 27 37
1584 // 46 56 66 76 47 57 67 77
1585 const __m128i tr2_0 = _mm_unpacklo_epi64(tr1_0, tr1_4);
1586 const __m128i tr2_1 = _mm_unpackhi_epi64(tr1_0, tr1_4);
1587 const __m128i tr2_2 = _mm_unpacklo_epi64(tr1_2, tr1_6);
1588 const __m128i tr2_3 = _mm_unpackhi_epi64(tr1_2, tr1_6);
1589 const __m128i tr2_4 = _mm_unpacklo_epi64(tr1_1, tr1_5);
1590 const __m128i tr2_5 = _mm_unpackhi_epi64(tr1_1, tr1_5);
1591 const __m128i tr2_6 = _mm_unpacklo_epi64(tr1_3, tr1_7);
1592 const __m128i tr2_7 = _mm_unpackhi_epi64(tr1_3, tr1_7);
1593 // 00 10 20 30 40 50 60 70
1594 // 01 11 21 31 41 51 61 71
1595 // 02 12 22 32 42 52 62 72
1596 // 03 13 23 33 43 53 63 73
1597 // 04 14 24 34 44 54 64 74
1598 // 05 15 25 35 45 55 65 75
1599 // 06 16 26 36 46 56 66 76
1600 // 07 17 27 37 47 57 67 77
1601 _mm_storeu_si128((__m128i *)(out + 0 * 16), tr2_0);
1602 _mm_storeu_si128((__m128i *)(out + 1 * 16), tr2_1);
1603 _mm_storeu_si128((__m128i *)(out + 2 * 16), tr2_2);
1604 _mm_storeu_si128((__m128i *)(out + 3 * 16), tr2_3);
1605 _mm_storeu_si128((__m128i *)(out + 4 * 16), tr2_4);
1606 _mm_storeu_si128((__m128i *)(out + 5 * 16), tr2_5);
1607 _mm_storeu_si128((__m128i *)(out + 6 * 16), tr2_6);
1608 _mm_storeu_si128((__m128i *)(out + 7 * 16), tr2_7);
1609 }
1610 {
1611 // 00 01 02 03 04 05 06 07
1612 // 10 11 12 13 14 15 16 17
1613 // 20 21 22 23 24 25 26 27
1614 // 30 31 32 33 34 35 36 37
1615 // 40 41 42 43 44 45 46 47
1616 // 50 51 52 53 54 55 56 57
1617 // 60 61 62 63 64 65 66 67
1618 // 70 71 72 73 74 75 76 77
1619 const __m128i tr0_0 = _mm_unpacklo_epi16(res08, res09);
1620 const __m128i tr0_1 = _mm_unpacklo_epi16(res10, res11);
1621 const __m128i tr0_2 = _mm_unpackhi_epi16(res08, res09);
1622 const __m128i tr0_3 = _mm_unpackhi_epi16(res10, res11);
1623 const __m128i tr0_4 = _mm_unpacklo_epi16(res12, res13);
1624 const __m128i tr0_5 = _mm_unpacklo_epi16(res14, res15);
1625 const __m128i tr0_6 = _mm_unpackhi_epi16(res12, res13);
1626 const __m128i tr0_7 = _mm_unpackhi_epi16(res14, res15);
1627 // 00 10 01 11 02 12 03 13
1628 // 20 30 21 31 22 32 23 33
1629 // 04 14 05 15 06 16 07 17
1630 // 24 34 25 35 26 36 27 37
1631 // 40 50 41 51 42 52 43 53
1632 // 60 70 61 71 62 72 63 73
1633 // 54 54 55 55 56 56 57 57
1634 // 64 74 65 75 66 76 67 77
1635 const __m128i tr1_0 = _mm_unpacklo_epi32(tr0_0, tr0_1);
1636 const __m128i tr1_1 = _mm_unpacklo_epi32(tr0_2, tr0_3);
1637 const __m128i tr1_2 = _mm_unpackhi_epi32(tr0_0, tr0_1);
1638 const __m128i tr1_3 = _mm_unpackhi_epi32(tr0_2, tr0_3);
1639 const __m128i tr1_4 = _mm_unpacklo_epi32(tr0_4, tr0_5);
1640 const __m128i tr1_5 = _mm_unpacklo_epi32(tr0_6, tr0_7);
1641 const __m128i tr1_6 = _mm_unpackhi_epi32(tr0_4, tr0_5);
1642 const __m128i tr1_7 = _mm_unpackhi_epi32(tr0_6, tr0_7);
1643 // 00 10 20 30 01 11 21 31
1644 // 40 50 60 70 41 51 61 71
1645 // 02 12 22 32 03 13 23 33
1646 // 42 52 62 72 43 53 63 73
1647 // 04 14 24 34 05 15 21 36
1648 // 44 54 64 74 45 55 61 76
1649 // 06 16 26 36 07 17 27 37
1650 // 46 56 66 76 47 57 67 77
1651 const __m128i tr2_0 = _mm_unpacklo_epi64(tr1_0, tr1_4);
1652 const __m128i tr2_1 = _mm_unpackhi_epi64(tr1_0, tr1_4);
1653 const __m128i tr2_2 = _mm_unpacklo_epi64(tr1_2, tr1_6);
1654 const __m128i tr2_3 = _mm_unpackhi_epi64(tr1_2, tr1_6);
1655 const __m128i tr2_4 = _mm_unpacklo_epi64(tr1_1, tr1_5);
1656 const __m128i tr2_5 = _mm_unpackhi_epi64(tr1_1, tr1_5);
1657 const __m128i tr2_6 = _mm_unpacklo_epi64(tr1_3, tr1_7);
1658 const __m128i tr2_7 = _mm_unpackhi_epi64(tr1_3, tr1_7);
1659 // 00 10 20 30 40 50 60 70
1660 // 01 11 21 31 41 51 61 71
1661 // 02 12 22 32 42 52 62 72
1662 // 03 13 23 33 43 53 63 73
1663 // 04 14 24 34 44 54 64 74
1664 // 05 15 25 35 45 55 65 75
1665 // 06 16 26 36 46 56 66 76
1666 // 07 17 27 37 47 57 67 77
1667 // Store results
1668 _mm_store_si128((__m128i *)(out + 8 + 0 * 16), tr2_0);
1669 _mm_store_si128((__m128i *)(out + 8 + 1 * 16), tr2_1);
1670 _mm_store_si128((__m128i *)(out + 8 + 2 * 16), tr2_2);
1671 _mm_store_si128((__m128i *)(out + 8 + 3 * 16), tr2_3);
1672 _mm_store_si128((__m128i *)(out + 8 + 4 * 16), tr2_4);
1673 _mm_store_si128((__m128i *)(out + 8 + 5 * 16), tr2_5);
1674 _mm_store_si128((__m128i *)(out + 8 + 6 * 16), tr2_6);
1675 _mm_store_si128((__m128i *)(out + 8 + 7 * 16), tr2_7);
1676 }
1677 out += 8*16;
1678 }
1679 // Setup in/out for next pass.
1680 in = intermediate;
1681 out = output;
1682 }
1683 }
1684
1685 static INLINE void load_buffer_16x16(const int16_t* input, __m128i *in0,
1686 __m128i *in1, int stride) {
1687 // load first 8 columns
1688 load_buffer_8x8(input, in0, stride);
1689 load_buffer_8x8(input + 8 * stride, in0 + 8, stride);
1690
1691 input += 8;
1692 // load second 8 columns
1693 load_buffer_8x8(input, in1, stride);
1694 load_buffer_8x8(input + 8 * stride, in1 + 8, stride);
1695 }
1696
1697 static INLINE void write_buffer_16x16(int16_t *output, __m128i *in0,
1698 __m128i *in1, int stride) {
1699 // write first 8 columns
1700 write_buffer_8x8(output, in0, stride);
1701 write_buffer_8x8(output + 8 * stride, in0 + 8, stride);
1702 // write second 8 columns
1703 output += 8;
1704 write_buffer_8x8(output, in1, stride);
1705 write_buffer_8x8(output + 8 * stride, in1 + 8, stride);
1706 }
1707
1708 static INLINE void array_transpose_16x16(__m128i *res0, __m128i *res1) {
1709 __m128i tbuf[8];
1710 array_transpose_8x8(res0, res0);
1711 array_transpose_8x8(res1, tbuf);
1712 array_transpose_8x8(res0 + 8, res1);
1713 array_transpose_8x8(res1 + 8, res1 + 8);
1714
1715 res0[8] = tbuf[0];
1716 res0[9] = tbuf[1];
1717 res0[10] = tbuf[2];
1718 res0[11] = tbuf[3];
1719 res0[12] = tbuf[4];
1720 res0[13] = tbuf[5];
1721 res0[14] = tbuf[6];
1722 res0[15] = tbuf[7];
1723 }
1724
1725 static INLINE void right_shift_16x16(__m128i *res0, __m128i *res1) {
1726 // perform rounding operations
1727 right_shift_8x8(res0, 2);
1728 right_shift_8x8(res0 + 8, 2);
1729 right_shift_8x8(res1, 2);
1730 right_shift_8x8(res1 + 8, 2);
1731 }
1732
1733 void fdct16_1d_8col(__m128i *in) {
1734 // perform 16x16 1-D DCT for 8 columns
1735 __m128i i[8], s[8], p[8], t[8], u[16], v[16];
1736 const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
1737 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
1738 const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
1739 const __m128i k__cospi_p24_p08 = pair_set_epi16(cospi_24_64, cospi_8_64);
1740 const __m128i k__cospi_m24_m08 = pair_set_epi16(-cospi_24_64, -cospi_8_64);
1741 const __m128i k__cospi_m08_p24 = pair_set_epi16(-cospi_8_64, cospi_24_64);
1742 const __m128i k__cospi_p28_p04 = pair_set_epi16(cospi_28_64, cospi_4_64);
1743 const __m128i k__cospi_m04_p28 = pair_set_epi16(-cospi_4_64, cospi_28_64);
1744 const __m128i k__cospi_p12_p20 = pair_set_epi16(cospi_12_64, cospi_20_64);
1745 const __m128i k__cospi_m20_p12 = pair_set_epi16(-cospi_20_64, cospi_12_64);
1746 const __m128i k__cospi_p30_p02 = pair_set_epi16(cospi_30_64, cospi_2_64);
1747 const __m128i k__cospi_p14_p18 = pair_set_epi16(cospi_14_64, cospi_18_64);
1748 const __m128i k__cospi_m02_p30 = pair_set_epi16(-cospi_2_64, cospi_30_64);
1749 const __m128i k__cospi_m18_p14 = pair_set_epi16(-cospi_18_64, cospi_14_64);
1750 const __m128i k__cospi_p22_p10 = pair_set_epi16(cospi_22_64, cospi_10_64);
1751 const __m128i k__cospi_p06_p26 = pair_set_epi16(cospi_6_64, cospi_26_64);
1752 const __m128i k__cospi_m10_p22 = pair_set_epi16(-cospi_10_64, cospi_22_64);
1753 const __m128i k__cospi_m26_p06 = pair_set_epi16(-cospi_26_64, cospi_6_64);
1754 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
1755
1756 // stage 1
1757 i[0] = _mm_add_epi16(in[0], in[15]);
1758 i[1] = _mm_add_epi16(in[1], in[14]);
1759 i[2] = _mm_add_epi16(in[2], in[13]);
1760 i[3] = _mm_add_epi16(in[3], in[12]);
1761 i[4] = _mm_add_epi16(in[4], in[11]);
1762 i[5] = _mm_add_epi16(in[5], in[10]);
1763 i[6] = _mm_add_epi16(in[6], in[9]);
1764 i[7] = _mm_add_epi16(in[7], in[8]);
1765
1766 s[0] = _mm_sub_epi16(in[7], in[8]);
1767 s[1] = _mm_sub_epi16(in[6], in[9]);
1768 s[2] = _mm_sub_epi16(in[5], in[10]);
1769 s[3] = _mm_sub_epi16(in[4], in[11]);
1770 s[4] = _mm_sub_epi16(in[3], in[12]);
1771 s[5] = _mm_sub_epi16(in[2], in[13]);
1772 s[6] = _mm_sub_epi16(in[1], in[14]);
1773 s[7] = _mm_sub_epi16(in[0], in[15]);
1774
1775 p[0] = _mm_add_epi16(i[0], i[7]);
1776 p[1] = _mm_add_epi16(i[1], i[6]);
1777 p[2] = _mm_add_epi16(i[2], i[5]);
1778 p[3] = _mm_add_epi16(i[3], i[4]);
1779 p[4] = _mm_sub_epi16(i[3], i[4]);
1780 p[5] = _mm_sub_epi16(i[2], i[5]);
1781 p[6] = _mm_sub_epi16(i[1], i[6]);
1782 p[7] = _mm_sub_epi16(i[0], i[7]);
1783
1784 u[0] = _mm_add_epi16(p[0], p[3]);
1785 u[1] = _mm_add_epi16(p[1], p[2]);
1786 u[2] = _mm_sub_epi16(p[1], p[2]);
1787 u[3] = _mm_sub_epi16(p[0], p[3]);
1788
1789 v[0] = _mm_unpacklo_epi16(u[0], u[1]);
1790 v[1] = _mm_unpackhi_epi16(u[0], u[1]);
1791 v[2] = _mm_unpacklo_epi16(u[2], u[3]);
1792 v[3] = _mm_unpackhi_epi16(u[2], u[3]);
1793
1794 u[0] = _mm_madd_epi16(v[0], k__cospi_p16_p16);
1795 u[1] = _mm_madd_epi16(v[1], k__cospi_p16_p16);
1796 u[2] = _mm_madd_epi16(v[0], k__cospi_p16_m16);
1797 u[3] = _mm_madd_epi16(v[1], k__cospi_p16_m16);
1798 u[4] = _mm_madd_epi16(v[2], k__cospi_p24_p08);
1799 u[5] = _mm_madd_epi16(v[3], k__cospi_p24_p08);
1800 u[6] = _mm_madd_epi16(v[2], k__cospi_m08_p24);
1801 u[7] = _mm_madd_epi16(v[3], k__cospi_m08_p24);
1802
1803 v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
1804 v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
1805 v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
1806 v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING);
1807 v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING);
1808 v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING);
1809 v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING);
1810 v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING);
1811
1812 u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS);
1813 u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS);
1814 u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS);
1815 u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS);
1816 u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS);
1817 u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS);
1818 u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS);
1819 u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS);
1820
1821 in[0] = _mm_packs_epi32(u[0], u[1]);
1822 in[4] = _mm_packs_epi32(u[4], u[5]);
1823 in[8] = _mm_packs_epi32(u[2], u[3]);
1824 in[12] = _mm_packs_epi32(u[6], u[7]);
1825
1826 u[0] = _mm_unpacklo_epi16(p[5], p[6]);
1827 u[1] = _mm_unpackhi_epi16(p[5], p[6]);
1828 v[0] = _mm_madd_epi16(u[0], k__cospi_m16_p16);
1829 v[1] = _mm_madd_epi16(u[1], k__cospi_m16_p16);
1830 v[2] = _mm_madd_epi16(u[0], k__cospi_p16_p16);
1831 v[3] = _mm_madd_epi16(u[1], k__cospi_p16_p16);
1832
1833 u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING);
1834 u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING);
1835 u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING);
1836 u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING);
1837
1838 v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS);
1839 v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS);
1840 v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS);
1841 v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS);
1842
1843 u[0] = _mm_packs_epi32(v[0], v[1]);
1844 u[1] = _mm_packs_epi32(v[2], v[3]);
1845
1846 t[0] = _mm_add_epi16(p[4], u[0]);
1847 t[1] = _mm_sub_epi16(p[4], u[0]);
1848 t[2] = _mm_sub_epi16(p[7], u[1]);
1849 t[3] = _mm_add_epi16(p[7], u[1]);
1850
1851 u[0] = _mm_unpacklo_epi16(t[0], t[3]);
1852 u[1] = _mm_unpackhi_epi16(t[0], t[3]);
1853 u[2] = _mm_unpacklo_epi16(t[1], t[2]);
1854 u[3] = _mm_unpackhi_epi16(t[1], t[2]);
1855
1856 v[0] = _mm_madd_epi16(u[0], k__cospi_p28_p04);
1857 v[1] = _mm_madd_epi16(u[1], k__cospi_p28_p04);
1858 v[2] = _mm_madd_epi16(u[2], k__cospi_p12_p20);
1859 v[3] = _mm_madd_epi16(u[3], k__cospi_p12_p20);
1860 v[4] = _mm_madd_epi16(u[2], k__cospi_m20_p12);
1861 v[5] = _mm_madd_epi16(u[3], k__cospi_m20_p12);
1862 v[6] = _mm_madd_epi16(u[0], k__cospi_m04_p28);
1863 v[7] = _mm_madd_epi16(u[1], k__cospi_m04_p28);
1864
1865 u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING);
1866 u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING);
1867 u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING);
1868 u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING);
1869 u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING);
1870 u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING);
1871 u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING);
1872 u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING);
1873
1874 v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS);
1875 v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS);
1876 v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS);
1877 v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS);
1878 v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS);
1879 v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS);
1880 v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS);
1881 v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS);
1882
1883 in[2] = _mm_packs_epi32(v[0], v[1]);
1884 in[6] = _mm_packs_epi32(v[4], v[5]);
1885 in[10] = _mm_packs_epi32(v[2], v[3]);
1886 in[14] = _mm_packs_epi32(v[6], v[7]);
1887
1888 // stage 2
1889 u[0] = _mm_unpacklo_epi16(s[2], s[5]);
1890 u[1] = _mm_unpackhi_epi16(s[2], s[5]);
1891 u[2] = _mm_unpacklo_epi16(s[3], s[4]);
1892 u[3] = _mm_unpackhi_epi16(s[3], s[4]);
1893
1894 v[0] = _mm_madd_epi16(u[0], k__cospi_m16_p16);
1895 v[1] = _mm_madd_epi16(u[1], k__cospi_m16_p16);
1896 v[2] = _mm_madd_epi16(u[2], k__cospi_m16_p16);
1897 v[3] = _mm_madd_epi16(u[3], k__cospi_m16_p16);
1898 v[4] = _mm_madd_epi16(u[2], k__cospi_p16_p16);
1899 v[5] = _mm_madd_epi16(u[3], k__cospi_p16_p16);
1900 v[6] = _mm_madd_epi16(u[0], k__cospi_p16_p16);
1901 v[7] = _mm_madd_epi16(u[1], k__cospi_p16_p16);
1902
1903 u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING);
1904 u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING);
1905 u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING);
1906 u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING);
1907 u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING);
1908 u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING);
1909 u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING);
1910 u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING);
1911
1912 v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS);
1913 v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS);
1914 v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS);
1915 v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS);
1916 v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS);
1917 v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS);
1918 v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS);
1919 v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS);
1920
1921 t[2] = _mm_packs_epi32(v[0], v[1]);
1922 t[3] = _mm_packs_epi32(v[2], v[3]);
1923 t[4] = _mm_packs_epi32(v[4], v[5]);
1924 t[5] = _mm_packs_epi32(v[6], v[7]);
1925
1926 // stage 3
1927 p[0] = _mm_add_epi16(s[0], t[3]);
1928 p[1] = _mm_add_epi16(s[1], t[2]);
1929 p[2] = _mm_sub_epi16(s[1], t[2]);
1930 p[3] = _mm_sub_epi16(s[0], t[3]);
1931 p[4] = _mm_sub_epi16(s[7], t[4]);
1932 p[5] = _mm_sub_epi16(s[6], t[5]);
1933 p[6] = _mm_add_epi16(s[6], t[5]);
1934 p[7] = _mm_add_epi16(s[7], t[4]);
1935
1936 // stage 4
1937 u[0] = _mm_unpacklo_epi16(p[1], p[6]);
1938 u[1] = _mm_unpackhi_epi16(p[1], p[6]);
1939 u[2] = _mm_unpacklo_epi16(p[2], p[5]);
1940 u[3] = _mm_unpackhi_epi16(p[2], p[5]);
1941
1942 v[0] = _mm_madd_epi16(u[0], k__cospi_m08_p24);
1943 v[1] = _mm_madd_epi16(u[1], k__cospi_m08_p24);
1944 v[2] = _mm_madd_epi16(u[2], k__cospi_m24_m08);
1945 v[3] = _mm_madd_epi16(u[3], k__cospi_m24_m08);
1946 v[4] = _mm_madd_epi16(u[2], k__cospi_m08_p24);
1947 v[5] = _mm_madd_epi16(u[3], k__cospi_m08_p24);
1948 v[6] = _mm_madd_epi16(u[0], k__cospi_p24_p08);
1949 v[7] = _mm_madd_epi16(u[1], k__cospi_p24_p08);
1950
1951 u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING);
1952 u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING);
1953 u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING);
1954 u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING);
1955 u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING);
1956 u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING);
1957 u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING);
1958 u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING);
1959
1960 v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS);
1961 v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS);
1962 v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS);
1963 v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS);
1964 v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS);
1965 v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS);
1966 v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS);
1967 v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS);
1968
1969 t[1] = _mm_packs_epi32(v[0], v[1]);
1970 t[2] = _mm_packs_epi32(v[2], v[3]);
1971 t[5] = _mm_packs_epi32(v[4], v[5]);
1972 t[6] = _mm_packs_epi32(v[6], v[7]);
1973
1974 // stage 5
1975 s[0] = _mm_add_epi16(p[0], t[1]);
1976 s[1] = _mm_sub_epi16(p[0], t[1]);
1977 s[2] = _mm_sub_epi16(p[3], t[2]);
1978 s[3] = _mm_add_epi16(p[3], t[2]);
1979 s[4] = _mm_add_epi16(p[4], t[5]);
1980 s[5] = _mm_sub_epi16(p[4], t[5]);
1981 s[6] = _mm_sub_epi16(p[7], t[6]);
1982 s[7] = _mm_add_epi16(p[7], t[6]);
1983
1984 // stage 6
1985 u[0] = _mm_unpacklo_epi16(s[0], s[7]);
1986 u[1] = _mm_unpackhi_epi16(s[0], s[7]);
1987 u[2] = _mm_unpacklo_epi16(s[1], s[6]);
1988 u[3] = _mm_unpackhi_epi16(s[1], s[6]);
1989 u[4] = _mm_unpacklo_epi16(s[2], s[5]);
1990 u[5] = _mm_unpackhi_epi16(s[2], s[5]);
1991 u[6] = _mm_unpacklo_epi16(s[3], s[4]);
1992 u[7] = _mm_unpackhi_epi16(s[3], s[4]);
1993
1994 v[0] = _mm_madd_epi16(u[0], k__cospi_p30_p02);
1995 v[1] = _mm_madd_epi16(u[1], k__cospi_p30_p02);
1996 v[2] = _mm_madd_epi16(u[2], k__cospi_p14_p18);
1997 v[3] = _mm_madd_epi16(u[3], k__cospi_p14_p18);
1998 v[4] = _mm_madd_epi16(u[4], k__cospi_p22_p10);
1999 v[5] = _mm_madd_epi16(u[5], k__cospi_p22_p10);
2000 v[6] = _mm_madd_epi16(u[6], k__cospi_p06_p26);
2001 v[7] = _mm_madd_epi16(u[7], k__cospi_p06_p26);
2002 v[8] = _mm_madd_epi16(u[6], k__cospi_m26_p06);
2003 v[9] = _mm_madd_epi16(u[7], k__cospi_m26_p06);
2004 v[10] = _mm_madd_epi16(u[4], k__cospi_m10_p22);
2005 v[11] = _mm_madd_epi16(u[5], k__cospi_m10_p22);
2006 v[12] = _mm_madd_epi16(u[2], k__cospi_m18_p14);
2007 v[13] = _mm_madd_epi16(u[3], k__cospi_m18_p14);
2008 v[14] = _mm_madd_epi16(u[0], k__cospi_m02_p30);
2009 v[15] = _mm_madd_epi16(u[1], k__cospi_m02_p30);
2010
2011 u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING);
2012 u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING);
2013 u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING);
2014 u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING);
2015 u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING);
2016 u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING);
2017 u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING);
2018 u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING);
2019 u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING);
2020 u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING);
2021 u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING);
2022 u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING);
2023 u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING);
2024 u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING);
2025 u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING);
2026 u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING);
2027
2028 v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS);
2029 v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS);
2030 v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS);
2031 v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS);
2032 v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS);
2033 v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS);
2034 v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS);
2035 v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS);
2036 v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS);
2037 v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS);
2038 v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS);
2039 v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS);
2040 v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS);
2041 v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS);
2042 v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS);
2043 v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS);
2044
2045 in[1] = _mm_packs_epi32(v[0], v[1]);
2046 in[9] = _mm_packs_epi32(v[2], v[3]);
2047 in[5] = _mm_packs_epi32(v[4], v[5]);
2048 in[13] = _mm_packs_epi32(v[6], v[7]);
2049 in[3] = _mm_packs_epi32(v[8], v[9]);
2050 in[11] = _mm_packs_epi32(v[10], v[11]);
2051 in[7] = _mm_packs_epi32(v[12], v[13]);
2052 in[15] = _mm_packs_epi32(v[14], v[15]);
2053 }
2054
2055 void fadst16_1d_8col(__m128i *in) {
2056 // perform 16x16 1-D ADST for 8 columns
2057 __m128i s[16], x[16], u[32], v[32];
2058 const __m128i k__cospi_p01_p31 = pair_set_epi16(cospi_1_64, cospi_31_64);
2059 const __m128i k__cospi_p31_m01 = pair_set_epi16(cospi_31_64, -cospi_1_64);
2060 const __m128i k__cospi_p05_p27 = pair_set_epi16(cospi_5_64, cospi_27_64);
2061 const __m128i k__cospi_p27_m05 = pair_set_epi16(cospi_27_64, -cospi_5_64);
2062 const __m128i k__cospi_p09_p23 = pair_set_epi16(cospi_9_64, cospi_23_64);
2063 const __m128i k__cospi_p23_m09 = pair_set_epi16(cospi_23_64, -cospi_9_64);
2064 const __m128i k__cospi_p13_p19 = pair_set_epi16(cospi_13_64, cospi_19_64);
2065 const __m128i k__cospi_p19_m13 = pair_set_epi16(cospi_19_64, -cospi_13_64);
2066 const __m128i k__cospi_p17_p15 = pair_set_epi16(cospi_17_64, cospi_15_64);
2067 const __m128i k__cospi_p15_m17 = pair_set_epi16(cospi_15_64, -cospi_17_64);
2068 const __m128i k__cospi_p21_p11 = pair_set_epi16(cospi_21_64, cospi_11_64);
2069 const __m128i k__cospi_p11_m21 = pair_set_epi16(cospi_11_64, -cospi_21_64);
2070 const __m128i k__cospi_p25_p07 = pair_set_epi16(cospi_25_64, cospi_7_64);
2071 const __m128i k__cospi_p07_m25 = pair_set_epi16(cospi_7_64, -cospi_25_64);
2072 const __m128i k__cospi_p29_p03 = pair_set_epi16(cospi_29_64, cospi_3_64);
2073 const __m128i k__cospi_p03_m29 = pair_set_epi16(cospi_3_64, -cospi_29_64);
2074 const __m128i k__cospi_p04_p28 = pair_set_epi16(cospi_4_64, cospi_28_64);
2075 const __m128i k__cospi_p28_m04 = pair_set_epi16(cospi_28_64, -cospi_4_64);
2076 const __m128i k__cospi_p20_p12 = pair_set_epi16(cospi_20_64, cospi_12_64);
2077 const __m128i k__cospi_p12_m20 = pair_set_epi16(cospi_12_64, -cospi_20_64);
2078 const __m128i k__cospi_m28_p04 = pair_set_epi16(-cospi_28_64, cospi_4_64);
2079 const __m128i k__cospi_m12_p20 = pair_set_epi16(-cospi_12_64, cospi_20_64);
2080 const __m128i k__cospi_p08_p24 = pair_set_epi16(cospi_8_64, cospi_24_64);
2081 const __m128i k__cospi_p24_m08 = pair_set_epi16(cospi_24_64, -cospi_8_64);
2082 const __m128i k__cospi_m24_p08 = pair_set_epi16(-cospi_24_64, cospi_8_64);
2083 const __m128i k__cospi_m16_m16 = _mm_set1_epi16(-cospi_16_64);
2084 const __m128i k__cospi_p16_p16 = _mm_set1_epi16(cospi_16_64);
2085 const __m128i k__cospi_p16_m16 = pair_set_epi16(cospi_16_64, -cospi_16_64);
2086 const __m128i k__cospi_m16_p16 = pair_set_epi16(-cospi_16_64, cospi_16_64);
2087 const __m128i k__DCT_CONST_ROUNDING = _mm_set1_epi32(DCT_CONST_ROUNDING);
2088 const __m128i kZero = _mm_set1_epi16(0);
2089
2090 u[0] = _mm_unpacklo_epi16(in[15], in[0]);
2091 u[1] = _mm_unpackhi_epi16(in[15], in[0]);
2092 u[2] = _mm_unpacklo_epi16(in[13], in[2]);
2093 u[3] = _mm_unpackhi_epi16(in[13], in[2]);
2094 u[4] = _mm_unpacklo_epi16(in[11], in[4]);
2095 u[5] = _mm_unpackhi_epi16(in[11], in[4]);
2096 u[6] = _mm_unpacklo_epi16(in[9], in[6]);
2097 u[7] = _mm_unpackhi_epi16(in[9], in[6]);
2098 u[8] = _mm_unpacklo_epi16(in[7], in[8]);
2099 u[9] = _mm_unpackhi_epi16(in[7], in[8]);
2100 u[10] = _mm_unpacklo_epi16(in[5], in[10]);
2101 u[11] = _mm_unpackhi_epi16(in[5], in[10]);
2102 u[12] = _mm_unpacklo_epi16(in[3], in[12]);
2103 u[13] = _mm_unpackhi_epi16(in[3], in[12]);
2104 u[14] = _mm_unpacklo_epi16(in[1], in[14]);
2105 u[15] = _mm_unpackhi_epi16(in[1], in[14]);
2106
2107 v[0] = _mm_madd_epi16(u[0], k__cospi_p01_p31);
2108 v[1] = _mm_madd_epi16(u[1], k__cospi_p01_p31);
2109 v[2] = _mm_madd_epi16(u[0], k__cospi_p31_m01);
2110 v[3] = _mm_madd_epi16(u[1], k__cospi_p31_m01);
2111 v[4] = _mm_madd_epi16(u[2], k__cospi_p05_p27);
2112 v[5] = _mm_madd_epi16(u[3], k__cospi_p05_p27);
2113 v[6] = _mm_madd_epi16(u[2], k__cospi_p27_m05);
2114 v[7] = _mm_madd_epi16(u[3], k__cospi_p27_m05);
2115 v[8] = _mm_madd_epi16(u[4], k__cospi_p09_p23);
2116 v[9] = _mm_madd_epi16(u[5], k__cospi_p09_p23);
2117 v[10] = _mm_madd_epi16(u[4], k__cospi_p23_m09);
2118 v[11] = _mm_madd_epi16(u[5], k__cospi_p23_m09);
2119 v[12] = _mm_madd_epi16(u[6], k__cospi_p13_p19);
2120 v[13] = _mm_madd_epi16(u[7], k__cospi_p13_p19);
2121 v[14] = _mm_madd_epi16(u[6], k__cospi_p19_m13);
2122 v[15] = _mm_madd_epi16(u[7], k__cospi_p19_m13);
2123 v[16] = _mm_madd_epi16(u[8], k__cospi_p17_p15);
2124 v[17] = _mm_madd_epi16(u[9], k__cospi_p17_p15);
2125 v[18] = _mm_madd_epi16(u[8], k__cospi_p15_m17);
2126 v[19] = _mm_madd_epi16(u[9], k__cospi_p15_m17);
2127 v[20] = _mm_madd_epi16(u[10], k__cospi_p21_p11);
2128 v[21] = _mm_madd_epi16(u[11], k__cospi_p21_p11);
2129 v[22] = _mm_madd_epi16(u[10], k__cospi_p11_m21);
2130 v[23] = _mm_madd_epi16(u[11], k__cospi_p11_m21);
2131 v[24] = _mm_madd_epi16(u[12], k__cospi_p25_p07);
2132 v[25] = _mm_madd_epi16(u[13], k__cospi_p25_p07);
2133 v[26] = _mm_madd_epi16(u[12], k__cospi_p07_m25);
2134 v[27] = _mm_madd_epi16(u[13], k__cospi_p07_m25);
2135 v[28] = _mm_madd_epi16(u[14], k__cospi_p29_p03);
2136 v[29] = _mm_madd_epi16(u[15], k__cospi_p29_p03);
2137 v[30] = _mm_madd_epi16(u[14], k__cospi_p03_m29);
2138 v[31] = _mm_madd_epi16(u[15], k__cospi_p03_m29);
2139
2140 u[0] = _mm_add_epi32(v[0], v[16]);
2141 u[1] = _mm_add_epi32(v[1], v[17]);
2142 u[2] = _mm_add_epi32(v[2], v[18]);
2143 u[3] = _mm_add_epi32(v[3], v[19]);
2144 u[4] = _mm_add_epi32(v[4], v[20]);
2145 u[5] = _mm_add_epi32(v[5], v[21]);
2146 u[6] = _mm_add_epi32(v[6], v[22]);
2147 u[7] = _mm_add_epi32(v[7], v[23]);
2148 u[8] = _mm_add_epi32(v[8], v[24]);
2149 u[9] = _mm_add_epi32(v[9], v[25]);
2150 u[10] = _mm_add_epi32(v[10], v[26]);
2151 u[11] = _mm_add_epi32(v[11], v[27]);
2152 u[12] = _mm_add_epi32(v[12], v[28]);
2153 u[13] = _mm_add_epi32(v[13], v[29]);
2154 u[14] = _mm_add_epi32(v[14], v[30]);
2155 u[15] = _mm_add_epi32(v[15], v[31]);
2156 u[16] = _mm_sub_epi32(v[0], v[16]);
2157 u[17] = _mm_sub_epi32(v[1], v[17]);
2158 u[18] = _mm_sub_epi32(v[2], v[18]);
2159 u[19] = _mm_sub_epi32(v[3], v[19]);
2160 u[20] = _mm_sub_epi32(v[4], v[20]);
2161 u[21] = _mm_sub_epi32(v[5], v[21]);
2162 u[22] = _mm_sub_epi32(v[6], v[22]);
2163 u[23] = _mm_sub_epi32(v[7], v[23]);
2164 u[24] = _mm_sub_epi32(v[8], v[24]);
2165 u[25] = _mm_sub_epi32(v[9], v[25]);
2166 u[26] = _mm_sub_epi32(v[10], v[26]);
2167 u[27] = _mm_sub_epi32(v[11], v[27]);
2168 u[28] = _mm_sub_epi32(v[12], v[28]);
2169 u[29] = _mm_sub_epi32(v[13], v[29]);
2170 u[30] = _mm_sub_epi32(v[14], v[30]);
2171 u[31] = _mm_sub_epi32(v[15], v[31]);
2172
2173 v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
2174 v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
2175 v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
2176 v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING);
2177 v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING);
2178 v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING);
2179 v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING);
2180 v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING);
2181 v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING);
2182 v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING);
2183 v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING);
2184 v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING);
2185 v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING);
2186 v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING);
2187 v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING);
2188 v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING);
2189 v[16] = _mm_add_epi32(u[16], k__DCT_CONST_ROUNDING);
2190 v[17] = _mm_add_epi32(u[17], k__DCT_CONST_ROUNDING);
2191 v[18] = _mm_add_epi32(u[18], k__DCT_CONST_ROUNDING);
2192 v[19] = _mm_add_epi32(u[19], k__DCT_CONST_ROUNDING);
2193 v[20] = _mm_add_epi32(u[20], k__DCT_CONST_ROUNDING);
2194 v[21] = _mm_add_epi32(u[21], k__DCT_CONST_ROUNDING);
2195 v[22] = _mm_add_epi32(u[22], k__DCT_CONST_ROUNDING);
2196 v[23] = _mm_add_epi32(u[23], k__DCT_CONST_ROUNDING);
2197 v[24] = _mm_add_epi32(u[24], k__DCT_CONST_ROUNDING);
2198 v[25] = _mm_add_epi32(u[25], k__DCT_CONST_ROUNDING);
2199 v[26] = _mm_add_epi32(u[26], k__DCT_CONST_ROUNDING);
2200 v[27] = _mm_add_epi32(u[27], k__DCT_CONST_ROUNDING);
2201 v[28] = _mm_add_epi32(u[28], k__DCT_CONST_ROUNDING);
2202 v[29] = _mm_add_epi32(u[29], k__DCT_CONST_ROUNDING);
2203 v[30] = _mm_add_epi32(u[30], k__DCT_CONST_ROUNDING);
2204 v[31] = _mm_add_epi32(u[31], k__DCT_CONST_ROUNDING);
2205
2206 u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS);
2207 u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS);
2208 u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS);
2209 u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS);
2210 u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS);
2211 u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS);
2212 u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS);
2213 u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS);
2214 u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS);
2215 u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS);
2216 u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS);
2217 u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS);
2218 u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS);
2219 u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS);
2220 u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS);
2221 u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS);
2222 u[16] = _mm_srai_epi32(v[16], DCT_CONST_BITS);
2223 u[17] = _mm_srai_epi32(v[17], DCT_CONST_BITS);
2224 u[18] = _mm_srai_epi32(v[18], DCT_CONST_BITS);
2225 u[19] = _mm_srai_epi32(v[19], DCT_CONST_BITS);
2226 u[20] = _mm_srai_epi32(v[20], DCT_CONST_BITS);
2227 u[21] = _mm_srai_epi32(v[21], DCT_CONST_BITS);
2228 u[22] = _mm_srai_epi32(v[22], DCT_CONST_BITS);
2229 u[23] = _mm_srai_epi32(v[23], DCT_CONST_BITS);
2230 u[24] = _mm_srai_epi32(v[24], DCT_CONST_BITS);
2231 u[25] = _mm_srai_epi32(v[25], DCT_CONST_BITS);
2232 u[26] = _mm_srai_epi32(v[26], DCT_CONST_BITS);
2233 u[27] = _mm_srai_epi32(v[27], DCT_CONST_BITS);
2234 u[28] = _mm_srai_epi32(v[28], DCT_CONST_BITS);
2235 u[29] = _mm_srai_epi32(v[29], DCT_CONST_BITS);
2236 u[30] = _mm_srai_epi32(v[30], DCT_CONST_BITS);
2237 u[31] = _mm_srai_epi32(v[31], DCT_CONST_BITS);
2238
2239 s[0] = _mm_packs_epi32(u[0], u[1]);
2240 s[1] = _mm_packs_epi32(u[2], u[3]);
2241 s[2] = _mm_packs_epi32(u[4], u[5]);
2242 s[3] = _mm_packs_epi32(u[6], u[7]);
2243 s[4] = _mm_packs_epi32(u[8], u[9]);
2244 s[5] = _mm_packs_epi32(u[10], u[11]);
2245 s[6] = _mm_packs_epi32(u[12], u[13]);
2246 s[7] = _mm_packs_epi32(u[14], u[15]);
2247 s[8] = _mm_packs_epi32(u[16], u[17]);
2248 s[9] = _mm_packs_epi32(u[18], u[19]);
2249 s[10] = _mm_packs_epi32(u[20], u[21]);
2250 s[11] = _mm_packs_epi32(u[22], u[23]);
2251 s[12] = _mm_packs_epi32(u[24], u[25]);
2252 s[13] = _mm_packs_epi32(u[26], u[27]);
2253 s[14] = _mm_packs_epi32(u[28], u[29]);
2254 s[15] = _mm_packs_epi32(u[30], u[31]);
2255
2256 // stage 2
2257 u[0] = _mm_unpacklo_epi16(s[8], s[9]);
2258 u[1] = _mm_unpackhi_epi16(s[8], s[9]);
2259 u[2] = _mm_unpacklo_epi16(s[10], s[11]);
2260 u[3] = _mm_unpackhi_epi16(s[10], s[11]);
2261 u[4] = _mm_unpacklo_epi16(s[12], s[13]);
2262 u[5] = _mm_unpackhi_epi16(s[12], s[13]);
2263 u[6] = _mm_unpacklo_epi16(s[14], s[15]);
2264 u[7] = _mm_unpackhi_epi16(s[14], s[15]);
2265
2266 v[0] = _mm_madd_epi16(u[0], k__cospi_p04_p28);
2267 v[1] = _mm_madd_epi16(u[1], k__cospi_p04_p28);
2268 v[2] = _mm_madd_epi16(u[0], k__cospi_p28_m04);
2269 v[3] = _mm_madd_epi16(u[1], k__cospi_p28_m04);
2270 v[4] = _mm_madd_epi16(u[2], k__cospi_p20_p12);
2271 v[5] = _mm_madd_epi16(u[3], k__cospi_p20_p12);
2272 v[6] = _mm_madd_epi16(u[2], k__cospi_p12_m20);
2273 v[7] = _mm_madd_epi16(u[3], k__cospi_p12_m20);
2274 v[8] = _mm_madd_epi16(u[4], k__cospi_m28_p04);
2275 v[9] = _mm_madd_epi16(u[5], k__cospi_m28_p04);
2276 v[10] = _mm_madd_epi16(u[4], k__cospi_p04_p28);
2277 v[11] = _mm_madd_epi16(u[5], k__cospi_p04_p28);
2278 v[12] = _mm_madd_epi16(u[6], k__cospi_m12_p20);
2279 v[13] = _mm_madd_epi16(u[7], k__cospi_m12_p20);
2280 v[14] = _mm_madd_epi16(u[6], k__cospi_p20_p12);
2281 v[15] = _mm_madd_epi16(u[7], k__cospi_p20_p12);
2282
2283 u[0] = _mm_add_epi32(v[0], v[8]);
2284 u[1] = _mm_add_epi32(v[1], v[9]);
2285 u[2] = _mm_add_epi32(v[2], v[10]);
2286 u[3] = _mm_add_epi32(v[3], v[11]);
2287 u[4] = _mm_add_epi32(v[4], v[12]);
2288 u[5] = _mm_add_epi32(v[5], v[13]);
2289 u[6] = _mm_add_epi32(v[6], v[14]);
2290 u[7] = _mm_add_epi32(v[7], v[15]);
2291 u[8] = _mm_sub_epi32(v[0], v[8]);
2292 u[9] = _mm_sub_epi32(v[1], v[9]);
2293 u[10] = _mm_sub_epi32(v[2], v[10]);
2294 u[11] = _mm_sub_epi32(v[3], v[11]);
2295 u[12] = _mm_sub_epi32(v[4], v[12]);
2296 u[13] = _mm_sub_epi32(v[5], v[13]);
2297 u[14] = _mm_sub_epi32(v[6], v[14]);
2298 u[15] = _mm_sub_epi32(v[7], v[15]);
2299
2300 v[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
2301 v[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
2302 v[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
2303 v[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING);
2304 v[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING);
2305 v[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING);
2306 v[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING);
2307 v[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING);
2308 v[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING);
2309 v[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING);
2310 v[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING);
2311 v[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING);
2312 v[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING);
2313 v[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING);
2314 v[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING);
2315 v[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING);
2316
2317 u[0] = _mm_srai_epi32(v[0], DCT_CONST_BITS);
2318 u[1] = _mm_srai_epi32(v[1], DCT_CONST_BITS);
2319 u[2] = _mm_srai_epi32(v[2], DCT_CONST_BITS);
2320 u[3] = _mm_srai_epi32(v[3], DCT_CONST_BITS);
2321 u[4] = _mm_srai_epi32(v[4], DCT_CONST_BITS);
2322 u[5] = _mm_srai_epi32(v[5], DCT_CONST_BITS);
2323 u[6] = _mm_srai_epi32(v[6], DCT_CONST_BITS);
2324 u[7] = _mm_srai_epi32(v[7], DCT_CONST_BITS);
2325 u[8] = _mm_srai_epi32(v[8], DCT_CONST_BITS);
2326 u[9] = _mm_srai_epi32(v[9], DCT_CONST_BITS);
2327 u[10] = _mm_srai_epi32(v[10], DCT_CONST_BITS);
2328 u[11] = _mm_srai_epi32(v[11], DCT_CONST_BITS);
2329 u[12] = _mm_srai_epi32(v[12], DCT_CONST_BITS);
2330 u[13] = _mm_srai_epi32(v[13], DCT_CONST_BITS);
2331 u[14] = _mm_srai_epi32(v[14], DCT_CONST_BITS);
2332 u[15] = _mm_srai_epi32(v[15], DCT_CONST_BITS);
2333
2334 x[0] = _mm_add_epi16(s[0], s[4]);
2335 x[1] = _mm_add_epi16(s[1], s[5]);
2336 x[2] = _mm_add_epi16(s[2], s[6]);
2337 x[3] = _mm_add_epi16(s[3], s[7]);
2338 x[4] = _mm_sub_epi16(s[0], s[4]);
2339 x[5] = _mm_sub_epi16(s[1], s[5]);
2340 x[6] = _mm_sub_epi16(s[2], s[6]);
2341 x[7] = _mm_sub_epi16(s[3], s[7]);
2342 x[8] = _mm_packs_epi32(u[0], u[1]);
2343 x[9] = _mm_packs_epi32(u[2], u[3]);
2344 x[10] = _mm_packs_epi32(u[4], u[5]);
2345 x[11] = _mm_packs_epi32(u[6], u[7]);
2346 x[12] = _mm_packs_epi32(u[8], u[9]);
2347 x[13] = _mm_packs_epi32(u[10], u[11]);
2348 x[14] = _mm_packs_epi32(u[12], u[13]);
2349 x[15] = _mm_packs_epi32(u[14], u[15]);
2350
2351 // stage 3
2352 u[0] = _mm_unpacklo_epi16(x[4], x[5]);
2353 u[1] = _mm_unpackhi_epi16(x[4], x[5]);
2354 u[2] = _mm_unpacklo_epi16(x[6], x[7]);
2355 u[3] = _mm_unpackhi_epi16(x[6], x[7]);
2356 u[4] = _mm_unpacklo_epi16(x[12], x[13]);
2357 u[5] = _mm_unpackhi_epi16(x[12], x[13]);
2358 u[6] = _mm_unpacklo_epi16(x[14], x[15]);
2359 u[7] = _mm_unpackhi_epi16(x[14], x[15]);
2360
2361 v[0] = _mm_madd_epi16(u[0], k__cospi_p08_p24);
2362 v[1] = _mm_madd_epi16(u[1], k__cospi_p08_p24);
2363 v[2] = _mm_madd_epi16(u[0], k__cospi_p24_m08);
2364 v[3] = _mm_madd_epi16(u[1], k__cospi_p24_m08);
2365 v[4] = _mm_madd_epi16(u[2], k__cospi_m24_p08);
2366 v[5] = _mm_madd_epi16(u[3], k__cospi_m24_p08);
2367 v[6] = _mm_madd_epi16(u[2], k__cospi_p08_p24);
2368 v[7] = _mm_madd_epi16(u[3], k__cospi_p08_p24);
2369 v[8] = _mm_madd_epi16(u[4], k__cospi_p08_p24);
2370 v[9] = _mm_madd_epi16(u[5], k__cospi_p08_p24);
2371 v[10] = _mm_madd_epi16(u[4], k__cospi_p24_m08);
2372 v[11] = _mm_madd_epi16(u[5], k__cospi_p24_m08);
2373 v[12] = _mm_madd_epi16(u[6], k__cospi_m24_p08);
2374 v[13] = _mm_madd_epi16(u[7], k__cospi_m24_p08);
2375 v[14] = _mm_madd_epi16(u[6], k__cospi_p08_p24);
2376 v[15] = _mm_madd_epi16(u[7], k__cospi_p08_p24);
2377
2378 u[0] = _mm_add_epi32(v[0], v[4]);
2379 u[1] = _mm_add_epi32(v[1], v[5]);
2380 u[2] = _mm_add_epi32(v[2], v[6]);
2381 u[3] = _mm_add_epi32(v[3], v[7]);
2382 u[4] = _mm_sub_epi32(v[0], v[4]);
2383 u[5] = _mm_sub_epi32(v[1], v[5]);
2384 u[6] = _mm_sub_epi32(v[2], v[6]);
2385 u[7] = _mm_sub_epi32(v[3], v[7]);
2386 u[8] = _mm_add_epi32(v[8], v[12]);
2387 u[9] = _mm_add_epi32(v[9], v[13]);
2388 u[10] = _mm_add_epi32(v[10], v[14]);
2389 u[11] = _mm_add_epi32(v[11], v[15]);
2390 u[12] = _mm_sub_epi32(v[8], v[12]);
2391 u[13] = _mm_sub_epi32(v[9], v[13]);
2392 u[14] = _mm_sub_epi32(v[10], v[14]);
2393 u[15] = _mm_sub_epi32(v[11], v[15]);
2394
2395 u[0] = _mm_add_epi32(u[0], k__DCT_CONST_ROUNDING);
2396 u[1] = _mm_add_epi32(u[1], k__DCT_CONST_ROUNDING);
2397 u[2] = _mm_add_epi32(u[2], k__DCT_CONST_ROUNDING);
2398 u[3] = _mm_add_epi32(u[3], k__DCT_CONST_ROUNDING);
2399 u[4] = _mm_add_epi32(u[4], k__DCT_CONST_ROUNDING);
2400 u[5] = _mm_add_epi32(u[5], k__DCT_CONST_ROUNDING);
2401 u[6] = _mm_add_epi32(u[6], k__DCT_CONST_ROUNDING);
2402 u[7] = _mm_add_epi32(u[7], k__DCT_CONST_ROUNDING);
2403 u[8] = _mm_add_epi32(u[8], k__DCT_CONST_ROUNDING);
2404 u[9] = _mm_add_epi32(u[9], k__DCT_CONST_ROUNDING);
2405 u[10] = _mm_add_epi32(u[10], k__DCT_CONST_ROUNDING);
2406 u[11] = _mm_add_epi32(u[11], k__DCT_CONST_ROUNDING);
2407 u[12] = _mm_add_epi32(u[12], k__DCT_CONST_ROUNDING);
2408 u[13] = _mm_add_epi32(u[13], k__DCT_CONST_ROUNDING);
2409 u[14] = _mm_add_epi32(u[14], k__DCT_CONST_ROUNDING);
2410 u[15] = _mm_add_epi32(u[15], k__DCT_CONST_ROUNDING);
2411
2412 v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS);
2413 v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS);
2414 v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS);
2415 v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS);
2416 v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS);
2417 v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS);
2418 v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS);
2419 v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS);
2420 v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS);
2421 v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS);
2422 v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS);
2423 v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS);
2424 v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS);
2425 v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS);
2426 v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS);
2427 v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS);
2428
2429 s[0] = _mm_add_epi16(x[0], x[2]);
2430 s[1] = _mm_add_epi16(x[1], x[3]);
2431 s[2] = _mm_sub_epi16(x[0], x[2]);
2432 s[3] = _mm_sub_epi16(x[1], x[3]);
2433 s[4] = _mm_packs_epi32(v[0], v[1]);
2434 s[5] = _mm_packs_epi32(v[2], v[3]);
2435 s[6] = _mm_packs_epi32(v[4], v[5]);
2436 s[7] = _mm_packs_epi32(v[6], v[7]);
2437 s[8] = _mm_add_epi16(x[8], x[10]);
2438 s[9] = _mm_add_epi16(x[9], x[11]);
2439 s[10] = _mm_sub_epi16(x[8], x[10]);
2440 s[11] = _mm_sub_epi16(x[9], x[11]);
2441 s[12] = _mm_packs_epi32(v[8], v[9]);
2442 s[13] = _mm_packs_epi32(v[10], v[11]);
2443 s[14] = _mm_packs_epi32(v[12], v[13]);
2444 s[15] = _mm_packs_epi32(v[14], v[15]);
2445
2446 // stage 4
2447 u[0] = _mm_unpacklo_epi16(s[2], s[3]);
2448 u[1] = _mm_unpackhi_epi16(s[2], s[3]);
2449 u[2] = _mm_unpacklo_epi16(s[6], s[7]);
2450 u[3] = _mm_unpackhi_epi16(s[6], s[7]);
2451 u[4] = _mm_unpacklo_epi16(s[10], s[11]);
2452 u[5] = _mm_unpackhi_epi16(s[10], s[11]);
2453 u[6] = _mm_unpacklo_epi16(s[14], s[15]);
2454 u[7] = _mm_unpackhi_epi16(s[14], s[15]);
2455
2456 v[0] = _mm_madd_epi16(u[0], k__cospi_m16_m16);
2457 v[1] = _mm_madd_epi16(u[1], k__cospi_m16_m16);
2458 v[2] = _mm_madd_epi16(u[0], k__cospi_p16_m16);
2459 v[3] = _mm_madd_epi16(u[1], k__cospi_p16_m16);
2460 v[4] = _mm_madd_epi16(u[2], k__cospi_p16_p16);
2461 v[5] = _mm_madd_epi16(u[3], k__cospi_p16_p16);
2462 v[6] = _mm_madd_epi16(u[2], k__cospi_m16_p16);
2463 v[7] = _mm_madd_epi16(u[3], k__cospi_m16_p16);
2464 v[8] = _mm_madd_epi16(u[4], k__cospi_p16_p16);
2465 v[9] = _mm_madd_epi16(u[5], k__cospi_p16_p16);
2466 v[10] = _mm_madd_epi16(u[4], k__cospi_m16_p16);
2467 v[11] = _mm_madd_epi16(u[5], k__cospi_m16_p16);
2468 v[12] = _mm_madd_epi16(u[6], k__cospi_m16_m16);
2469 v[13] = _mm_madd_epi16(u[7], k__cospi_m16_m16);
2470 v[14] = _mm_madd_epi16(u[6], k__cospi_p16_m16);
2471 v[15] = _mm_madd_epi16(u[7], k__cospi_p16_m16);
2472
2473 u[0] = _mm_add_epi32(v[0], k__DCT_CONST_ROUNDING);
2474 u[1] = _mm_add_epi32(v[1], k__DCT_CONST_ROUNDING);
2475 u[2] = _mm_add_epi32(v[2], k__DCT_CONST_ROUNDING);
2476 u[3] = _mm_add_epi32(v[3], k__DCT_CONST_ROUNDING);
2477 u[4] = _mm_add_epi32(v[4], k__DCT_CONST_ROUNDING);
2478 u[5] = _mm_add_epi32(v[5], k__DCT_CONST_ROUNDING);
2479 u[6] = _mm_add_epi32(v[6], k__DCT_CONST_ROUNDING);
2480 u[7] = _mm_add_epi32(v[7], k__DCT_CONST_ROUNDING);
2481 u[8] = _mm_add_epi32(v[8], k__DCT_CONST_ROUNDING);
2482 u[9] = _mm_add_epi32(v[9], k__DCT_CONST_ROUNDING);
2483 u[10] = _mm_add_epi32(v[10], k__DCT_CONST_ROUNDING);
2484 u[11] = _mm_add_epi32(v[11], k__DCT_CONST_ROUNDING);
2485 u[12] = _mm_add_epi32(v[12], k__DCT_CONST_ROUNDING);
2486 u[13] = _mm_add_epi32(v[13], k__DCT_CONST_ROUNDING);
2487 u[14] = _mm_add_epi32(v[14], k__DCT_CONST_ROUNDING);
2488 u[15] = _mm_add_epi32(v[15], k__DCT_CONST_ROUNDING);
2489
2490 v[0] = _mm_srai_epi32(u[0], DCT_CONST_BITS);
2491 v[1] = _mm_srai_epi32(u[1], DCT_CONST_BITS);
2492 v[2] = _mm_srai_epi32(u[2], DCT_CONST_BITS);
2493 v[3] = _mm_srai_epi32(u[3], DCT_CONST_BITS);
2494 v[4] = _mm_srai_epi32(u[4], DCT_CONST_BITS);
2495 v[5] = _mm_srai_epi32(u[5], DCT_CONST_BITS);
2496 v[6] = _mm_srai_epi32(u[6], DCT_CONST_BITS);
2497 v[7] = _mm_srai_epi32(u[7], DCT_CONST_BITS);
2498 v[8] = _mm_srai_epi32(u[8], DCT_CONST_BITS);
2499 v[9] = _mm_srai_epi32(u[9], DCT_CONST_BITS);
2500 v[10] = _mm_srai_epi32(u[10], DCT_CONST_BITS);
2501 v[11] = _mm_srai_epi32(u[11], DCT_CONST_BITS);
2502 v[12] = _mm_srai_epi32(u[12], DCT_CONST_BITS);
2503 v[13] = _mm_srai_epi32(u[13], DCT_CONST_BITS);
2504 v[14] = _mm_srai_epi32(u[14], DCT_CONST_BITS);
2505 v[15] = _mm_srai_epi32(u[15], DCT_CONST_BITS);
2506
2507 in[0] = s[0];
2508 in[1] = _mm_sub_epi16(kZero, s[8]);
2509 in[2] = s[12];
2510 in[3] = _mm_sub_epi16(kZero, s[4]);
2511 in[4] = _mm_packs_epi32(v[4], v[5]);
2512 in[5] = _mm_packs_epi32(v[12], v[13]);
2513 in[6] = _mm_packs_epi32(v[8], v[9]);
2514 in[7] = _mm_packs_epi32(v[0], v[1]);
2515 in[8] = _mm_packs_epi32(v[2], v[3]);
2516 in[9] = _mm_packs_epi32(v[10], v[11]);
2517 in[10] = _mm_packs_epi32(v[14], v[15]);
2518 in[11] = _mm_packs_epi32(v[6], v[7]);
2519 in[12] = s[5];
2520 in[13] = _mm_sub_epi16(kZero, s[13]);
2521 in[14] = s[9];
2522 in[15] = _mm_sub_epi16(kZero, s[1]);
2523 }
2524
2525 void fdct16_1d_sse2(__m128i *in0, __m128i *in1) {
2526 fdct16_1d_8col(in0);
2527 fdct16_1d_8col(in1);
2528 array_transpose_16x16(in0, in1);
2529 }
2530
2531 void fadst16_1d_sse2(__m128i *in0, __m128i *in1) {
2532 fadst16_1d_8col(in0);
2533 fadst16_1d_8col(in1);
2534 array_transpose_16x16(in0, in1);
2535 }
2536
2537 void vp9_short_fht16x16_sse2(const int16_t *input, int16_t *output,
2538 int stride, int tx_type) {
2539 __m128i in0[16], in1[16];
2540 load_buffer_16x16(input, in0, in1, stride);
2541 switch (tx_type) {
2542 case 0: // DCT_DCT
2543 fdct16_1d_sse2(in0, in1);
2544 right_shift_16x16(in0, in1);
2545 fdct16_1d_sse2(in0, in1);
2546 break;
2547 case 1: // ADST_DCT
2548 fadst16_1d_sse2(in0, in1);
2549 right_shift_16x16(in0, in1);
2550 fdct16_1d_sse2(in0, in1);
2551 break;
2552 case 2: // DCT_ADST
2553 fdct16_1d_sse2(in0, in1);
2554 right_shift_16x16(in0, in1);
2555 fadst16_1d_sse2(in0, in1);
2556 break;
2557 case 3: // ADST_ADST
2558 fadst16_1d_sse2(in0, in1);
2559 right_shift_16x16(in0, in1);
2560 fadst16_1d_sse2(in0, in1);
2561 break;
2562 default:
2563 assert(0);
2564 break;
2565 }
2566 write_buffer_16x16(output, in0, in1, 16);
2567 }
2568
2569 #define FDCT32x32_2D vp9_fdct32x32_rd_sse2
2570 #define FDCT32x32_HIGH_PRECISION 0
2571 #include "vp9/encoder/x86/vp9_dct32x32_sse2.c"
2572 #undef FDCT32x32_2D
2573 #undef FDCT32x32_HIGH_PRECISION
2574
2575 #define FDCT32x32_2D vp9_fdct32x32_sse2
2576 #define FDCT32x32_HIGH_PRECISION 1
2577 #include "vp9/encoder/x86/vp9_dct32x32_sse2.c" // NOLINT
2578 #undef FDCT32x32_2D
2579 #undef FDCT32x32_HIGH_PRECISION

Mercurial Repository: The Tor Browser

Europalab SCM Repository Server

mercurial