media/libyuv/source/scale_neon.cc@b8a032363ba2
media/libyuv/source/scale_neon.cc
Thu, 22 Jan 2015 13:21:57 +0100
- author
- Michael Schloh von Bennewitz <michael@schloh.com>
- date
- Thu, 22 Jan 2015 13:21:57 +0100
- branch
- TOR_BUG_9701
- changeset 15
- b8a032363ba2
- permissions
- -rw-r--r--
Incorporate requested changes from Mozilla in review:
https://bugzilla.mozilla.org/show_bug.cgi?id=1123480#c6
1 /*
2 * Copyright 2011 The LibYuv Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
11 #include "libyuv/row.h"
13 #ifdef __cplusplus
14 namespace libyuv {
15 extern "C" {
16 #endif
18 // This module is for GCC Neon.
19 #if !defined(LIBYUV_DISABLE_NEON) && defined(__ARM_NEON__)
21 // NEON downscalers with interpolation.
22 // Provided by Fritz Koenig
24 // Read 32x1 throw away even pixels, and write 16x1.
25 void ScaleRowDown2_NEON(const uint8* src_ptr, ptrdiff_t src_stride,
26 uint8* dst, int dst_width) {
27 asm volatile (
28 ".p2align 2 \n"
29 "1: \n"
30 // load even pixels into q0, odd into q1
31 "vld2.8 {q0, q1}, [%0]! \n"
32 "subs %2, %2, #16 \n" // 16 processed per loop
33 "vst1.8 {q1}, [%1]! \n" // store odd pixels
34 "bgt 1b \n"
35 : "+r"(src_ptr), // %0
36 "+r"(dst), // %1
37 "+r"(dst_width) // %2
38 :
39 : "q0", "q1" // Clobber List
40 );
41 }
43 // Read 32x2 average down and write 16x1.
44 void ScaleRowDown2Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride,
45 uint8* dst, int dst_width) {
46 asm volatile (
47 // change the stride to row 2 pointer
48 "add %1, %0 \n"
49 ".p2align 2 \n"
50 "1: \n"
51 "vld1.8 {q0, q1}, [%0]! \n" // load row 1 and post inc
52 "vld1.8 {q2, q3}, [%1]! \n" // load row 2 and post inc
53 "subs %3, %3, #16 \n" // 16 processed per loop
54 "vpaddl.u8 q0, q0 \n" // row 1 add adjacent
55 "vpaddl.u8 q1, q1 \n"
56 "vpadal.u8 q0, q2 \n" // row 2 add adjacent + row1
57 "vpadal.u8 q1, q3 \n"
58 "vrshrn.u16 d0, q0, #2 \n" // downshift, round and pack
59 "vrshrn.u16 d1, q1, #2 \n"
60 "vst1.8 {q0}, [%2]! \n"
61 "bgt 1b \n"
62 : "+r"(src_ptr), // %0
63 "+r"(src_stride), // %1
64 "+r"(dst), // %2
65 "+r"(dst_width) // %3
66 :
67 : "q0", "q1", "q2", "q3" // Clobber List
68 );
69 }
71 void ScaleRowDown4_NEON(const uint8* src_ptr, ptrdiff_t src_stride,
72 uint8* dst_ptr, int dst_width) {
73 asm volatile (
74 ".p2align 2 \n"
75 "1: \n"
76 "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
77 "subs %2, %2, #8 \n" // 8 processed per loop
78 "vst1.8 {d2}, [%1]! \n"
79 "bgt 1b \n"
80 : "+r"(src_ptr), // %0
81 "+r"(dst_ptr), // %1
82 "+r"(dst_width) // %2
83 :
84 : "q0", "q1", "memory", "cc"
85 );
86 }
88 void ScaleRowDown4Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride,
89 uint8* dst_ptr, int dst_width) {
90 asm volatile (
91 "add r4, %0, %3 \n"
92 "add r5, r4, %3 \n"
93 "add %3, r5, %3 \n"
94 ".p2align 2 \n"
95 "1: \n"
96 "vld1.8 {q0}, [%0]! \n" // load up 16x4
97 "vld1.8 {q1}, [r4]! \n"
98 "vld1.8 {q2}, [r5]! \n"
99 "vld1.8 {q3}, [%3]! \n"
100 "subs %2, %2, #4 \n"
101 "vpaddl.u8 q0, q0 \n"
102 "vpadal.u8 q0, q1 \n"
103 "vpadal.u8 q0, q2 \n"
104 "vpadal.u8 q0, q3 \n"
105 "vpaddl.u16 q0, q0 \n"
106 "vrshrn.u32 d0, q0, #4 \n" // divide by 16 w/rounding
107 "vmovn.u16 d0, q0 \n"
108 "vst1.32 {d0[0]}, [%1]! \n"
109 "bgt 1b \n"
110 : "+r"(src_ptr), // %0
111 "+r"(dst_ptr), // %1
112 "+r"(dst_width) // %2
113 : "r"(src_stride) // %3
114 : "r4", "r5", "q0", "q1", "q2", "q3", "memory", "cc"
115 );
116 }
118 // Down scale from 4 to 3 pixels. Use the neon multilane read/write
119 // to load up the every 4th pixel into a 4 different registers.
120 // Point samples 32 pixels to 24 pixels.
121 void ScaleRowDown34_NEON(const uint8* src_ptr,
122 ptrdiff_t src_stride,
123 uint8* dst_ptr, int dst_width) {
124 asm volatile (
125 ".p2align 2 \n"
126 "1: \n"
127 "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
128 "subs %2, %2, #24 \n"
129 "vmov d2, d3 \n" // order d0, d1, d2
130 "vst3.8 {d0, d1, d2}, [%1]! \n"
131 "bgt 1b \n"
132 : "+r"(src_ptr), // %0
133 "+r"(dst_ptr), // %1
134 "+r"(dst_width) // %2
135 :
136 : "d0", "d1", "d2", "d3", "memory", "cc"
137 );
138 }
140 void ScaleRowDown34_0_Box_NEON(const uint8* src_ptr,
141 ptrdiff_t src_stride,
142 uint8* dst_ptr, int dst_width) {
143 asm volatile (
144 "vmov.u8 d24, #3 \n"
145 "add %3, %0 \n"
146 ".p2align 2 \n"
147 "1: \n"
148 "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
149 "vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1
150 "subs %2, %2, #24 \n"
152 // filter src line 0 with src line 1
153 // expand chars to shorts to allow for room
154 // when adding lines together
155 "vmovl.u8 q8, d4 \n"
156 "vmovl.u8 q9, d5 \n"
157 "vmovl.u8 q10, d6 \n"
158 "vmovl.u8 q11, d7 \n"
160 // 3 * line_0 + line_1
161 "vmlal.u8 q8, d0, d24 \n"
162 "vmlal.u8 q9, d1, d24 \n"
163 "vmlal.u8 q10, d2, d24 \n"
164 "vmlal.u8 q11, d3, d24 \n"
166 // (3 * line_0 + line_1) >> 2
167 "vqrshrn.u16 d0, q8, #2 \n"
168 "vqrshrn.u16 d1, q9, #2 \n"
169 "vqrshrn.u16 d2, q10, #2 \n"
170 "vqrshrn.u16 d3, q11, #2 \n"
172 // a0 = (src[0] * 3 + s[1] * 1) >> 2
173 "vmovl.u8 q8, d1 \n"
174 "vmlal.u8 q8, d0, d24 \n"
175 "vqrshrn.u16 d0, q8, #2 \n"
177 // a1 = (src[1] * 1 + s[2] * 1) >> 1
178 "vrhadd.u8 d1, d1, d2 \n"
180 // a2 = (src[2] * 1 + s[3] * 3) >> 2
181 "vmovl.u8 q8, d2 \n"
182 "vmlal.u8 q8, d3, d24 \n"
183 "vqrshrn.u16 d2, q8, #2 \n"
185 "vst3.8 {d0, d1, d2}, [%1]! \n"
187 "bgt 1b \n"
188 : "+r"(src_ptr), // %0
189 "+r"(dst_ptr), // %1
190 "+r"(dst_width), // %2
191 "+r"(src_stride) // %3
192 :
193 : "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "d24", "memory", "cc"
194 );
195 }
197 void ScaleRowDown34_1_Box_NEON(const uint8* src_ptr,
198 ptrdiff_t src_stride,
199 uint8* dst_ptr, int dst_width) {
200 asm volatile (
201 "vmov.u8 d24, #3 \n"
202 "add %3, %0 \n"
203 ".p2align 2 \n"
204 "1: \n"
205 "vld4.8 {d0, d1, d2, d3}, [%0]! \n" // src line 0
206 "vld4.8 {d4, d5, d6, d7}, [%3]! \n" // src line 1
207 "subs %2, %2, #24 \n"
208 // average src line 0 with src line 1
209 "vrhadd.u8 q0, q0, q2 \n"
210 "vrhadd.u8 q1, q1, q3 \n"
212 // a0 = (src[0] * 3 + s[1] * 1) >> 2
213 "vmovl.u8 q3, d1 \n"
214 "vmlal.u8 q3, d0, d24 \n"
215 "vqrshrn.u16 d0, q3, #2 \n"
217 // a1 = (src[1] * 1 + s[2] * 1) >> 1
218 "vrhadd.u8 d1, d1, d2 \n"
220 // a2 = (src[2] * 1 + s[3] * 3) >> 2
221 "vmovl.u8 q3, d2 \n"
222 "vmlal.u8 q3, d3, d24 \n"
223 "vqrshrn.u16 d2, q3, #2 \n"
225 "vst3.8 {d0, d1, d2}, [%1]! \n"
226 "bgt 1b \n"
227 : "+r"(src_ptr), // %0
228 "+r"(dst_ptr), // %1
229 "+r"(dst_width), // %2
230 "+r"(src_stride) // %3
231 :
232 : "r4", "q0", "q1", "q2", "q3", "d24", "memory", "cc"
233 );
234 }
236 #define HAS_SCALEROWDOWN38_NEON
237 static uvec8 kShuf38 =
238 { 0, 3, 6, 8, 11, 14, 16, 19, 22, 24, 27, 30, 0, 0, 0, 0 };
239 static uvec8 kShuf38_2 =
240 { 0, 8, 16, 2, 10, 17, 4, 12, 18, 6, 14, 19, 0, 0, 0, 0 };
241 static vec16 kMult38_Div6 =
242 { 65536 / 12, 65536 / 12, 65536 / 12, 65536 / 12,
243 65536 / 12, 65536 / 12, 65536 / 12, 65536 / 12 };
244 static vec16 kMult38_Div9 =
245 { 65536 / 18, 65536 / 18, 65536 / 18, 65536 / 18,
246 65536 / 18, 65536 / 18, 65536 / 18, 65536 / 18 };
248 // 32 -> 12
249 void ScaleRowDown38_NEON(const uint8* src_ptr,
250 ptrdiff_t src_stride,
251 uint8* dst_ptr, int dst_width) {
252 asm volatile (
253 "vld1.8 {q3}, [%3] \n"
254 ".p2align 2 \n"
255 "1: \n"
256 "vld1.8 {d0, d1, d2, d3}, [%0]! \n"
257 "subs %2, %2, #12 \n"
258 "vtbl.u8 d4, {d0, d1, d2, d3}, d6 \n"
259 "vtbl.u8 d5, {d0, d1, d2, d3}, d7 \n"
260 "vst1.8 {d4}, [%1]! \n"
261 "vst1.32 {d5[0]}, [%1]! \n"
262 "bgt 1b \n"
263 : "+r"(src_ptr), // %0
264 "+r"(dst_ptr), // %1
265 "+r"(dst_width) // %2
266 : "r"(&kShuf38) // %3
267 : "d0", "d1", "d2", "d3", "d4", "d5", "memory", "cc"
268 );
269 }
271 // 32x3 -> 12x1
272 void OMITFP ScaleRowDown38_3_Box_NEON(const uint8* src_ptr,
273 ptrdiff_t src_stride,
274 uint8* dst_ptr, int dst_width) {
275 asm volatile (
276 "vld1.16 {q13}, [%4] \n"
277 "vld1.8 {q14}, [%5] \n"
278 "vld1.8 {q15}, [%6] \n"
279 "add r4, %0, %3, lsl #1 \n"
280 "add %3, %0 \n"
281 ".p2align 2 \n"
282 "1: \n"
284 // d0 = 00 40 01 41 02 42 03 43
285 // d1 = 10 50 11 51 12 52 13 53
286 // d2 = 20 60 21 61 22 62 23 63
287 // d3 = 30 70 31 71 32 72 33 73
288 "vld4.8 {d0, d1, d2, d3}, [%0]! \n"
289 "vld4.8 {d4, d5, d6, d7}, [%3]! \n"
290 "vld4.8 {d16, d17, d18, d19}, [r4]! \n"
291 "subs %2, %2, #12 \n"
293 // Shuffle the input data around to get align the data
294 // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7
295 // d0 = 00 10 01 11 02 12 03 13
296 // d1 = 40 50 41 51 42 52 43 53
297 "vtrn.u8 d0, d1 \n"
298 "vtrn.u8 d4, d5 \n"
299 "vtrn.u8 d16, d17 \n"
301 // d2 = 20 30 21 31 22 32 23 33
302 // d3 = 60 70 61 71 62 72 63 73
303 "vtrn.u8 d2, d3 \n"
304 "vtrn.u8 d6, d7 \n"
305 "vtrn.u8 d18, d19 \n"
307 // d0 = 00+10 01+11 02+12 03+13
308 // d2 = 40+50 41+51 42+52 43+53
309 "vpaddl.u8 q0, q0 \n"
310 "vpaddl.u8 q2, q2 \n"
311 "vpaddl.u8 q8, q8 \n"
313 // d3 = 60+70 61+71 62+72 63+73
314 "vpaddl.u8 d3, d3 \n"
315 "vpaddl.u8 d7, d7 \n"
316 "vpaddl.u8 d19, d19 \n"
318 // combine source lines
319 "vadd.u16 q0, q2 \n"
320 "vadd.u16 q0, q8 \n"
321 "vadd.u16 d4, d3, d7 \n"
322 "vadd.u16 d4, d19 \n"
324 // dst_ptr[3] = (s[6 + st * 0] + s[7 + st * 0]
325 // + s[6 + st * 1] + s[7 + st * 1]
326 // + s[6 + st * 2] + s[7 + st * 2]) / 6
327 "vqrdmulh.s16 q2, q2, q13 \n"
328 "vmovn.u16 d4, q2 \n"
330 // Shuffle 2,3 reg around so that 2 can be added to the
331 // 0,1 reg and 3 can be added to the 4,5 reg. This
332 // requires expanding from u8 to u16 as the 0,1 and 4,5
333 // registers are already expanded. Then do transposes
334 // to get aligned.
335 // q2 = xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33
336 "vmovl.u8 q1, d2 \n"
337 "vmovl.u8 q3, d6 \n"
338 "vmovl.u8 q9, d18 \n"
340 // combine source lines
341 "vadd.u16 q1, q3 \n"
342 "vadd.u16 q1, q9 \n"
344 // d4 = xx 20 xx 30 xx 22 xx 32
345 // d5 = xx 21 xx 31 xx 23 xx 33
346 "vtrn.u32 d2, d3 \n"
348 // d4 = xx 20 xx 21 xx 22 xx 23
349 // d5 = xx 30 xx 31 xx 32 xx 33
350 "vtrn.u16 d2, d3 \n"
352 // 0+1+2, 3+4+5
353 "vadd.u16 q0, q1 \n"
355 // Need to divide, but can't downshift as the the value
356 // isn't a power of 2. So multiply by 65536 / n
357 // and take the upper 16 bits.
358 "vqrdmulh.s16 q0, q0, q15 \n"
360 // Align for table lookup, vtbl requires registers to
361 // be adjacent
362 "vmov.u8 d2, d4 \n"
364 "vtbl.u8 d3, {d0, d1, d2}, d28 \n"
365 "vtbl.u8 d4, {d0, d1, d2}, d29 \n"
367 "vst1.8 {d3}, [%1]! \n"
368 "vst1.32 {d4[0]}, [%1]! \n"
369 "bgt 1b \n"
370 : "+r"(src_ptr), // %0
371 "+r"(dst_ptr), // %1
372 "+r"(dst_width), // %2
373 "+r"(src_stride) // %3
374 : "r"(&kMult38_Div6), // %4
375 "r"(&kShuf38_2), // %5
376 "r"(&kMult38_Div9) // %6
377 : "r4", "q0", "q1", "q2", "q3", "q8", "q9",
378 "q13", "q14", "q15", "memory", "cc"
379 );
380 }
382 // 32x2 -> 12x1
383 void ScaleRowDown38_2_Box_NEON(const uint8* src_ptr,
384 ptrdiff_t src_stride,
385 uint8* dst_ptr, int dst_width) {
386 asm volatile (
387 "vld1.16 {q13}, [%4] \n"
388 "vld1.8 {q14}, [%5] \n"
389 "add %3, %0 \n"
390 ".p2align 2 \n"
391 "1: \n"
393 // d0 = 00 40 01 41 02 42 03 43
394 // d1 = 10 50 11 51 12 52 13 53
395 // d2 = 20 60 21 61 22 62 23 63
396 // d3 = 30 70 31 71 32 72 33 73
397 "vld4.8 {d0, d1, d2, d3}, [%0]! \n"
398 "vld4.8 {d4, d5, d6, d7}, [%3]! \n"
399 "subs %2, %2, #12 \n"
401 // Shuffle the input data around to get align the data
402 // so adjacent data can be added. 0,1 - 2,3 - 4,5 - 6,7
403 // d0 = 00 10 01 11 02 12 03 13
404 // d1 = 40 50 41 51 42 52 43 53
405 "vtrn.u8 d0, d1 \n"
406 "vtrn.u8 d4, d5 \n"
408 // d2 = 20 30 21 31 22 32 23 33
409 // d3 = 60 70 61 71 62 72 63 73
410 "vtrn.u8 d2, d3 \n"
411 "vtrn.u8 d6, d7 \n"
413 // d0 = 00+10 01+11 02+12 03+13
414 // d2 = 40+50 41+51 42+52 43+53
415 "vpaddl.u8 q0, q0 \n"
416 "vpaddl.u8 q2, q2 \n"
418 // d3 = 60+70 61+71 62+72 63+73
419 "vpaddl.u8 d3, d3 \n"
420 "vpaddl.u8 d7, d7 \n"
422 // combine source lines
423 "vadd.u16 q0, q2 \n"
424 "vadd.u16 d4, d3, d7 \n"
426 // dst_ptr[3] = (s[6] + s[7] + s[6+st] + s[7+st]) / 4
427 "vqrshrn.u16 d4, q2, #2 \n"
429 // Shuffle 2,3 reg around so that 2 can be added to the
430 // 0,1 reg and 3 can be added to the 4,5 reg. This
431 // requires expanding from u8 to u16 as the 0,1 and 4,5
432 // registers are already expanded. Then do transposes
433 // to get aligned.
434 // q2 = xx 20 xx 30 xx 21 xx 31 xx 22 xx 32 xx 23 xx 33
435 "vmovl.u8 q1, d2 \n"
436 "vmovl.u8 q3, d6 \n"
438 // combine source lines
439 "vadd.u16 q1, q3 \n"
441 // d4 = xx 20 xx 30 xx 22 xx 32
442 // d5 = xx 21 xx 31 xx 23 xx 33
443 "vtrn.u32 d2, d3 \n"
445 // d4 = xx 20 xx 21 xx 22 xx 23
446 // d5 = xx 30 xx 31 xx 32 xx 33
447 "vtrn.u16 d2, d3 \n"
449 // 0+1+2, 3+4+5
450 "vadd.u16 q0, q1 \n"
452 // Need to divide, but can't downshift as the the value
453 // isn't a power of 2. So multiply by 65536 / n
454 // and take the upper 16 bits.
455 "vqrdmulh.s16 q0, q0, q13 \n"
457 // Align for table lookup, vtbl requires registers to
458 // be adjacent
459 "vmov.u8 d2, d4 \n"
461 "vtbl.u8 d3, {d0, d1, d2}, d28 \n"
462 "vtbl.u8 d4, {d0, d1, d2}, d29 \n"
464 "vst1.8 {d3}, [%1]! \n"
465 "vst1.32 {d4[0]}, [%1]! \n"
466 "bgt 1b \n"
467 : "+r"(src_ptr), // %0
468 "+r"(dst_ptr), // %1
469 "+r"(dst_width), // %2
470 "+r"(src_stride) // %3
471 : "r"(&kMult38_Div6), // %4
472 "r"(&kShuf38_2) // %5
473 : "q0", "q1", "q2", "q3", "q13", "q14", "memory", "cc"
474 );
475 }
477 // 16x2 -> 16x1
478 void ScaleFilterRows_NEON(uint8* dst_ptr,
479 const uint8* src_ptr, ptrdiff_t src_stride,
480 int dst_width, int source_y_fraction) {
481 asm volatile (
482 "cmp %4, #0 \n"
483 "beq 100f \n"
484 "add %2, %1 \n"
485 "cmp %4, #64 \n"
486 "beq 75f \n"
487 "cmp %4, #128 \n"
488 "beq 50f \n"
489 "cmp %4, #192 \n"
490 "beq 25f \n"
492 "vdup.8 d5, %4 \n"
493 "rsb %4, #256 \n"
494 "vdup.8 d4, %4 \n"
495 // General purpose row blend.
496 "1: \n"
497 "vld1.8 {q0}, [%1]! \n"
498 "vld1.8 {q1}, [%2]! \n"
499 "subs %3, %3, #16 \n"
500 "vmull.u8 q13, d0, d4 \n"
501 "vmull.u8 q14, d1, d4 \n"
502 "vmlal.u8 q13, d2, d5 \n"
503 "vmlal.u8 q14, d3, d5 \n"
504 "vrshrn.u16 d0, q13, #8 \n"
505 "vrshrn.u16 d1, q14, #8 \n"
506 "vst1.8 {q0}, [%0]! \n"
507 "bgt 1b \n"
508 "b 99f \n"
510 // Blend 25 / 75.
511 "25: \n"
512 "vld1.8 {q0}, [%1]! \n"
513 "vld1.8 {q1}, [%2]! \n"
514 "subs %3, %3, #16 \n"
515 "vrhadd.u8 q0, q1 \n"
516 "vrhadd.u8 q0, q1 \n"
517 "vst1.8 {q0}, [%0]! \n"
518 "bgt 25b \n"
519 "b 99f \n"
521 // Blend 50 / 50.
522 "50: \n"
523 "vld1.8 {q0}, [%1]! \n"
524 "vld1.8 {q1}, [%2]! \n"
525 "subs %3, %3, #16 \n"
526 "vrhadd.u8 q0, q1 \n"
527 "vst1.8 {q0}, [%0]! \n"
528 "bgt 50b \n"
529 "b 99f \n"
531 // Blend 75 / 25.
532 "75: \n"
533 "vld1.8 {q1}, [%1]! \n"
534 "vld1.8 {q0}, [%2]! \n"
535 "subs %3, %3, #16 \n"
536 "vrhadd.u8 q0, q1 \n"
537 "vrhadd.u8 q0, q1 \n"
538 "vst1.8 {q0}, [%0]! \n"
539 "bgt 75b \n"
540 "b 99f \n"
542 // Blend 100 / 0 - Copy row unchanged.
543 "100: \n"
544 "vld1.8 {q0}, [%1]! \n"
545 "subs %3, %3, #16 \n"
546 "vst1.8 {q0}, [%0]! \n"
547 "bgt 100b \n"
549 "99: \n"
550 "vst1.8 {d1[7]}, [%0] \n"
551 : "+r"(dst_ptr), // %0
552 "+r"(src_ptr), // %1
553 "+r"(src_stride), // %2
554 "+r"(dst_width), // %3
555 "+r"(source_y_fraction) // %4
556 :
557 : "q0", "q1", "d4", "d5", "q13", "q14", "memory", "cc"
558 );
559 }
561 void ScaleARGBRowDown2_NEON(const uint8* src_ptr, ptrdiff_t src_stride,
562 uint8* dst, int dst_width) {
563 asm volatile (
564 ".p2align 2 \n"
565 "1: \n"
566 // load even pixels into q0, odd into q1
567 "vld2.32 {q0, q1}, [%0]! \n"
568 "vld2.32 {q2, q3}, [%0]! \n"
569 "subs %2, %2, #8 \n" // 8 processed per loop
570 "vst1.8 {q1}, [%1]! \n" // store odd pixels
571 "vst1.8 {q3}, [%1]! \n"
572 "bgt 1b \n"
573 : "+r"(src_ptr), // %0
574 "+r"(dst), // %1
575 "+r"(dst_width) // %2
576 :
577 : "memory", "cc", "q0", "q1", "q2", "q3" // Clobber List
578 );
579 }
581 void ScaleARGBRowDown2Box_NEON(const uint8* src_ptr, ptrdiff_t src_stride,
582 uint8* dst, int dst_width) {
583 asm volatile (
584 // change the stride to row 2 pointer
585 "add %1, %1, %0 \n"
586 ".p2align 2 \n"
587 "1: \n"
588 "vld4.8 {d0, d2, d4, d6}, [%0]! \n" // load 8 ARGB pixels.
589 "vld4.8 {d1, d3, d5, d7}, [%0]! \n" // load next 8 ARGB pixels.
590 "subs %3, %3, #8 \n" // 8 processed per loop.
591 "vpaddl.u8 q0, q0 \n" // B 16 bytes -> 8 shorts.
592 "vpaddl.u8 q1, q1 \n" // G 16 bytes -> 8 shorts.
593 "vpaddl.u8 q2, q2 \n" // R 16 bytes -> 8 shorts.
594 "vpaddl.u8 q3, q3 \n" // A 16 bytes -> 8 shorts.
595 "vld4.8 {d16, d18, d20, d22}, [%1]! \n" // load 8 more ARGB pixels.
596 "vld4.8 {d17, d19, d21, d23}, [%1]! \n" // load last 8 ARGB pixels.
597 "vpadal.u8 q0, q8 \n" // B 16 bytes -> 8 shorts.
598 "vpadal.u8 q1, q9 \n" // G 16 bytes -> 8 shorts.
599 "vpadal.u8 q2, q10 \n" // R 16 bytes -> 8 shorts.
600 "vpadal.u8 q3, q11 \n" // A 16 bytes -> 8 shorts.
601 "vrshrn.u16 d0, q0, #2 \n" // downshift, round and pack
602 "vrshrn.u16 d1, q1, #2 \n"
603 "vrshrn.u16 d2, q2, #2 \n"
604 "vrshrn.u16 d3, q3, #2 \n"
605 "vst4.8 {d0, d1, d2, d3}, [%2]! \n"
606 "bgt 1b \n"
607 : "+r"(src_ptr), // %0
608 "+r"(src_stride), // %1
609 "+r"(dst), // %2
610 "+r"(dst_width) // %3
611 :
612 : "memory", "cc", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11"
613 );
614 }
616 // Reads 4 pixels at a time.
617 // Alignment requirement: src_argb 4 byte aligned.
618 void ScaleARGBRowDownEven_NEON(const uint8* src_argb, ptrdiff_t src_stride,
619 int src_stepx, uint8* dst_argb, int dst_width) {
620 asm volatile (
621 "mov r12, %3, lsl #2 \n"
622 ".p2align 2 \n"
623 "1: \n"
624 "vld1.32 {d0[0]}, [%0], r12 \n"
625 "vld1.32 {d0[1]}, [%0], r12 \n"
626 "vld1.32 {d1[0]}, [%0], r12 \n"
627 "vld1.32 {d1[1]}, [%0], r12 \n"
628 "subs %2, %2, #4 \n" // 4 pixels per loop.
629 "vst1.8 {q0}, [%1]! \n"
630 "bgt 1b \n"
631 : "+r"(src_argb), // %0
632 "+r"(dst_argb), // %1
633 "+r"(dst_width) // %2
634 : "r"(src_stepx) // %3
635 : "memory", "cc", "r12", "q0"
636 );
637 }
639 // Reads 4 pixels at a time.
640 // Alignment requirement: src_argb 4 byte aligned.
641 void ScaleARGBRowDownEvenBox_NEON(const uint8* src_argb, ptrdiff_t src_stride,
642 int src_stepx,
643 uint8* dst_argb, int dst_width) {
644 asm volatile (
645 "mov r12, %4, lsl #2 \n"
646 "add %1, %1, %0 \n"
647 ".p2align 2 \n"
648 "1: \n"
649 "vld1.8 {d0}, [%0], r12 \n" // Read 4 2x2 blocks -> 2x1
650 "vld1.8 {d1}, [%1], r12 \n"
651 "vld1.8 {d2}, [%0], r12 \n"
652 "vld1.8 {d3}, [%1], r12 \n"
653 "vld1.8 {d4}, [%0], r12 \n"
654 "vld1.8 {d5}, [%1], r12 \n"
655 "vld1.8 {d6}, [%0], r12 \n"
656 "vld1.8 {d7}, [%1], r12 \n"
657 "vaddl.u8 q0, d0, d1 \n"
658 "vaddl.u8 q1, d2, d3 \n"
659 "vaddl.u8 q2, d4, d5 \n"
660 "vaddl.u8 q3, d6, d7 \n"
661 "vswp.8 d1, d2 \n" // ab_cd -> ac_bd
662 "vswp.8 d5, d6 \n" // ef_gh -> eg_fh
663 "vadd.u16 q0, q0, q1 \n" // (a+b)_(c+d)
664 "vadd.u16 q2, q2, q3 \n" // (e+f)_(g+h)
665 "vrshrn.u16 d0, q0, #2 \n" // first 2 pixels.
666 "vrshrn.u16 d1, q2, #2 \n" // next 2 pixels.
667 "subs %3, %3, #4 \n" // 4 pixels per loop.
668 "vst1.8 {q0}, [%2]! \n"
669 "bgt 1b \n"
670 : "+r"(src_argb), // %0
671 "+r"(src_stride), // %1
672 "+r"(dst_argb), // %2
673 "+r"(dst_width) // %3
674 : "r"(src_stepx) // %4
675 : "memory", "cc", "r12", "q0", "q1", "q2", "q3"
676 );
677 }
679 #endif // __ARM_NEON__
681 #ifdef __cplusplus
682 } // extern "C"
683 } // namespace libyuv
684 #endif
