michael@0: /* NEON optimized code (C) COPYRIGHT 2009 Motorola
michael@0:  *
michael@0:  * Use of this source code is governed by a BSD-style license that can be
michael@0:  * found in the LICENSE file.
michael@0:  */
michael@0: 
michael@0: #include "SkBitmapProcState.h"
michael@0: #include "SkPerspIter.h"
michael@0: #include "SkShader.h"
michael@0: #include "SkUtilsArm.h"
michael@0: #include "SkBitmapProcState_utils.h"
michael@0: 
michael@0: #include <arm_neon.h>
michael@0: 
michael@0: extern const SkBitmapProcState::MatrixProc ClampX_ClampY_Procs_neon[];
michael@0: extern const SkBitmapProcState::MatrixProc RepeatX_RepeatY_Procs_neon[];
michael@0: 
michael@0: static void decal_nofilter_scale_neon(uint32_t dst[], SkFixed fx, SkFixed dx, int count);
michael@0: static void decal_filter_scale_neon(uint32_t dst[], SkFixed fx, SkFixed dx, int count);
michael@0: 
michael@0: // TILEX_PROCF(fx, max)    SkClampMax((fx) >> 16, max)
michael@0: static inline int16x8_t sbpsm_clamp_tile8(int32x4_t low, int32x4_t high, unsigned max) {
michael@0:     int16x8_t res;
michael@0: 
michael@0:     // get the hi 16s of all those 32s
michael@0:     res = vuzpq_s16(vreinterpretq_s16_s32(low), vreinterpretq_s16_s32(high)).val[1];
michael@0: 
michael@0:     // clamp
michael@0:     res = vmaxq_s16(res, vdupq_n_s16(0));
michael@0:     res = vminq_s16(res, vdupq_n_s16(max));
michael@0: 
michael@0:     return res;
michael@0: }
michael@0: 
michael@0: // TILEX_PROCF(fx, max)    SkClampMax((fx) >> 16, max)
michael@0: static inline int32x4_t sbpsm_clamp_tile4(int32x4_t f, unsigned max) {
michael@0:     int32x4_t res;
michael@0: 
michael@0:     // get the hi 16s of all those 32s
michael@0:     res = vshrq_n_s32(f, 16);
michael@0: 
michael@0:     // clamp
michael@0:     res = vmaxq_s32(res, vdupq_n_s32(0));
michael@0:     res = vminq_s32(res, vdupq_n_s32(max));
michael@0: 
michael@0:     return res;
michael@0: }
michael@0: 
michael@0: // TILEY_LOW_BITS(fy, max)         (((fy) >> 12) & 0xF)
michael@0: static inline int32x4_t sbpsm_clamp_tile4_low_bits(int32x4_t fx) {
michael@0:     int32x4_t ret;
michael@0: 
michael@0:     ret = vshrq_n_s32(fx, 12);
michael@0: 
michael@0:     /* We don't need the mask below because the caller will
michael@0:      * overwrite the non-masked bits
michael@0:      */
michael@0:     //ret = vandq_s32(ret, vdupq_n_s32(0xF));
michael@0: 
michael@0:     return ret;
michael@0: }
michael@0: 
michael@0: // TILEX_PROCF(fx, max) (((fx)&0xFFFF)*((max)+1)>> 16)
michael@0: static inline int16x8_t sbpsm_repeat_tile8(int32x4_t low, int32x4_t high, unsigned max) {
michael@0:     uint16x8_t res;
michael@0:     uint32x4_t tmpl, tmph;
michael@0: 
michael@0:     // get the lower 16 bits
michael@0:     res = vuzpq_u16(vreinterpretq_u16_s32(low), vreinterpretq_u16_s32(high)).val[0];
michael@0: 
michael@0:     // bare multiplication, not SkFixedMul
michael@0:     tmpl = vmull_u16(vget_low_u16(res), vdup_n_u16(max+1));
michael@0:     tmph = vmull_u16(vget_high_u16(res), vdup_n_u16(max+1));
michael@0: 
michael@0:     // extraction of the 16 upper bits
michael@0:     res = vuzpq_u16(vreinterpretq_u16_u32(tmpl), vreinterpretq_u16_u32(tmph)).val[1];
michael@0: 
michael@0:     return vreinterpretq_s16_u16(res);
michael@0: }
michael@0: 
michael@0: // TILEX_PROCF(fx, max) (((fx)&0xFFFF)*((max)+1)>> 16)
michael@0: static inline int32x4_t sbpsm_repeat_tile4(int32x4_t f, unsigned max) {
michael@0:     uint16x4_t res;
michael@0:     uint32x4_t tmp;
michael@0: 
michael@0:     // get the lower 16 bits
michael@0:     res = vmovn_u32(vreinterpretq_u32_s32(f));
michael@0: 
michael@0:     // bare multiplication, not SkFixedMul
michael@0:     tmp = vmull_u16(res, vdup_n_u16(max+1));
michael@0: 
michael@0:     // extraction of the 16 upper bits
michael@0:     tmp = vshrq_n_u32(tmp, 16);
michael@0: 
michael@0:     return vreinterpretq_s32_u32(tmp);
michael@0: }
michael@0: 
michael@0: // TILEX_LOW_BITS(fx, max)         ((((fx) & 0xFFFF) * ((max) + 1) >> 12) & 0xF)
michael@0: static inline int32x4_t sbpsm_repeat_tile4_low_bits(int32x4_t fx, unsigned max) {
michael@0:     uint16x4_t res;
michael@0:     uint32x4_t tmp;
michael@0:     int32x4_t ret;
michael@0: 
michael@0:     // get the lower 16 bits
michael@0:     res = vmovn_u32(vreinterpretq_u32_s32(fx));
michael@0: 
michael@0:     // bare multiplication, not SkFixedMul
michael@0:     tmp = vmull_u16(res, vdup_n_u16(max + 1));
michael@0: 
michael@0:     // shift and mask
michael@0:     ret = vshrq_n_s32(vreinterpretq_s32_u32(tmp), 12);
michael@0: 
michael@0:     /* We don't need the mask below because the caller will
michael@0:      * overwrite the non-masked bits
michael@0:      */
michael@0:     //ret = vandq_s32(ret, vdupq_n_s32(0xF));
michael@0: 
michael@0:     return ret;
michael@0: }
michael@0: 
michael@0: #define MAKENAME(suffix)                ClampX_ClampY ## suffix ## _neon
michael@0: #define TILEX_PROCF(fx, max)            SkClampMax((fx) >> 16, max)
michael@0: #define TILEY_PROCF(fy, max)            SkClampMax((fy) >> 16, max)
michael@0: #define TILEX_PROCF_NEON8(l, h, max)    sbpsm_clamp_tile8(l, h, max)
michael@0: #define TILEY_PROCF_NEON8(l, h, max)    sbpsm_clamp_tile8(l, h, max)
michael@0: #define TILEX_PROCF_NEON4(fx, max)      sbpsm_clamp_tile4(fx, max)
michael@0: #define TILEY_PROCF_NEON4(fy, max)      sbpsm_clamp_tile4(fy, max)
michael@0: #define TILEX_LOW_BITS(fx, max)         (((fx) >> 12) & 0xF)
michael@0: #define TILEY_LOW_BITS(fy, max)         (((fy) >> 12) & 0xF)
michael@0: #define TILEX_LOW_BITS_NEON4(fx, max)   sbpsm_clamp_tile4_low_bits(fx)
michael@0: #define TILEY_LOW_BITS_NEON4(fy, max)   sbpsm_clamp_tile4_low_bits(fy)
michael@0: #define CHECK_FOR_DECAL
michael@0: #include "SkBitmapProcState_matrix_neon.h"
michael@0: 
michael@0: #define MAKENAME(suffix)                RepeatX_RepeatY ## suffix ## _neon
michael@0: #define TILEX_PROCF(fx, max)            SK_USHIFT16(((fx) & 0xFFFF) * ((max) + 1))
michael@0: #define TILEY_PROCF(fy, max)            SK_USHIFT16(((fy) & 0xFFFF) * ((max) + 1))
michael@0: #define TILEX_PROCF_NEON8(l, h, max)    sbpsm_repeat_tile8(l, h, max)
michael@0: #define TILEY_PROCF_NEON8(l, h, max)    sbpsm_repeat_tile8(l, h, max)
michael@0: #define TILEX_PROCF_NEON4(fx, max)      sbpsm_repeat_tile4(fx, max)
michael@0: #define TILEY_PROCF_NEON4(fy, max)      sbpsm_repeat_tile4(fy, max)
michael@0: #define TILEX_LOW_BITS(fx, max)         ((((fx) & 0xFFFF) * ((max) + 1) >> 12) & 0xF)
michael@0: #define TILEY_LOW_BITS(fy, max)         ((((fy) & 0xFFFF) * ((max) + 1) >> 12) & 0xF)
michael@0: #define TILEX_LOW_BITS_NEON4(fx, max)   sbpsm_repeat_tile4_low_bits(fx, max)
michael@0: #define TILEY_LOW_BITS_NEON4(fy, max)   sbpsm_repeat_tile4_low_bits(fy, max)
michael@0: #include "SkBitmapProcState_matrix_neon.h"
michael@0: 
michael@0: 
michael@0: 
michael@0: void decal_nofilter_scale_neon(uint32_t dst[], SkFixed fx, SkFixed dx, int count) {
michael@0:     if (count >= 8) {
michael@0:         // SkFixed is 16.16 fixed point
michael@0:         SkFixed dx8 = dx * 8;
michael@0:         int32x4_t vdx8 = vdupq_n_s32(dx8);
michael@0: 
michael@0:         // setup lbase and hbase
michael@0:         int32x4_t lbase, hbase;
michael@0:         lbase = vdupq_n_s32(fx);
michael@0:         lbase = vsetq_lane_s32(fx + dx, lbase, 1);
michael@0:         lbase = vsetq_lane_s32(fx + dx + dx, lbase, 2);
michael@0:         lbase = vsetq_lane_s32(fx + dx + dx + dx, lbase, 3);
michael@0:         hbase = lbase + vdupq_n_s32(4 * dx);
michael@0: 
michael@0:         do {
michael@0:             // store the upper 16 bits
michael@0:             vst1q_u32(dst, vreinterpretq_u32_s16(
michael@0:                 vuzpq_s16(vreinterpretq_s16_s32(lbase), vreinterpretq_s16_s32(hbase)).val[1]
michael@0:             ));
michael@0: 
michael@0:             // on to the next group of 8
michael@0:             lbase += vdx8;
michael@0:             hbase += vdx8;
michael@0:             dst += 4; // we did 8 elements but the result is twice smaller
michael@0:             count -= 8;
michael@0:             fx += dx8;
michael@0:         } while (count >= 8);
michael@0:     }
michael@0: 
michael@0:     uint16_t* xx = (uint16_t*)dst;
michael@0:     for (int i = count; i > 0; --i) {
michael@0:         *xx++ = SkToU16(fx >> 16); fx += dx;
michael@0:     }
michael@0: }
michael@0: 
michael@0: void decal_filter_scale_neon(uint32_t dst[], SkFixed fx, SkFixed dx, int count) {
michael@0:     if (count >= 8) {
michael@0:         SkFixed dx8 = dx * 8;
michael@0:         int32x4_t vdx8 = vdupq_n_s32(dx8);
michael@0: 
michael@0:         int32x4_t wide_fx, wide_fx2;
michael@0:         wide_fx = vdupq_n_s32(fx);
michael@0:         wide_fx = vsetq_lane_s32(fx + dx, wide_fx, 1);
michael@0:         wide_fx = vsetq_lane_s32(fx + dx + dx, wide_fx, 2);
michael@0:         wide_fx = vsetq_lane_s32(fx + dx + dx + dx, wide_fx, 3);
michael@0: 
michael@0:         wide_fx2 = vaddq_s32(wide_fx, vdupq_n_s32(4 * dx));
michael@0: 
michael@0:         while (count >= 8) {
michael@0:             int32x4_t wide_out;
michael@0:             int32x4_t wide_out2;
michael@0: 
michael@0:             wide_out = vshlq_n_s32(vshrq_n_s32(wide_fx, 12), 14);
michael@0:             wide_out = wide_out | (vshrq_n_s32(wide_fx,16) + vdupq_n_s32(1));
michael@0: 
michael@0:             wide_out2 = vshlq_n_s32(vshrq_n_s32(wide_fx2, 12), 14);
michael@0:             wide_out2 = wide_out2 | (vshrq_n_s32(wide_fx2,16) + vdupq_n_s32(1));
michael@0: 
michael@0:             vst1q_u32(dst, vreinterpretq_u32_s32(wide_out));
michael@0:             vst1q_u32(dst+4, vreinterpretq_u32_s32(wide_out2));
michael@0: 
michael@0:             dst += 8;
michael@0:             fx += dx8;
michael@0:             wide_fx += vdx8;
michael@0:             wide_fx2 += vdx8;
michael@0:             count -= 8;
michael@0:         }
michael@0:     }
michael@0: 
michael@0:     if (count & 1)
michael@0:     {
michael@0:         SkASSERT((fx >> (16 + 14)) == 0);
michael@0:         *dst++ = (fx >> 12 << 14) | ((fx >> 16) + 1);
michael@0:         fx += dx;
michael@0:     }
michael@0:     while ((count -= 2) >= 0)
michael@0:     {
michael@0:         SkASSERT((fx >> (16 + 14)) == 0);
michael@0:         *dst++ = (fx >> 12 << 14) | ((fx >> 16) + 1);
michael@0:         fx += dx;
michael@0: 
michael@0:         *dst++ = (fx >> 12 << 14) | ((fx >> 16) + 1);
michael@0:         fx += dx;
michael@0:     }
michael@0: }